fix build

.coveragerc

 # .coveragerc for edx-video-pipeline
 [run]
-data_file = reports/.coverage
+data_file = .coverage
 source =
+    VEDA
     VEDA_OS01
     control
     frontend
-    youtube_callback
-    scripts
+# do not calculate coverage for these for now
+#    youtube_callback
+#    scripts
 
 omit =
     templates/*
     frontend/tests/*
     dependencies/*
-    control/*
+    control/tests/*
     VEDA/tests/*
+    VEDA_OS01/tests/*
 
 concurrency=multiprocessing
 
@@ -28,5 +31,3 @@ exclude_lines =
 title = edx-video-worker Python Test Coverage Report
 directory = reports/cover
 
-[xml]
-output = reports/coverage.xml

.gitignore

@@ -13,3 +13,7 @@ coverage/
 
 # TODO remove this once migrations are shipped with the code
 */migrations/*
+
+reports/
+
+.cache/

.travis.yml

@@ -5,13 +5,14 @@ python:
 
 sudo: required
 
+before_install:
+  - export BOTO_CONFIG=/dev/null
+
 install:
-  - pip install -r requirements.txt
-  - pip install -r test_requirements.txt
+  - make requirements
 
-# build tests
 script:
-    - make validate
+  - make validate
 
 after_success:
   - pip install -U codecov

AUTHORS

 Mushtaq Ali <mushtaak@gmail.com>
+Muhammad Ammar <mammar@gmail.com>

Makefile

 PACKAGES = VEDA VEDA_OS01 control frontend youtube_callback scripts
 
+requirements:
+	pip install -r requirements.txt
+	pip install -r test_requirements.txt
+
 validate: test ## Run tests and quality checks
 
 test: clean
-	 nosetests --with-coverage --cover-inclusive --cover-branches \
-		--cover-html --cover-html-dir=build/coverage/html/ \
-		--cover-xml --cover-xml-file=build/coverage/coverage.xml --verbosity=2 \
-		$(foreach package,$(PACKAGES),--cover-package=$(package)) \
-		$(PACKAGES)
+	coverage run -m pytest --durations=10
+	coverage combine
+	coverage report
 
 clean:
 	coverage erase

VEDA/test_settings.py

+"""
+Test Settings
+
+"""
+from settings import *
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': 'pipeline.db',
+    }
+}

VEDA_OS01/tests/test_transcripts.py

@@ -14,26 +14,12 @@ from moto import mock_s3_deprecated
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from VEDA_OS01 import transcripts
+from VEDA_OS01 import transcripts, utils
 from VEDA_OS01.models import (Course, TranscriptPreferences,
                               TranscriptProcessMetadata, TranscriptProvider,
                               TranscriptStatus, Video)
 
-CONFIG_DATA = {
-    'cielo24_get_caption_url': 'http://api.cielo24.com/job/get_caption',
-    'transcript_bucket_access_key': 'bucket_access_key',
-    'transcript_bucket_secret_key': 'bucket_secret_key',
-    'transcript_bucket_name': 'bucket_name',
-    'val_token_url': 'http://val.edx.org/token',
-    'val_username': 'val_username',
-    'val_password': 'val_password',
-    'val_client_id': 'val_client_id',
-    'val_secret_key': 'val_secret_key',
-    'val_transcript_create_url': 'http://val.edx.org/transcript/create',
-    'val_video_transcript_status_url': 'http://val.edx.org/video/status',
-    'transcript_provider_request_token': '1234a5a67cr890',
-    'transcript_bucket_directory': 'video-transcripts/',
-}
+CONFIG_DATA = utils.get_config('test_config.yaml')
 
 VIDEO_DATA = {
     'studio_id': '12345'
@@ -217,7 +203,7 @@ class Cielo24TranscriptTests(APITestCase):
         responses.add(responses.PATCH, CONFIG_DATA['val_video_transcript_status_url'], status=200)
 
         # create s3 bucket -- all this is happening in moto's virtual environment
-        conn = S3Connection(CONFIG_DATA['transcript_bucket_access_key'], CONFIG_DATA['transcript_bucket_secret_key'])
+        conn = S3Connection()
         conn.create_bucket(CONFIG_DATA['transcript_bucket_name'])
 
         transcripts.cielo24_transcript_callback(None, **REQUEST_PARAMS)

control/tests/test_deliver_cielo.py

 """
 Cielo24 transcription testing
 """
-from unittest import TestCase
+from django.test import TestCase
 
 import responses
 from ddt import ddt
@@ -13,22 +13,6 @@ from VEDA_OS01.models import (Cielo24Fidelity, Cielo24Turnaround, Course,
                               Video)
 from VEDA_OS01.utils import build_url
 
-CONFIG_DATA = {
-    'cielo24_get_caption_url': 'http://api.cielo24.com/job/get_caption',
-    'transcript_bucket_access_key': 'bucket_access_key',
-    'transcript_bucket_secret_key': 'bucket_secret_key',
-    'transcript_bucket_name': 'bucket_name',
-    'val_token_url': 'http://val.edx.org/token',
-    'val_username': 'username',
-    'val_password': 'password',
-    'val_client_id': 'client',
-    'val_secret_key': 'secret',
-    'val_transcript_create_url': 'http://val.edx.org/transcript/create',
-    'val_video_transcript_status_url': 'http://val.edx.org/video/status',
-    'veda_base_url': 'https://veda.edx.org',
-    'transcript_provider_request_token': '1234a5a67cr890'
-}
-
 VIDEO_DATA = {
     'studio_id': '12345'
 }

control/tests/test_encode.py

@@ -2,6 +2,10 @@
 import os
 import sys
 import unittest
+from django.test import TestCase
+
+from control.veda_encode import VedaEncode
+from VEDA_OS01.models import URL, Course, Destination, Encode, Video
 
 """
 Test encode profiler
@@ -12,18 +16,39 @@ sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
 
-from control.veda_encode import VedaEncode
-from VEDA_OS01.models import Course, URL, Video, Encode
+
+VIDEO_DATA = {
+    'studio_id': '12345'
+}
 
 
-class TestEncode(unittest.TestCase):
+class TestEncode(TestCase):
 
     def setUp(self):
-        self.course_object = Course.objects.get(
+        self.veda_id = 'XXXXXXXX2016-V00TEST'
+        self.course_object = Course.objects.create(
             institution='XXX',
             edx_classid='XXXXX'
         )
-        self.veda_id = 'XXXXXXXX2016-V00TEST'
+
+        self.video = Video.objects.create(
+            inst_class=self.course_object,
+            studio_id='12345',
+            edx_id=self.veda_id,
+        )
+
+        Encode.objects.create(
+            product_spec='mobile_low',
+            encode_destination=Destination.objects.create(destination_name='destination_name'),
+            profile_active=True
+        )
+
+        Encode.objects.create(
+            product_spec='desktop_mp4',
+            encode_destination=Destination.objects.create(destination_name='destination_name'),
+            profile_active=True
+        )
+
         self.E = VedaEncode(
             course_object=self.course_object,
             veda_id=self.veda_id
@@ -32,7 +57,6 @@ class TestEncode(unittest.TestCase):
     def test_encode_url(self):
         """
         gen baseline, gen a url, test against baseline
-
         """
         URL.objects.filter(
             videoID=Video.objects.filter(edx_id=self.veda_id).latest()
@@ -41,18 +65,17 @@ class TestEncode(unittest.TestCase):
         baseline = len(encode_list)
         self.assertTrue(isinstance(encode_list, list))
 
-        self.E.encode_list = []
-        U = URL(
+        self.E.encode_list = set()
+        url = URL(
             videoID=Video.objects.filter(edx_id=self.veda_id).latest(),
             encode_profile=Encode.objects.get(product_spec='mobile_low'),
             encode_url='THIS Is A TEST'
         )
-        U.save()
+        url.save()
         encode_list = self.E.determine_encodes()
         self.assertTrue(len(encode_list) == baseline - 1)
 
-        self.E.encode_list = []
-
+        self.E.encode_list = set()
         URL.objects.filter(
             videoID=Video.objects.filter(edx_id=self.veda_id).latest(),
         ).delete()

control/tests/test_file_discovery.py

 import os
 import sys
 import unittest
+from django.test import TestCase
 
 """
 Test VEDA API
@@ -12,7 +13,7 @@ sys.path.append(os.path.dirname(os.path.dirname(
 from control.veda_file_discovery import FileDiscovery
 
 
-class TestValidation(unittest.TestCase):
+class TestValidation(TestCase):
 
     def setUp(self):
 

control/tests/test_file_ingest.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import unittest
+from django.test import TestCase
 import requests
 import ast
 import yaml
@@ -17,7 +18,7 @@ from veda_file_ingest import VideoProto, VedaIngest
 requests.packages.urllib3.disable_warnings()
 
 
-class TestIngest(unittest.TestCase):
+class TestIngest(TestCase):
 
     def setUp(self):
         self.VP = VideoProto(

control/tests/test_heal.py

-
+"""
+Test heal processor
+"""
+import datetime
+import json
 import os
 import sys
-import unittest
+from django.test import TestCase
+from datetime import timedelta
 
-"""
-Test heal processor
+import responses
+from django.utils.timezone import utc
+from mock import PropertyMock, patch
+
+from control.veda_heal import VedaHeal
+from VEDA_OS01.models import URL, Course, Destination, Encode, Video
+from VEDA_OS01.utils import build_url, get_config
 
-"""
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
-from control.veda_heal import VedaHeal
-from VEDA_OS01.models import URL, Video, Encode
+
+CONFIG_DATA = get_config('test_config.yaml')
 
 
-class TestEncode(unittest.TestCase):
+class TestHeal(TestCase):
 
     def setUp(self):
-        U = URL(
-            videoID=Video.objects.filter(edx_id='XXXXXXXX2014-V00TES1').latest(),
-            encode_profile=Encode.objects.get(product_spec='mobile_low'),
-            encode_url='THIS Is A TEST')
-        U.save()
+        self.video_id = '12345'
+        self.course_object = Course.objects.create(
+            institution='XXX',
+            edx_classid='XXXXX',
+            local_storedir='WestonHS/PFLC1x/3T2015'
+        )
 
-    def test_encode_url(self):
-        H = VedaHeal()
-        H.discovery()
+        self.video = Video.objects.create(
+            inst_class=self.course_object,
+            studio_id=self.video_id,
+            edx_id='XXXXXXXX2014-V00TES1',
+            video_trans_start=datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(
+                hours=CONFIG_DATA['heal_start']
+            ),
+            video_trans_end=datetime.datetime.utcnow().replace(tzinfo=utc),
+        )
 
+        self.encode = Encode.objects.create(
+            product_spec='mobile_low',
+            encode_destination=Destination.objects.create(destination_name='destination_name')
+        )
 
-def main():
-    unittest.main()
+        url = URL(
+            videoID=self.video,
+            encode_profile=self.encode,
+            encode_bitdepth='22',
+            encode_url='http://veda.edx.org/encode')
+        url.save()
 
+    @patch('control.veda_heal.VALAPICall._AUTH', PropertyMock(return_value=lambda: CONFIG_DATA))
+    @responses.activate
+    def test_heal(self):
+        val_response = {
+            'courses': [{u'WestonHS/PFLC1x/3T2015': None}],
+            'encoded_videos': [{
+                'url': 'https://testurl.mp4',
+                'file_size': 8499040,
+                'bitrate': 131,
+                'profile': 'mobile_low',
+            }]
+        }
+        responses.add(
+            responses.POST,
+            CONFIG_DATA['val_token_url'],
+            '{"access_token": "1234567890"}',
+            status=200
+        )
+        responses.add(
+            responses.GET,
+            build_url(CONFIG_DATA['val_api_url'], self.video_id),
+            body=json.dumps(val_response),
+            content_type='application/json',
+            status=200
+        )
+        responses.add(
+            responses.PUT,
+            build_url(CONFIG_DATA['val_api_url'], self.video_id),
+            status=200
+        )
 
-if __name__ == '__main__':
-    sys.exit(main())
+        heal = VedaHeal()
+        heal.discovery()

control/tests/test_hotstore.py

+"""
+Test upload processes
+"""
 
 import os
 import sys
-import unittest
+from django.test import TestCase
 
-"""
-Test upload processes
+from boto.s3.connection import S3Connection
+from mock import PropertyMock, patch
+from moto import mock_s3_deprecated
+from VEDA_OS01 import utils
+
+from control.veda_file_ingest import VideoProto
+from control.veda_hotstore import Hotstore
 
-"""
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
 )))
 
-from control.veda_hotstore import Hotstore
-from control.veda_file_ingest import VideoProto
+
+CONFIG_DATA = utils.get_config('test_config.yaml')
 
 
-class TestHotstore(unittest.TestCase):
+class TestHotstore(TestCase):
 
     def setUp(self):
-        VP = VideoProto()
-        VP.veda_id = 'XXXXXXXX2014-V00TEST'
+        video_proto = VideoProto()
+        video_proto.veda_id = 'XXXXXXXX2014-V00TEST'
         self.upload_filepath = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
             'test_files',
             'OVTESTFILE_01.mp4'
         )
 
-        self.H1 = Hotstore(
-            video_object=VP,
-            upload_filepath=self.upload_filepath,
-            video_proto=VP
-        )
+        with patch.object(Hotstore, '_READ_AUTH', PropertyMock(return_value=lambda: CONFIG_DATA)):
+            self.hotstore = Hotstore(
+                video_object=video_proto,
+                upload_filepath=self.upload_filepath,
+                video_proto=video_proto
+            )
+
+        # do s3 mocking
+        mock = mock_s3_deprecated()
+        mock.start()
+        conn = S3Connection()
+        conn.create_bucket(CONFIG_DATA['veda_s3_hotstore_bucket'])
+        self.addCleanup(mock.stop)
 
     def test_single_upload(self):
-        if self.H1.auth_dict is None:
-            self.assertTrue(self.H1.upload() is False)
+        """
+        Verify S3 single part upload.
+        """
+        if self.hotstore.auth_dict is None:
+            self.assertTrue(self.hotstore.upload() is False)
             return None
 
-        self.assertTrue(self.H1.upload())
+        self.hotstore.auth_dict['multi_upload_barrier'] = os.stat(self.upload_filepath).st_size + 1
+        self.assertTrue(self.hotstore.upload())
 
     def test_multi_upload(self):
-        if self.H1.auth_dict is None:
-            self.assertTrue(self.H1.upload() is None)
+        """
+        Verify S3 single multi-part upload.
+        """
+        if self.hotstore.auth_dict is None:
+            self.assertTrue(self.hotstore.upload() is None)
             return None
 
-        self.H1.auth_dict['multi_upload_barrier'] = 0
-        self.assertTrue(self.H1.upload())
-
-
-def main():
-    unittest.main()
-
-
-if __name__ == '__main__':
-    sys.exit(main())
+        self.hotstore.auth_dict['multi_upload_barrier'] = 0
+        self.assertTrue(self.hotstore.upload())

control/tests/test_reporting.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import unittest
+from django.test import TestCase
 
 """
 A basic unittest for the "Course Addition Tool"
@@ -14,7 +15,7 @@ sys.path.append(
 from veda_utils import Report
 
 
-class TestReporting(unittest.TestCase):
+class TestReporting(TestCase):
 
     def setUp(self):
         self.R = Report(

control/tests/test_val.py

 
+import ast
 import os
 import sys
-import unittest
+from django.test import TestCase
+
 import requests
-import ast
+import responses
 import yaml
+from mock import PropertyMock, patch
+
+from control.veda_val import VALAPICall
+from veda_file_ingest import VideoProto
+from VEDA_OS01 import utils
 
 requests.packages.urllib3.disable_warnings()
 """
 This is an API connection test
 set to pass if instance_config.yaml is missing
-
 """
 
 sys.path.append(os.path.dirname(os.path.dirname(__file__)))
-from veda_val import VALAPICall
-from veda_file_ingest import VideoProto
+
+CONFIG_DATA = utils.get_config('test_config.yaml')
 
 
-class TestVALAPI(unittest.TestCase):
+class TestVALAPI(TestCase):
 
     def setUp(self):
         self.VP = VideoProto(
@@ -26,10 +32,11 @@ class TestVALAPI(unittest.TestCase):
             veda_id='TESTID'
         )
 
-        self.VAC = VALAPICall(
-            video_proto=self.VP,
-            val_status='complete'
-        )
+        with patch.object(VALAPICall, '_AUTH', PropertyMock(return_value=lambda: CONFIG_DATA)):
+            self.VAC = VALAPICall(
+                video_proto=self.VP,
+                val_status='complete'
+            )
 
         self.auth_yaml = os.path.join(
             os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
@@ -41,6 +48,9 @@ class TestVALAPI(unittest.TestCase):
             self.assertTrue(self.VAC.auth_dict is None)
             return None
 
+        # register val url to send api response
+        responses.add(responses.POST, CONFIG_DATA['val_token_url'], '{"access_token": "1234567890"}', status=200)
+
         salient_variables = [
             'val_api_url',
             'val_client_id',
@@ -49,30 +59,27 @@ class TestVALAPI(unittest.TestCase):
             'val_username',
             'val_token_url',
         ]
-        for s in salient_variables:
-            self.assertTrue(len(self.VAC.auth_dict[s]) > 0)
+        for salient_variable in salient_variables:
+            self.assertTrue(len(self.VAC.auth_dict[salient_variable]) > 0)
 
+    @responses.activate
     def test_val_connection(self):
         if not os.path.exists(self.auth_yaml):
             self.assertTrue(self.VAC.auth_dict is None)
             return None
 
+        # register val url to send api response
+        responses.add(responses.POST, CONFIG_DATA['val_token_url'], '{"access_token": "1234567890"}', status=200)
+        responses.add(responses.GET, CONFIG_DATA['val_api_url'], status=200)
+
         self.VAC.val_tokengen()
         self.assertFalse(self.VAC.val_token is None)
 
-        s = requests.get(
+        response = requests.get(
             self.VAC.auth_dict['val_api_url'],
             headers=self.VAC.headers,
             timeout=20
         )
 
-        self.assertFalse(s.status_code == 404)
-        self.assertFalse(s.status_code > 299)
-
-
-def main():
-    unittest.main()
-
-
-if __name__ == '__main__':
-    sys.exit(main())
+        self.assertFalse(response.status_code == 404)
+        self.assertFalse(response.status_code > 299)

control/tests/test_video_validation.py

 import os
 import sys
 import unittest
+from django.test import TestCase
 
 """
 Test VEDA API
@@ -12,7 +13,7 @@ sys.path.append(os.path.dirname(os.path.dirname(
 from control.veda_video_validation import Validation
 
 
-class TestValidation(unittest.TestCase):
+class TestValidation(TestCase):
     """
     Test class for Validation
     """
@@ -28,6 +29,10 @@ class TestValidation(unittest.TestCase):
             videofile=self.videofile
         )
 
+    @unittest.skipIf(
+        'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true',
+        'Skipping this test on Travis CI due to unavailability of required ffprobe version.'
+    )
     def test_validation(self):
         """
         Check a known file for validity

control/veda_deliver.py

@@ -3,6 +3,7 @@ import datetime
 import ftplib
 import logging
 import os
+import shutil
 import sys
 from os.path import expanduser
 
@@ -19,7 +20,8 @@ from control_env import *
 from veda_deliver_cielo import Cielo24Transcript
 from veda_deliver_youtube import DeliverYoutube
 from VEDA_OS01 import utils
-from VEDA_OS01.models import TranscriptPreferences, VideoStatus, TranscriptProvider
+from VEDA_OS01.models import (TranscriptPreferences, TranscriptProvider,
+                              VideoStatus)
 from VEDA_OS01.utils import build_url
 from veda_utils import ErrorObject, Metadata, Output, VideoProto
 from veda_val import VALAPICall

frontend/tests/test_abvid_reporting.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import unittest
+from django.test import TestCase
 
 """
 A basic unittest for the "Course Addition Tool"
@@ -14,7 +15,7 @@ sys.path.append(
 import abvid_reporting
 from frontend.course_validate import VEDACat
 
-class TestVariables(unittest.TestCase):
+class TestVariables(TestCase):
 
     def setUp(self):
         self.VCT = VEDACat()

frontend/tests/test_cat_tool.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import unittest
+from django.test import TestCase
 
 """
 A basic unittest for the "Course Addition Tool"
@@ -14,7 +15,7 @@ sys.path.append(
 from course_validate import VEDACat
 
 
-class TestVariables(unittest.TestCase):
+class TestVariables(TestCase):
 
     def setUp(self):
         self.VCT = VEDACat()

instance_config.yaml

@@ -45,7 +45,7 @@ s3_base_url: https://s3.amazonaws.com
 
 # transcript bucket config
 transcript_bucket_name:
-transcript_bucket_directory:  video-transcripts/
+transcript_bucket_directory: video-transcripts/
 
 
 # a token identifying a valid request from transcript provider

pytest.ini

+[pytest]
+DJANGO_SETTINGS_MODULE = VEDA.test_settings

test_config.yaml

+---
+veda_s3_hotstore_bucket: s3_hotstore_bucket
+multi_upload_barrier: 2000000000
+veda_base_url: https://veda.edx.org
+s3_base_url: https://s3.amazonaws.com
+
+
+# transcript bucket config
+transcript_bucket_name: bucket_name
+transcript_bucket_directory:  video-transcripts/
+
+
+# a token identifying a valid request from transcript provider
+transcript_provider_request_token: 1234a5a67cr890
+
+# ---
+# VAL
+# ---
+val_api_url: http://val.edx.org/api
+val_token_url: http://val.edx.org/token
+val_video_images_url:
+# Credentials
+val_client_id: client
+val_secret_key: secret
+val_password: password
+val_username: username
+val_transcript_create_url: http://val.edx.org/transcript/create
+val_video_transcript_status_url: http://val.edx.org/video/status
+
+# This is a list of encode profiles and their val profile matches
+# boolean matches
+val_profile_dict:
+    mobile_low:
+        - mobile_low
+
+    desktop_mp4:
+        - desktop_mp4
+
+    override:
+        - desktop_mp4
+        - mobile_low
+        - mobile_high
+
+    mobile_high:
+        - mobile_high
+
+    audio_mp3:
+        - audio_mp3
+
+    desktop_webm:
+        - desktop_webm
+
+    youtube:
+        - youtube
+    review:
+    hls:
+        - hls
+
+# Heal settings
+heal_start: 2
+heal_end: 50

test_requirements.txt

 ## NOTE:  Test requirements.
 codecov==2.0.9
 pep8==1.7.0
-coverage==3.7.1
+coverage==4.2
 isort==4.2.15
 ddt==1.1.1
 moto==1.0.1
-responses==0.6.1
+responses==0.6.1
\ No newline at end of file
+pytest==3.0.6
+pytest-django==3.1.2
+pytest-django-ordering==1.0.1
\ No newline at end of file