Fixing few bugs that were found during sandbox testing.

VEDA_OS01/tests/test_transcripts.py

@@ -117,7 +117,7 @@ class Cielo24TranscriptTests(APITestCase):
             'file_format': transcripts.TRANSCRIPT_SJSON,
             'video_id': self.video.studio_id,
             'name': '{directory}{uuid}.sjson'.format(
-                directory=CONFIG_DATA['transcript_bucket_directory'], uuid=self.uuid_hex
+                directory=CONFIG_DATA['aws_video_transcripts_prefix'], uuid=self.uuid_hex
             ),
             'language_code': 'en',
             'provider': TranscriptProvider.CIELO24
@@ -204,7 +204,7 @@ class Cielo24TranscriptTests(APITestCase):
 
         # create s3 bucket -- all this is happening in moto's virtual environment
         conn = S3Connection()
-        conn.create_bucket(CONFIG_DATA['transcript_bucket_name'])
+        conn.create_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
 
         transcripts.cielo24_transcript_callback(None, **REQUEST_PARAMS)
 
@@ -225,7 +225,7 @@ class Cielo24TranscriptTests(APITestCase):
         self.assertEqual(json.loads(responses.calls[3].request.body), self.video_transcript_ready_status_data)
 
         # verify sjson data uploaded to s3
-        bucket = conn.get_bucket(CONFIG_DATA['transcript_bucket_name'])
+        bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
         key = Key(bucket)
         key.key = transcript_create_request_data['name']
         sjson = json.loads(key.get_contents_as_string())

VEDA_OS01/transcripts.py

@@ -24,6 +24,7 @@ from VEDA_OS01.models import (TranscriptPreferences, TranscriptProcessMetadata,
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 
+logging.basicConfig()
 LOGGER = logging.getLogger(__name__)
 TRANSCRIPT_SJSON = 'sjson'
 CIELO24_TRANSCRIPT_COMPLETED = django.dispatch.Signal(providing_args=['job_id', 'lang_code', 'org', 'video_id'])
@@ -178,7 +179,7 @@ def cielo24_transcript_callback(sender, **kwargs):
             sjson_file_name = upload_sjson_to_s3(CONFIG, sjson)
         except Exception:
             LOGGER.exception(
-                '[CIELO24 TRANSCRIPTS] Request failed for video=%s -- lang=%s -- job_id=%s -- message=%s',
+                '[CIELO24 TRANSCRIPTS] Request failed for video=%s -- lang=%s -- job_id=%s',
                 video_id,
                 lang_code,
                 job_id
@@ -256,11 +257,11 @@ def upload_sjson_to_s3(config, sjson_data):
     Upload sjson data to s3.
     """
     s3_conn = boto.connect_s3()
-    bucket = s3_conn.get_bucket(config['transcript_bucket_name'])
+    bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
     k = Key(bucket)
     k.content_type = 'application/json'
     k.key = '{directory}{uuid}.sjson'.format(
-        directory=config['transcript_bucket_directory'],
+        directory=config['aws_video_transcripts_prefix'],
         uuid=uuid.uuid4().hex
     )
     k.set_contents_from_string(json.dumps(sjson_data))

bin/loop.py

@@ -15,6 +15,10 @@ if project_path not in sys.path:
 This is a cheapo way to get a pager (using SES)
 
 """
+
+import django
+django.setup()
+
 from control.veda_file_discovery import FileDiscovery
 from youtube_callback.daemon import generate_course_list
 from youtube_callback.sftp_id_retrieve import callfunction

control/tests/test_heal.py

@@ -105,165 +105,163 @@ class HealTests(TestCase):
         heal = VedaHeal()
         heal.discovery()
 
-
-@data(
-    {
-        'edx_id': '1',
-        'video_trans_status': 'Corrupt File',
-        'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-        'video_active': True,
-    },
-    {
-        'edx_id': '1',
-        'video_trans_status': 'Review Reject',
-        'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-        'video_active': True,
-    },
-    {
-        'edx_id': '1',
-        'video_trans_status': 'Review Hold',
-        'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-        'video_active': True,
-    },
-    {
-        'edx_id': '1',
-        'video_trans_status': 'Complete',
-        'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-        'video_active': False,
-    },
-    {
-        'edx_id': '2',
-        'video_trans_status': 'Ingest',
-        'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-        'video_active': True,
-    },
-    {
-        'edx_id': '1',
-        'video_trans_status': 'Corrupt File',
-        'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-        'video_active': True,
-    },
-)
-@unpack
-def test_determine_fault(self, edx_id, video_trans_status, video_trans_start, video_active):
-    """
-    Tests that determine_fault works in various video states.
-    """
-    video_instance = Video(
-        edx_id=edx_id,
-        video_trans_status=video_trans_status,
-        video_trans_start=video_trans_start,
-        video_active=video_active,
-        inst_class=Course()
-    )
-    encode_list = self.heal_instance.determine_fault(video_instance)
-
-    if video_instance.edx_id == '1':
-        self.assertEqual(encode_list, [])
-    elif video_instance.edx_id == '2':
-        for e in encode_list:
-            self.assertTrue(e in self.encode_list)
-
-
-@data(
-    {
-        'uncompleted_encodes': [],
-        'expected_encodes': ['test_obj'],
-        'video_object': {
+    @data(
+        {
             'edx_id': '1',
-            'video_trans_status': 'Complete',
+            'video_trans_status': 'Corrupt File',
             'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
             'video_active': True,
-        }
-    },
-    {
-        'uncompleted_encodes': ['test_obj'],
-        'expected_encodes': ['test_obj'],
-        'video_object': {
-            'edx_id': '2',
-            'video_trans_status': 'Ingest',
+        },
+        {
+            'edx_id': '1',
+            'video_trans_status': 'Review Reject',
             'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
             'video_active': True,
-        }
-    }
-)
-@unpack
-def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
-    """
-    Tests that differentiate_encodes list comparison works as expected. This doesn't test video states,
-    just the list comparison function.
-    """
-    video_instance = Video(
-        edx_id=video_object['edx_id'],
-        video_trans_status=video_object['video_trans_status'],
-        video_trans_start=video_object['video_trans_start'],
-        video_active=video_object['video_active'],
-        inst_class=Course()
-    )
-
-    encode_list = self.heal_instance.differentiate_encodes(
-        uncompleted_encodes,
-        expected_encodes,
-        video_instance
-    )
-
-    if video_instance.edx_id == '1':
-        self.assertEqual(encode_list, [])
-    elif video_instance.edx_id == '2':
-        self.assertEqual(encode_list, ['test_obj'])
-
-
-@data(
-    {
-        'uncompleted_encodes': ['test_encode', 'test_encode'],
-        'expected_encodes': ['test_encode', 'test_encode'],
-        'video_object': {
+        },
+        {
             'edx_id': '1',
-            'video_trans_status': 'Complete',
+            'video_trans_status': 'Review Hold',
             'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
             'video_active': True,
-        }
-    },
-    {
-        'uncompleted_encodes': ['test_encode', 'test_encode', 'hls'],
-        'expected_encodes': ['test_encode', 'test_encode', 'hls'],
-        'video_object': {
-            'edx_id': '2',
+        },
+        {
+            'edx_id': '1',
             'video_trans_status': 'Complete',
             'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
-            'video_active': True,
-        }
-    },
-    {
-        'uncompleted_encodes': ['test_encode', 'test_encode', 'hls'],
-        'expected_encodes': ['test_encode', 'test_encode', 'hls'],
-        'video_object': {
-            'edx_id': '3',
+            'video_active': False,
+        },
+        {
+            'edx_id': '2',
             'video_trans_status': 'Ingest',
-            'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=10),
+            'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
             'video_active': True,
+        },
+        {
+            'edx_id': '1',
+            'video_trans_status': 'Corrupt File',
+            'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+            'video_active': True,
+        },
+    )
+    @unpack
+    def test_determine_fault(self, edx_id, video_trans_status, video_trans_start, video_active):
+        """
+        Tests that determine_fault works in various video states.
+        """
+        video_instance = Video(
+            edx_id=edx_id,
+            video_trans_status=video_trans_status,
+            video_trans_start=video_trans_start,
+            video_active=video_active,
+            inst_class=Course()
+        )
+        encode_list = self.heal_instance.determine_fault(video_instance)
+
+        if video_instance.edx_id == '1':
+            self.assertEqual(encode_list, [])
+        elif video_instance.edx_id == '2':
+            for e in encode_list:
+                self.assertTrue(e in self.encode_list)
+
+    @data(
+        {
+            'uncompleted_encodes': [],
+            'expected_encodes': ['test_obj'],
+            'video_object': {
+                'edx_id': '1',
+                'video_trans_status': 'Complete',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+            }
+        },
+        {
+            'uncompleted_encodes': ['test_obj'],
+            'expected_encodes': ['test_obj'],
+            'video_object': {
+                'edx_id': '2',
+                'video_trans_status': 'Ingest',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+            }
         }
-    }
-)
-@unpack
-def test_determine_longterm_corrupt(self, uncompleted_encodes, expected_encodes, video_object):
-    video_instance = Video(
-        edx_id=video_object['edx_id'],
-        video_trans_status=video_object['video_trans_status'],
-        video_trans_start=video_object['video_trans_start'],
-        video_active=video_object['video_active'],
-        inst_class=Course()
     )
+    @unpack
+    def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
+        """
+        Tests that differentiate_encodes list comparison works as expected. This doesn't test video states,
+        just the list comparison function.
+        """
+        video_instance = Video(
+            edx_id=video_object['edx_id'],
+            video_trans_status=video_object['video_trans_status'],
+            video_trans_start=video_object['video_trans_start'],
+            video_active=video_object['video_active'],
+            inst_class=Course()
+        )
+
+        encode_list = self.heal_instance.differentiate_encodes(
+            uncompleted_encodes,
+            expected_encodes,
+            video_instance
+        )
+
+        if video_instance.edx_id == '1':
+            self.assertEqual(encode_list, [])
+        elif video_instance.edx_id == '2':
+            self.assertEqual(encode_list, ['test_obj'])
 
-    longterm_corrupt = self.heal_instance.determine_longterm_corrupt(
-        uncompleted_encodes,
-        expected_encodes,
-        video_instance
+
+    @data(
+        {
+            'uncompleted_encodes': ['test_encode', 'test_encode'],
+            'expected_encodes': ['test_encode', 'test_encode'],
+            'video_object': {
+                'edx_id': '1',
+                'video_trans_status': 'Complete',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+            }
+        },
+        {
+            'uncompleted_encodes': ['test_encode', 'test_encode', 'hls'],
+            'expected_encodes': ['test_encode', 'test_encode', 'hls'],
+            'video_object': {
+                'edx_id': '2',
+                'video_trans_status': 'Complete',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+            }
+        },
+        {
+            'uncompleted_encodes': ['test_encode', 'test_encode', 'hls'],
+            'expected_encodes': ['test_encode', 'test_encode', 'hls'],
+            'video_object': {
+                'edx_id': '3',
+                'video_trans_status': 'Ingest',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=10),
+                'video_active': True,
+            }
+        }
     )
+    @unpack
+    def test_determine_longterm_corrupt(self, uncompleted_encodes, expected_encodes, video_object):
+        video_instance = Video(
+            edx_id=video_object['edx_id'],
+            video_trans_status=video_object['video_trans_status'],
+            video_trans_start=video_object['video_trans_start'],
+            video_active=video_object['video_active'],
+            inst_class=Course()
+        )
+
+        longterm_corrupt = self.heal_instance.determine_longterm_corrupt(
+            uncompleted_encodes,
+            expected_encodes,
+            video_instance
+        )
 
-    if video_instance.edx_id == '1':
-        self.assertEqual(longterm_corrupt, False)
-    elif video_instance.edx_id == '2':
-        self.assertEqual(longterm_corrupt, False)
-    elif video_instance.edx_id == '3':
-        self.assertEqual(longterm_corrupt, True)
+        if video_instance.edx_id == '1':
+            self.assertEqual(longterm_corrupt, False)
+        elif video_instance.edx_id == '2':
+            self.assertEqual(longterm_corrupt, False)
+        elif video_instance.edx_id == '3':
+            self.assertEqual(longterm_corrupt, True)

control/veda_deliver.py

@@ -181,6 +181,15 @@ class VedaDelivery:
         """
         Transcript, Xuetang
         """
+
+        self._XUETANG_ROUTE()
+
+        self.status = self._DETERMINE_STATUS()
+
+        self._UPDATE_DATA()
+
+        self._CLEANUP()
+
         self._THREEPLAY_UPLOAD()
         # Transcription Process
         # We only want to generate transcripts for `desktop_mp4` profile.
@@ -194,13 +203,6 @@ class VedaDelivery:
             if self.video_query.provider == TranscriptProvider.CIELO24:
                 self.cielo24_transcription_flow()
 
-        self._XUETANG_ROUTE()
-
-        self.status = self._DETERMINE_STATUS()
-
-        self._UPDATE_DATA()
-
-        self._CLEANUP()
 
     def _INFORM_INTAKE(self):
         """

control/veda_file_discovery.py

@@ -8,6 +8,7 @@ from boto.exception import S3ResponseError, S3DataError
 import yaml
 
 from VEDA_OS01.models import TranscriptPreferences
+from VEDA_OS01.utils import extract_course_org
 
 try:
     boto.config.add_section('Boto')
@@ -237,9 +238,7 @@ class FileDiscovery(object):
         try:
             transcript_preferences = json.loads(transcript_preferences)
             TranscriptPreferences.objects.get(
-                # TODO: Once ammar is done with cielo24.
-                # org=extract_course_org(course_url),
-                org=transcript_preferences.get('org'),
+                org=extract_course_org(course_url),
                 provider=transcript_preferences.get('provider')
             )
             process_transcription = True

instance_config.yaml

@@ -32,11 +32,15 @@ debug: false
 edx_s3_ingest_prefix:
 edx_s3_ingest_bucket:
 edx_s3_endpoint_bucket:
+
 # CF
 edx_cloudfront_prefix:
+
 # Images
 aws_video_images_bucket:
 aws_video_images_prefix: "video-images/"
+
+
 # VEDA Internal
 veda_s3_upload_bucket:
 veda_s3_hotstore_bucket:
@@ -47,13 +51,12 @@ veda_base_url:
 s3_base_url: https://s3.amazonaws.com
 
 
-# transcript bucket config
-transcript_bucket_name:
-transcript_bucket_directory: video-transcripts/
-
+# Transcripts
+aws_video_transcripts_bucket:
+aws_video_transcripts_prefix: video-transcripts/
 
 # a token identifying a valid request from transcript provider
-transcript_provider_request_token:
+transcript_provider_request_token: test-token
 
 # Ingest Secret
 # TODO: Elminate access key after AWS Support ticket 08/20/17 regarding cross-account IAM role access.

test_config.yaml

@@ -6,8 +6,8 @@ s3_base_url: https://s3.amazonaws.com
 
 
 # transcript bucket config
-transcript_bucket_name: bucket_name
-transcript_bucket_directory:  video-transcripts/
+aws_video_transcripts_bucket: bucket_name
+aws_video_transcripts_prefix:  video-transcripts/
 
 
 # a token identifying a valid request from transcript provider