Fixing few bugs that were found during sandbox testing.

VEDA_OS01/tests/test_transcripts.py

@@ -117,7 +117,7 @@ class Cielo24TranscriptTests(APITestCase):
             'file_format': transcripts.TRANSCRIPT_SJSON,
             'video_id': self.video.studio_id,
             'name': '{directory}{uuid}.sjson'.format(
-                directory=CONFIG_DATA['transcript_bucket_directory'], uuid=self.uuid_hex
+                directory=CONFIG_DATA['aws_video_transcripts_prefix'], uuid=self.uuid_hex
             ),
             'language_code': 'en',
             'provider': TranscriptProvider.CIELO24
@@ -204,7 +204,7 @@ class Cielo24TranscriptTests(APITestCase):
 
         # create s3 bucket -- all this is happening in moto's virtual environment
         conn = S3Connection()
-        conn.create_bucket(CONFIG_DATA['transcript_bucket_name'])
+        conn.create_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
 
         transcripts.cielo24_transcript_callback(None, **REQUEST_PARAMS)
 
@@ -225,7 +225,7 @@ class Cielo24TranscriptTests(APITestCase):
         self.assertEqual(json.loads(responses.calls[3].request.body), self.video_transcript_ready_status_data)
 
         # verify sjson data uploaded to s3
-        bucket = conn.get_bucket(CONFIG_DATA['transcript_bucket_name'])
+        bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
         key = Key(bucket)
         key.key = transcript_create_request_data['name']
         sjson = json.loads(key.get_contents_as_string())

VEDA_OS01/transcripts.py

@@ -24,6 +24,7 @@ from VEDA_OS01.models import (TranscriptPreferences, TranscriptProcessMetadata,
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 
+logging.basicConfig()
 LOGGER = logging.getLogger(__name__)
 TRANSCRIPT_SJSON = 'sjson'
 CIELO24_TRANSCRIPT_COMPLETED = django.dispatch.Signal(providing_args=['job_id', 'lang_code', 'org', 'video_id'])
@@ -178,7 +179,7 @@ def cielo24_transcript_callback(sender, **kwargs):
             sjson_file_name = upload_sjson_to_s3(CONFIG, sjson)
         except Exception:
             LOGGER.exception(
-                '[CIELO24 TRANSCRIPTS] Request failed for video=%s -- lang=%s -- job_id=%s -- message=%s',
+                '[CIELO24 TRANSCRIPTS] Request failed for video=%s -- lang=%s -- job_id=%s',
                 video_id,
                 lang_code,
                 job_id
@@ -256,11 +257,11 @@ def upload_sjson_to_s3(config, sjson_data):
     Upload sjson data to s3.
     """
     s3_conn = boto.connect_s3()
-    bucket = s3_conn.get_bucket(config['transcript_bucket_name'])
+    bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
     k = Key(bucket)
     k.content_type = 'application/json'
     k.key = '{directory}{uuid}.sjson'.format(
-        directory=config['transcript_bucket_directory'],
+        directory=config['aws_video_transcripts_prefix'],
         uuid=uuid.uuid4().hex
     )
     k.set_contents_from_string(json.dumps(sjson_data))

bin/loop.py

@@ -15,6 +15,10 @@ if project_path not in sys.path:
 This is a cheapo way to get a pager (using SES)
 
 """
+
+import django
+django.setup()
+
 from control.veda_file_discovery import FileDiscovery
 from youtube_callback.daemon import generate_course_list
 from youtube_callback.sftp_id_retrieve import callfunction

control/veda_deliver.py

@@ -181,6 +181,15 @@ class VedaDelivery:
         """
         Transcript, Xuetang
         """
+
+        self._XUETANG_ROUTE()
+
+        self.status = self._DETERMINE_STATUS()
+
+        self._UPDATE_DATA()
+
+        self._CLEANUP()
+
         self._THREEPLAY_UPLOAD()
         # Transcription Process
         # We only want to generate transcripts for `desktop_mp4` profile.
@@ -194,13 +203,6 @@ class VedaDelivery:
             if self.video_query.provider == TranscriptProvider.CIELO24:
                 self.cielo24_transcription_flow()
 
-        self._XUETANG_ROUTE()
-
-        self.status = self._DETERMINE_STATUS()
-
-        self._UPDATE_DATA()
-
-        self._CLEANUP()
 
     def _INFORM_INTAKE(self):
         """

control/veda_file_discovery.py

@@ -8,6 +8,7 @@ from boto.exception import S3ResponseError, S3DataError
 import yaml
 
 from VEDA_OS01.models import TranscriptPreferences
+from VEDA_OS01.utils import extract_course_org
 
 try:
     boto.config.add_section('Boto')
@@ -237,9 +238,7 @@ class FileDiscovery(object):
         try:
             transcript_preferences = json.loads(transcript_preferences)
             TranscriptPreferences.objects.get(
-                # TODO: Once ammar is done with cielo24.
-                # org=extract_course_org(course_url),
-                org=transcript_preferences.get('org'),
+                org=extract_course_org(course_url),
                 provider=transcript_preferences.get('provider')
             )
             process_transcription = True

instance_config.yaml

@@ -32,11 +32,15 @@ debug: false
 edx_s3_ingest_prefix:
 edx_s3_ingest_bucket:
 edx_s3_endpoint_bucket:
+
 # CF
 edx_cloudfront_prefix:
+
 # Images
 aws_video_images_bucket:
 aws_video_images_prefix: "video-images/"
+
+
 # VEDA Internal
 veda_s3_upload_bucket:
 veda_s3_hotstore_bucket:
@@ -47,13 +51,12 @@ veda_base_url:
 s3_base_url: https://s3.amazonaws.com
 
 
-# transcript bucket config
-transcript_bucket_name:
-transcript_bucket_directory: video-transcripts/
-
+# Transcripts
+aws_video_transcripts_bucket:
+aws_video_transcripts_prefix: video-transcripts/
 
 # a token identifying a valid request from transcript provider
-transcript_provider_request_token:
+transcript_provider_request_token: test-token
 
 # Ingest Secret
 # TODO: Elminate access key after AWS Support ticket 08/20/17 regarding cross-account IAM role access.

test_config.yaml

@@ -6,8 +6,8 @@ s3_base_url: https://s3.amazonaws.com
 
 
 # transcript bucket config
-transcript_bucket_name: bucket_name
-transcript_bucket_directory:  video-transcripts/
+aws_video_transcripts_bucket: bucket_name
+aws_video_transcripts_prefix:  video-transcripts/
 
 
 # a token identifying a valid request from transcript provider