3rd Party Trnascription Fixes

- improve logging
- VEDA vedio status updation
- Remove celery task for retriving 3Play translations, accomodate it in heal process
- improve __unicode__ method and add help text
- Add an ability to override developer configs

.gitignore

@@ -14,3 +14,5 @@ coverage/
 reports/
 
 .cache/
+
+VEDA/private.py

VEDA/settings.py

@@ -2,6 +2,8 @@
 Settings
 
 """
+from os.path import join, dirname, abspath
+
 DATABASES = None
 
 import os
@@ -176,3 +178,7 @@ LOGGING = {
         },
     }
 }
+
+# See if the developer has any local overrides.
+if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):
+    from .private import *  # pylint: disable=import-error, wildcard-import

VEDA_OS01/migrations/0002_auto_20170911_1113.py → VEDA_OS01/migrations/0002_auto_20170913_0753.py

 # -*- coding: utf-8 -*-
-# Generated by Django 1.9 on 2017-09-11 11:13
+# Generated by Django 1.9 on 2017-09-13 07:53
 from __future__ import unicode_literals
 
 import VEDA_OS01.models
@@ -39,7 +39,7 @@ class Migration(migrations.Migration):
                 ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
                 ('provider', models.CharField(choices=[(b'3PlayMedia', b'3PlayMedia'), (b'Cielo24', b'Cielo24')], max_length=50, verbose_name=b'Transcript provider')),
                 ('process_id', models.CharField(max_length=255, verbose_name=b'Process id')),
-                ('translation_id', models.CharField(blank=True, max_length=255, null=True, verbose_name=b'Translation id')),
+                ('translation_id', models.CharField(blank=True, help_text=b'Keeps track of 3Play Translations', max_length=255, null=True, verbose_name=b'Translation id')),
                 ('lang_code', models.CharField(max_length=8, verbose_name=b'Language code')),
                 ('status', models.CharField(choices=[(b'PENDING', b'PENDING'), (b'IN PROGRESS', b'IN PROGRESS'), (b'FAILED', b'FAILED'), (b'READY', b'READY')], default=b'PENDING', max_length=50, verbose_name=b'Transcript status')),
             ],

VEDA_OS01/models.py

@@ -647,8 +647,9 @@ class TranscriptProcessMetadata(TimeStampedModel):
     video = models.ForeignKey(Video)
     provider = models.CharField('Transcript provider', max_length=50, choices=TranscriptProvider.CHOICES)
     process_id = models.CharField('Process id', max_length=255)
-    # To keep track of 3Play Translations.
-    translation_id = models.CharField('Translation id', max_length=255, null=True, blank=True)
+    translation_id = models.CharField(
+        'Translation id', help_text='Keeps track of 3Play Translations', max_length=255, null=True, blank=True
+    )
     lang_code = models.CharField('Language code', max_length=8)
     status = models.CharField(
         'Transcript status',
@@ -673,8 +674,9 @@ class TranscriptProcessMetadata(TimeStampedModel):
         self.save()
 
     def __unicode__(self):
-        return u'{video} - {provider} - {lang}'.format(
+        return u'{video} - {provider} - {lang} - {status}'.format(
             video=self.video.edx_id,
             provider=self.provider,
-            lang=self.lang_code
+            lang=self.lang_code,
+            status=self.status,
         )

VEDA_OS01/tests/test_transcripts.py

@@ -236,6 +236,10 @@ class Cielo24TranscriptTests(APITestCase):
         self.assertEqual(responses.calls[3].request.url, CONFIG_DATA['val_video_transcript_status_url'])
         self.assertEqual(json.loads(responses.calls[3].request.body), self.video_transcript_ready_status_data)
 
+        # Assert edx-video-pipeline's video status
+        video = Video.objects.get(studio_id=self.video.studio_id)
+        self.assertEqual(video.video_trans_status, transcripts.VideoStatus.TRANSCRIPT_READY)
+
         # verify sjson data uploaded to s3
         bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
         key = Key(bucket)
@@ -586,6 +590,10 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
                 expected_request.pop('decode_func', None)
             )
 
+        # Assert edx-video-pipeline's video status
+        video = Video.objects.get(studio_id=self.video.studio_id)
+        self.assertEqual(video.video_trans_status, transcripts.VideoStatus.TRANSCRIPT_READY)
+
         # verify transcript sjson data uploaded to s3
         self.assert_uploaded_transcript_on_s3(connection=conn)
 
@@ -1202,6 +1210,10 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             decode_func=json.loads,
         )
 
+        # Asserts edx-video-pipeline's video status
+        video = Video.objects.get(studio_id=self.video.studio_id)
+        self.assertEqual(video.video_trans_status, transcripts.VideoStatus.TRANSCRIPT_READY)
+
     @data(
         # not-an-ok response on translation status fetch request.
         (

VEDA_OS01/tests/test_utils.py

 """
 Tests common utils
 """
-from unittest import TestCase
-
 from ddt import data, ddt, unpack
+from mock import Mock
+from unittest import TestCase
 
 from VEDA_OS01 import utils
 
@@ -43,7 +43,7 @@ class UtilTests(TestCase):
     @unpack
     def test_build_url(self, urls, params, expected_url):
         """
-        Tests that urils.build_url works as expected.
+        Tests that utils.build_url works as expected.
         """
         url = utils.build_url(
             *urls,
@@ -72,7 +72,7 @@ class UtilTests(TestCase):
     @unpack
     def test_extract_course_org(self, course_id, expected_org):
         """
-        Tests that urils.extract_course_org works as expected.
+        Tests that utils.extract_course_org works as expected.
         """
         org = utils.extract_course_org(course_id)
         self.assertEqual(
@@ -82,7 +82,28 @@ class UtilTests(TestCase):
 
     def test_get_config(self):
         """
-        Tests that urils.get_config works as expected.
+        Tests that utils.get_config works as expected.
         """
         config = utils.get_config()
         self.assertNotEqual(config, {})
+
+    def test_video_status_update(self):
+        """
+        Tests that  utils.video_status_update works as expected.
+        """
+        def update_video_status(*args):
+            expected_args = ('1234', 'afterwards status')
+            self.assertEqual(args, expected_args)
+
+        video = Mock(studio_id='1234', video_trans_status='earlier status')
+        val_api_client = Mock(update_video_status=update_video_status)
+
+        # Make call to update_video_status.
+        utils.update_video_status(
+            val_api_client=val_api_client,
+            video=video,
+            status='afterwards status'
+        )
+
+        # assert the status
+        self.assertEqual(video.video_trans_status, 'afterwards status')

VEDA_OS01/transcripts.py

@@ -230,7 +230,11 @@ def cielo24_transcript_callback(sender, **kwargs):
         # update transcript status for video in edx-val only if all langauge transcripts are ready
         video_jobs = TranscriptProcessMetadata.objects.filter(video__studio_id=video_id)
         if all(video_job.status == TranscriptStatus.READY for video_job in video_jobs):
-            val_api.update_video_status(process_metadata.video.studio_id, VideoStatus.TRANSCRIPT_READY)
+            utils.update_video_status(
+                val_api_client=val_api,
+                video=process_metadata.video,
+                status=VideoStatus.TRANSCRIPT_READY
+            )
 
 
 def fetch_srt_data(url, **request_params):
@@ -713,7 +717,11 @@ def three_play_transcription_callback(sender, **kwargs):
         # It will be for edx-val as well as edx-video-pipeline and this will be the case when
         # there is only one transcript language for a video(that is, already been processed).
         if not target_languages:
-            val_api.update_video_status(process.video.studio_id, VideoStatus.TRANSCRIPT_READY)
+            utils.update_video_status(
+                val_api_client=val_api,
+                video=process.video,
+                status=VideoStatus.TRANSCRIPT_READY
+            )
 
         # On success, a happy farewell log.
         LOGGER.info(
@@ -907,4 +915,8 @@ def retrieve_three_play_translations():
             # update transcript status for video in edx-val as well as edx-video-pipeline.
             video_jobs = TranscriptProcessMetadata.objects.filter(video__studio_id=translation_process.video.studio_id)
             if all(video_job.status == TranscriptStatus.READY for video_job in video_jobs):
-                val_api.update_video_status(translation_process.video.studio_id, VideoStatus.TRANSCRIPT_READY)
+                utils.update_video_status(
+                    val_api_client=val_api,
+                    video=translation_process.video,
+                    status=VideoStatus.TRANSCRIPT_READY
+                )

VEDA_OS01/utils.py

@@ -66,3 +66,21 @@ def build_url(*urls, **query_params):
         url = '{}?{}'.format(url, urllib.urlencode(query_params))
 
     return url
+
+
+def update_video_status(val_api_client, video, status):
+    """
+    Updates video status both in edx-val and edx-video-pipeline.
+
+    Arguments:
+        video(Video): Video data model object
+        status(Str): Video status to be updated
+    """
+    # update edx-val's video status
+    val_api_client.update_video_status(
+        video.studio_id, status
+    )
+
+    # update edx-video-pipeline's video status
+    video.video_trans_status = status
+    video.save()

bin/heal

@@ -12,7 +12,8 @@ if project_path not in sys.path:
 
 from control.celeryapp import maintainer_healer
 from control.veda_heal import VedaHeal
-from VEDA_OS01.models import Video
+from VEDA_OS01.models import Course, Video
+from VEDA_OS01.transcripts import retrieve_three_play_translations
 
 """
 Deliver
@@ -79,6 +80,11 @@ def main():
         VH = VedaHeal()
         VH.discovery()
         VH.purge()
+
+        # Kicks off a round of retrieving successful
+        # translations from 3Play Media
+        retrieve_three_play_translations()
+
         HC = HealCli()
         HC.schedule()
         return None

control/celeryapp.py

@@ -2,9 +2,6 @@
 from __future__ import absolute_import
 from celery import Celery
 import yaml
-
-from VEDA_OS01.transcripts import retrieve_three_play_translations
-
 """
 Start Celery Worker
 
@@ -51,19 +48,6 @@ app.conf.update(
     CELERY_ACCEPT_CONTENT=['pickle', 'json', 'msgpack', 'yaml']
 )
 
-app.conf.beat_schedule = {
-    'check-3play-translations-every-30-seconds': {
-        'task': 'tasks.fetch_three_play_translations',
-        'schedule': 30.0,
-    },
-}
-
-
-@app.task(name='fetch_three_play_translations')
-def fetch_three_play_translations():
-    retrieve_three_play_translations()
-
-
 @app.task(name='worker_encode')
 def worker_task_fire(veda_id, encode_profile, jobid):
     pass

control/veda_deliver.py

@@ -553,9 +553,12 @@ class VedaDelivery:
             )
         )
 
-        # update transcript status for video in edx-val
-        VALAPICall(video_proto=None, val_status=None).update_video_status(
-            self.video_query.studio_id, VideoStatus.TRANSCRIPTION_IN_PROGRESS
+        # update transcript status for video.
+        val_api_client = VALAPICall(video_proto=None, val_status=None)
+        utils.update_video_status(
+            val_api_client=val_api_client,
+            video=self.video_query,
+            status=VideoStatus.TRANSCRIPTION_IN_PROGRESS
         )
 
         cielo24 = Cielo24Transcript(
@@ -580,9 +583,12 @@ class VedaDelivery:
             org = utils.extract_course_org(self.video_proto.platform_course_url[0])
             transcript_secrets = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider)
 
-            # update transcript status for video in edx-val
-            VALAPICall(video_proto=None, val_status=None).update_video_status(
-                self.video_query.studio_id, VideoStatus.TRANSCRIPTION_IN_PROGRESS
+            # update transcript status for video.
+            val_api_client = VALAPICall(video_proto=None, val_status=None)
+            utils.update_video_status(
+                val_api_client=val_api_client,
+                video=self.video_query,
+                status=VideoStatus.TRANSCRIPTION_IN_PROGRESS
             )
 
             # Initialize 3playMedia client and start transcription process

control/veda_val.py

@@ -413,10 +413,11 @@ class VALAPICall():
 
         if not response.ok:
             LOGGER.error(
-                'update_val_transcript failed -- video_id=%s -- provider=% -- status=%s',
+                'update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s',
                 video_id,
                 provider,
-                response.status_code
+                response.status_code,
+                response.content,
             )
 
     def update_video_status(self, video_id, status):