Separate transcript statuses from Video transcode statuses.

EDUCATOR-1431 - HEAL process was overwriting transcript statuses as they were combined with transcode statuses. So, separated them out to avoid unnecassary masking of Video's transcript statuses.

VEDA_OS01/migrations/0002_auto_20171003_1022.py → VEDA_OS01/migrations/0002_auto_20171003_1504.py

 # -*- coding: utf-8 -*-
-# Generated by Django 1.9 on 2017-10-03 10:22
+# Generated by Django 1.9 on 2017-10-03 15:04
 from __future__ import unicode_literals
 
 import VEDA_OS01.models
@@ -41,7 +41,7 @@ class Migration(migrations.Migration):
                 ('process_id', models.CharField(max_length=255, verbose_name=b'Process id')),
                 ('translation_id', models.CharField(blank=True, help_text=b'Keeps track of 3Play Translations', max_length=255, null=True, verbose_name=b'Translation id')),
                 ('lang_code', models.CharField(max_length=50, verbose_name=b'Language code')),
-                ('status', models.CharField(choices=[(b'PENDING', b'PENDING'), (b'IN PROGRESS', b'IN PROGRESS'), (b'FAILED', b'FAILED'), (b'READY', b'READY')], default=b'PENDING', max_length=50, verbose_name=b'Transcript status')),
+                ('status', models.CharField(choices=[(b'N/A', b'N/A'), (b'PENDING', b'PENDING'), (b'IN PROGRESS', b'IN PROGRESS'), (b'FAILED', b'FAILED'), (b'READY', b'READY')], default=b'PENDING', max_length=50, verbose_name=b'Transcript status')),
             ],
             options={
                 'get_latest_by': 'modified',
@@ -83,10 +83,10 @@ class Migration(migrations.Migration):
             name='three_play_turnaround',
             field=models.CharField(blank=True, choices=[(b'extended_service', b'10-Day/Extended'), (b'default', b'4-Day/Default'), (b'expedited_service', b'2-Day/Expedited'), (b'rush_service', b'24 hour/Rush'), (b'same_day_service', b'Same Day')], max_length=20, null=True, verbose_name=b'3PlayMedia Turnaround'),
         ),
-        migrations.AlterField(
+        migrations.AddField(
             model_name='video',
-            name='video_trans_status',
-            field=models.CharField(choices=[(b'Ingest', b'System Ingest'), (b'Transcode Queue', b'Transcode Queue'), (b'Active Transcode', b'Active Transcode'), (b'Transcode Retry', b'Transcode Retry'), (b'Transcode Complete', b'Transcode Complete'), (b'Deliverable Upload', b'Deliverable Upload'), (b'File Complete', b'File Complete'), (b'Transcode Error', b'Transcode Error'), (b'Corrupt File', b'Corrupt File on Ingest'), (b'Review Hold', b'Review Hold'), (b'Review Reject', b'Review Rejected'), (b'Final Publish', b'Review to Final Publish'), (b'Youtube Duplicate', b'Youtube Duplicate'), (b'In Encode Queue', b'In Encode Queue'), (b'Progress', b'In Progress'), (b'Complete', b'Complete'), (b'transcription_in_progress', b'Transcription In Progress'), (b'transcript_ready', b'Transcript Ready')], default=b'Ingest', max_length=100, verbose_name=b'Transcode Status'),
+            name='transcript_status',
+            field=models.CharField(choices=[(b'N/A', b'N/A'), (b'PENDING', b'PENDING'), (b'IN PROGRESS', b'IN PROGRESS'), (b'FAILED', b'FAILED'), (b'READY', b'READY')], default=b'N/A', max_length=100, verbose_name=b'Transcription Status'),
         ),
         migrations.AddField(
             model_name='transcriptprocessmetadata',

VEDA_OS01/models.py

@@ -27,13 +27,20 @@ class TranscriptProvider(object):
 class TranscriptStatus(object):
     """
     Transcript statuses.
-    """
 
+    Note:
+        If this status is 'PENDING' for:
+            1. Video, then transcription is applicable to a video but its not started yet.
+            2. TranscriptProcessMetadata, then transcript/translation process is created but its not actually
+            sent for processing to the 3rd Party Transcription Services.
+    """
+    NOT_APPLICABLE = 'N/A'
     PENDING = 'PENDING'
     IN_PROGRESS = 'IN PROGRESS'
     FAILED = 'FAILED'
     READY = 'READY'
     CHOICES = (
+        (NOT_APPLICABLE, NOT_APPLICABLE),
         (PENDING, PENDING),
         (IN_PROGRESS, IN_PROGRESS),
         (FAILED, FAILED),
@@ -123,11 +130,9 @@ class VideoStatus(object):
     RR = 'Review Reject'
     RP = 'Final Publish'
     YD = 'Youtube Duplicate'
-    QUEUE = 'In Encode Queue'
+    QUEUE = 'Queue'
     PROGRESS = 'Progress'
     COMPLETE = 'Complete'
-    TRANSCRIPTION_IN_PROGRESS = 'transcription_in_progress'
-    TRANSCRIPT_READY = 'transcript_ready'
 
     CHOICES = (
         (SI, 'System Ingest'),
@@ -146,8 +151,6 @@ class VideoStatus(object):
         (QUEUE, 'In Encode Queue'),
         (PROGRESS, 'In Progress'),
         (COMPLETE, 'Complete'),
-        (TRANSCRIPTION_IN_PROGRESS, 'Transcription In Progress'),
-        (TRANSCRIPT_READY, 'Transcript Ready'),
     )
 
 
@@ -435,6 +438,12 @@ class Video (models.Model):
         choices=VideoStatus.CHOICES,
         default=VideoStatus.SI
     )
+    transcript_status = models.CharField(
+        'Transcription Status',
+        max_length=100,
+        choices=TranscriptStatus.CHOICES,
+        default=TranscriptStatus.NOT_APPLICABLE
+    )
 
     video_glacierid = models.CharField('Glacier Archive ID String', max_length=200, null=True, blank=True)
     abvid_serial = models.CharField('VEDA Upload Process Serial', max_length=20, null=True, blank=True)

VEDA_OS01/tests/test_transcripts.py

@@ -12,7 +12,7 @@ from boto.s3.connection import S3Connection
 from boto.s3.key import Key
 from ddt import data, ddt, unpack
 from django.core.urlresolvers import reverse
-from mock import Mock, PropertyMock, patch, ANY
+from mock import Mock, PropertyMock, patch
 from moto import mock_s3_deprecated
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -128,7 +128,7 @@ class Cielo24TranscriptTests(APITestCase):
         }
 
         self.video_transcript_ready_status_data = {
-            'status': transcripts.VideoStatus.TRANSCRIPT_READY,
+            'status': utils.ValTranscriptStatus.TRANSCRIPT_READY,
             'edx_video_id': self.video.studio_id
         }
 
@@ -238,7 +238,7 @@ class Cielo24TranscriptTests(APITestCase):
 
         # Assert edx-video-pipeline's video status
         video = Video.objects.get(studio_id=self.video.studio_id)
-        self.assertEqual(video.video_trans_status, transcripts.VideoStatus.TRANSCRIPT_READY)
+        self.assertEqual(video.transcript_status, TranscriptStatus.READY)
 
         # verify sjson data uploaded to s3
         bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
@@ -579,7 +579,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             {
                 'url': CONFIG_DATA['val_video_transcript_status_url'],
                 'body': {
-                    'status': transcripts.VideoStatus.TRANSCRIPT_READY,
+                    'status': utils.ValTranscriptStatus.TRANSCRIPT_READY,
                     'edx_video_id': self.video.studio_id
                 },
                 'headers': {
@@ -599,7 +599,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
 
         # Assert edx-video-pipeline's video status
         video = Video.objects.get(studio_id=self.video.studio_id)
-        self.assertEqual(video.video_trans_status, transcripts.VideoStatus.TRANSCRIPT_READY)
+        self.assertEqual(video.transcript_status, TranscriptStatus.READY)
 
         # verify transcript sjson data uploaded to s3
         self.assert_uploaded_transcript_on_s3(connection=conn)
@@ -1205,7 +1205,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         expected_video_status_update_request = {
             'url': CONFIG_DATA['val_video_transcript_status_url'],
             'body': {
-                'status': transcripts.VideoStatus.TRANSCRIPT_READY,
+                'status': utils.ValTranscriptStatus.TRANSCRIPT_READY,
                 'edx_video_id': self.video.studio_id
             },
             'headers': {
@@ -1221,7 +1221,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
 
         # Asserts edx-video-pipeline's video status
         video = Video.objects.get(studio_id=self.video.studio_id)
-        self.assertEqual(video.video_trans_status, transcripts.VideoStatus.TRANSCRIPT_READY)
+        self.assertEqual(video.transcript_status, TranscriptStatus.READY)
 
     @data(
         # not-an-ok response on translation status fetch request.

VEDA_OS01/tests/test_utils.py

@@ -2,7 +2,7 @@
 Tests common utils
 """
 from ddt import data, ddt, unpack
-from mock import Mock
+from mock import MagicMock, Mock
 from unittest import TestCase
 
 from VEDA_OS01 import utils
@@ -87,23 +87,19 @@ class UtilTests(TestCase):
         config = utils.get_config()
         self.assertNotEqual(config, {})
 
-    def test_video_status_update(self):
+    @data(
+        ('IN PROGRESS', True),
+        ('FAILED', False)
+    )
+    @unpack
+    def test_video_status_update(self, status, update_val_status):
         """
         Tests that  utils.video_status_update works as expected.
         """
-        def update_video_status(*args):
-            expected_args = ('1234', 'afterwards status')
-            self.assertEqual(args, expected_args)
-
-        video = Mock(studio_id='1234', video_trans_status='earlier status')
-        val_api_client = Mock(update_video_status=update_video_status)
-
+        val_api_client = MagicMock()
+        video = Mock(studio_id='1234', transcript_status='earlier status')
         # Make call to update_video_status.
-        utils.update_video_status(
-            val_api_client=val_api_client,
-            video=video,
-            status='afterwards status'
-        )
-
-        # assert the status
-        self.assertEqual(video.video_trans_status, 'afterwards status')
+        utils.update_video_status(val_api_client=val_api_client, video=video, status=status)
+        # Assert the status and call to edx-val api method.
+        self.assertEqual(val_api_client.update_video_status.called, update_val_status)
+        self.assertEqual(video.transcript_status, status)

VEDA_OS01/transcripts.py

@@ -21,8 +21,7 @@ from rest_framework.views import APIView
 from control.veda_val import VALAPICall
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProcessMetadata,
-                              TranscriptProvider, TranscriptStatus,
-                              VideoStatus)
+                              TranscriptProvider, TranscriptStatus)
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 
@@ -252,7 +251,7 @@ def cielo24_transcript_callback(sender, **kwargs):
             utils.update_video_status(
                 val_api_client=val_api,
                 video=process_metadata.video,
-                status=VideoStatus.TRANSCRIPT_READY
+                status=TranscriptStatus.READY
             )
 
 
@@ -750,7 +749,7 @@ def three_play_transcription_callback(sender, **kwargs):
             utils.update_video_status(
                 val_api_client=val_api,
                 video=process.video,
-                status=VideoStatus.TRANSCRIPT_READY
+                status=TranscriptStatus.READY
             )
 
         # On success, a happy farewell log.
@@ -948,5 +947,5 @@ def retrieve_three_play_translations():
                 utils.update_video_status(
                     val_api_client=val_api,
                     video=translation_process.video,
-                    status=VideoStatus.TRANSCRIPT_READY
+                    status=TranscriptStatus.READY
                 )

VEDA_OS01/utils.py

@@ -8,6 +8,23 @@ import yaml
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 
+from VEDA_OS01.models import TranscriptStatus
+
+
+class ValTranscriptStatus(object):
+    """
+    VAL supported video transcript statuses.
+    """
+    TRANSCRIPTION_IN_PROGRESS = 'transcription_in_progress'
+    TRANSCRIPT_READY = 'transcript_ready'
+
+
+# Maps the edx-video-pipeline video transcript statuses to edx-val statuses.
+VAL_TRANSCRIPT_STATUS_MAP = {
+    TranscriptStatus.IN_PROGRESS: ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS,
+    TranscriptStatus.READY: ValTranscriptStatus.TRANSCRIPT_READY
+}
+
 
 def get_config(yaml_config_file='instance_config.yaml'):
     """
@@ -77,10 +94,13 @@ def update_video_status(val_api_client, video, status):
         status(Str): Video status to be updated
     """
     # update edx-val's video status
-    val_api_client.update_video_status(
-        video.studio_id, status
-    )
+    try:
+        val_status = VAL_TRANSCRIPT_STATUS_MAP[status]
+        val_api_client.update_video_status(video.studio_id, val_status)
+    except KeyError:
+        # Don't update edx-val's video status.
+        pass
 
     # update edx-video-pipeline's video status
-    video.video_trans_status = status
+    video.transcript_status = status
     video.save()

control/veda_deliver.py

@@ -18,7 +18,7 @@ from veda_deliver_cielo import Cielo24Transcript
 from veda_deliver_youtube import DeliverYoutube
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
-                              VideoStatus)
+                              TranscriptStatus)
 from VEDA_OS01.utils import build_url
 from veda_utils import ErrorObject, Metadata, Output, VideoProto
 from veda_val import VALAPICall
@@ -563,7 +563,7 @@ class VedaDelivery:
         utils.update_video_status(
             val_api_client=val_api_client,
             video=self.video_query,
-            status=VideoStatus.TRANSCRIPTION_IN_PROGRESS
+            status=TranscriptStatus.IN_PROGRESS
         )
 
         cielo24 = Cielo24Transcript(
@@ -594,7 +594,7 @@ class VedaDelivery:
             utils.update_video_status(
                 val_api_client=val_api_client,
                 video=self.video_query,
-                status=VideoStatus.TRANSCRIPTION_IN_PROGRESS
+                status=TranscriptStatus.IN_PROGRESS
             )
 
             # Initialize 3playMedia client and start transcription process

control/veda_file_ingest.py

@@ -33,6 +33,8 @@ from veda_val import VALAPICall
 from veda_encode import VedaEncode
 import celeryapp
 
+from VEDA_OS01.models import TranscriptStatus
+
 LOGGER = logging.getLogger(__name__)
 
 '''
@@ -339,6 +341,7 @@ class VedaIngest:
         # Update transcription preferences for the Video
         if self.video_proto.process_transcription:
             v1.process_transcription = self.video_proto.process_transcription
+            v1.transcript_status = TranscriptStatus.PENDING
             v1.provider = self.video_proto.provider
             v1.three_play_turnaround = self.video_proto.three_play_turnaround
             v1.cielo24_turnaround = self.video_proto.cielo24_turnaround