Heal fixes: To avoid transcript status override in VAL

- Fix - HEAL overwrite transcript statuses for completed videos in VAL.
- Add source_language and transcript_status to VideoSerializer and Admin.
- Add tests coverage for Model Serializers.

VEDA_OS01/admin.py

@@ -34,8 +34,10 @@ class VideoAdmin(admin.ModelAdmin):
         'studio_id',
         'video_trans_start',
         'video_trans_status',
+        'transcript_status',
         'video_active',
         'process_transcription',
+        'source_language',
         'provider',
         'three_play_turnaround',
         'cielo24_turnaround',

VEDA_OS01/serializers.py

@@ -72,9 +72,11 @@ class VideoSerializer(serializers.ModelSerializer):
             'video_trans_start',
             'video_trans_end',
             'video_trans_status',
+            'transcript_status',
             'video_glacierid',
             'course_ids',
             'process_transcription',
+            'source_language',
             'provider',
             'three_play_turnaround',
             'cielo24_turnaround',
@@ -130,6 +132,10 @@ class VideoSerializer(serializers.ModelSerializer):
             'video_trans_status',
             instance.video_trans_status
         )
+        instance.transcript_status = validated_data.get(
+            'transcript_status',
+            instance.transcript_status
+        )
         instance.video_glacierid = validated_data.get(
             'video_glacierid',
             instance.video_glacierid
@@ -138,6 +144,10 @@ class VideoSerializer(serializers.ModelSerializer):
             'process_transcription',
             instance.process_transcription
         )
+        instance.source_language = validated_data.get(
+            'source_language',
+            instance.source_language
+        )
         instance.provider = validated_data.get(
             'provider',
             instance.provider

VEDA_OS01/tests/test_serializers.py

+from django.test import TestCase
+
+from VEDA_OS01.models import Course, Destination, Encode, URL, Video
+from VEDA_OS01.serializers import CourseSerializer, EncodeSerializer, URLSerializer, VideoSerializer
+
+
+class TestCourseSerializer(TestCase):
+    """
+    Tests for `CourseSerializer`.
+    """
+    def setUp(self):
+        self.course_props = dict(
+            course_name=u'Intro to VEDA',
+            institution=u'MAx',
+            edx_classid=u'123',
+            semesterid=u'2017',
+        )
+
+    def test_create_course(self):
+        """
+        Tests that `CourseSerializer.create` works as expected.
+        """
+        course_serializer = CourseSerializer(data=self.course_props)
+        course_serializer.is_valid(raise_exception=True)
+        course_serializer.save()
+        # Now, get the created course record.
+        serialized_course = CourseSerializer(
+            instance=Course.objects.get(**self.course_props)
+        ).data
+        self.assertDictEqual(serialized_course, course_serializer.data)
+
+    def test_update_course(self):
+        """
+        Tests that `CourseSerializer.update` works as expected.
+        """
+        course = Course.objects.create(**self.course_props)
+        # Perform the update via serializer.
+        updated_course_props = dict(self.course_props, course_name=u'Intro to edx-video-pipeline')
+        course_serializer = CourseSerializer(instance=course, data=updated_course_props, partial=True)
+        course_serializer.is_valid(raise_exception=True)
+        course_serializer.save()
+        # Now, see if its updated
+        serialized_course = CourseSerializer(
+            instance=Course.objects.first()
+        ).data
+        self.assertDictEqual(serialized_course, course_serializer.data)
+
+
+class TestVideoSerializer(TestCase):
+    """
+    Tests for `VideoSerializer`.
+    """
+    def setUp(self):
+        self.course = Course.objects.create(
+            course_name=u'Intro to VEDA',
+            institution=u'MAx',
+            edx_classid=u'123',
+            semesterid=u'2017',
+            local_storedir='course_id1, course_id2',
+        )
+
+        self.video_props = dict(
+            inst_class=self.course.pk,
+            client_title=u'Intro to video',
+            edx_id=u'12345678',
+            studio_id=u'43211234',
+            video_active=True,
+            process_transcription=True,
+            source_language=u'fr',
+        )
+
+    def test_create_video(self):
+        """
+        Tests that `VideoSerializer.create` works as expected.
+        """
+        video_serializer = VideoSerializer(data=self.video_props)
+        video_serializer.is_valid(raise_exception=True)
+        video_serializer.save()
+        # Now, get the created video record.
+        serialized_video = VideoSerializer(
+            instance=Video.objects.get(**self.video_props)
+        ).data
+        self.assertDictEqual(serialized_video, video_serializer.data)
+
+    def test_update_video(self):
+        """
+        Tests that `VideoSerializer.update` works as expected.
+        """
+        video = Video.objects.create(**dict(self.video_props, inst_class=self.course))
+        # Perform the update via serializer.
+        updated_video_props = dict(self.video_props, client_title=u'Intro to new Video')
+        video_serializer = VideoSerializer(instance=video, data=updated_video_props, partial=True)
+        video_serializer.is_valid(raise_exception=True)
+        video_serializer.save()
+        # Now, see if its updated
+        serialized_video = VideoSerializer(
+            instance=Video.objects.first()
+        ).data
+        self.assertDictEqual(serialized_video, video_serializer.data)
+
+
+class TestURLSerializer(TestCase):
+    """
+    Tests for `URLSerializer`.
+    """
+    def setUp(self):
+        # Setup an encode
+        destination = Destination.objects.create(
+            destination_name='test_destination',
+            destination_nick='des',
+            destination_active=True
+        )
+        encode = Encode.objects.create(
+            encode_destination=destination,
+            encode_name='desktop_mp4',
+            profile_active=True,
+        )
+
+        # Setup a video
+        course = Course.objects.create(
+            course_name=u'Intro to VEDA',
+            institution=u'MAx',
+            edx_classid=u'123',
+            semesterid=u'2017',
+            local_storedir='course_id1, course_id2',
+        )
+        video = Video.objects.create(
+            inst_class=course,
+            client_title=u'Intro to video',
+            edx_id=u'12345678',
+            studio_id=u'43211234'
+        )
+
+        # Setup URL properties
+        self.url_props = dict(
+            encode_profile=encode.pk,
+            videoID=video.pk,
+            encode_url='https://www.s3.amazon.com/123.mp4'
+        )
+
+    def test_create_url(self):
+        """
+        Tests that `URLSerializer.create` works as expected.
+        """
+        url_serializer = URLSerializer(data=self.url_props)
+        url_serializer.is_valid(raise_exception=True)
+        url_serializer.save()
+        # Now, get the created URL record.
+        serialized_url = URLSerializer(
+            instance=URL.objects.first()
+        ).data
+        self.assertDictEqual(serialized_url, url_serializer.data)
+
+
+class TestEncodeSerializer(TestCase):
+    """
+    Tests for `EncodeSerializer`.
+    """
+    def test_serialized_encode(self):
+        """
+        Tests that serializing/de-serializing 'Encode' works as expected.
+        """
+        destination = Destination.objects.create(
+            destination_name='test_destination',
+            destination_nick='des',
+            destination_active=True
+        )
+        encode = Encode.objects.create(
+            encode_destination=destination,
+            encode_name='desktop_mp4',
+            profile_active=True,
+        )
+        self.assertEqual(Encode.objects.count(), 1)
+
+        actual_serialized_encode = EncodeSerializer(encode).data
+        for attr, actual_value in actual_serialized_encode.iteritems():
+            expected_value = getattr(encode, attr)
+            self.assertEqual(actual_value, expected_value)

control/tests/test_heal.py

@@ -8,14 +8,13 @@ import sys
 from django.test import TestCase
 from datetime import timedelta
 from ddt import data, ddt, unpack
-from unittest import skip
 import responses
 from django.utils.timezone import utc
 from mock import PropertyMock, patch
 
 from control.veda_heal import VedaHeal
-from VEDA_OS01.models import URL, Course, Destination, Encode, Video
-from VEDA_OS01.utils import build_url, get_config
+from VEDA_OS01.models import URL, Course, Destination, Encode, Video, TranscriptStatus
+from VEDA_OS01.utils import build_url, get_config, ValTranscriptStatus
 
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)
@@ -169,50 +168,103 @@ class HealTests(TestCase):
         {
             'uncompleted_encodes': [],
             'expected_encodes': ['test_obj'],
-            'video_object': {
+            'video_props': {
                 'edx_id': '1',
                 'video_trans_status': 'Complete',
                 'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
                 'video_active': True,
-            }
+            },
+            'result': []
         },
         {
             'uncompleted_encodes': ['test_obj'],
             'expected_encodes': ['test_obj'],
-            'video_object': {
+            'video_props': {
                 'edx_id': '2',
                 'video_trans_status': 'Ingest',
                 'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
                 'video_active': True,
-            }
+            },
+            'result': ['test_obj']
         }
     )
     @unpack
-    def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
+    def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_props, result):
         """
         Tests that differentiate_encodes list comparison works as expected. This doesn't test video states,
         just the list comparison function.
         """
-        video_instance = Video(
-            edx_id=video_object['edx_id'],
-            video_trans_status=video_object['video_trans_status'],
-            video_trans_start=video_object['video_trans_start'],
-            video_active=video_object['video_active'],
-            inst_class=self.course_object
-        )
-
-        video_instance.save()
-
+        video_instance = Video.objects.create(inst_class=self.course_object, **video_props)
         encode_list = self.heal_instance.differentiate_encodes(
             uncompleted_encodes,
             expected_encodes,
             video_instance
         )
+        self.assertEqual(encode_list, result)
 
-        if video_instance.edx_id == '1':
-            self.assertEqual(encode_list, [])
-        elif video_instance.edx_id == '2':
-            self.assertEqual(encode_list, ['test_obj'])
+    @data(
+        {
+            'uncompleted_encodes': [],
+            'expected_encodes': ['test_obj'],
+            'video_props': {
+                'edx_id': '1',
+                'video_trans_status': 'Complete',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+                'transcript_status': TranscriptStatus.PENDING
+            },
+            'expected_val_status': 'file_complete'
+        },
+        {
+            'uncompleted_encodes': [],
+            'expected_encodes': ['test_obj'],
+            'video_props': {
+                'edx_id': '1',
+                'video_trans_status': 'Complete',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+                'transcript_status': TranscriptStatus.IN_PROGRESS
+            },
+            'expected_val_status': ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS
+        },
+        {
+            'uncompleted_encodes': [],
+            'expected_encodes': ['test_obj'],
+            'video_props': {
+                'edx_id': '1',
+                'video_trans_status': 'Complete',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+                'transcript_status': TranscriptStatus.READY
+            },
+            'expected_val_status': ValTranscriptStatus.TRANSCRIPT_READY
+        },
+        {
+            'uncompleted_encodes': ['test_obj'],
+            'expected_encodes': ['test_obj'],
+            'video_props': {
+                'edx_id': '2',
+                'video_trans_status': 'Ingest',
+                'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
+                'video_active': True,
+                'transcript_status': TranscriptStatus.READY
+            },
+            'expected_val_status': 'transcode_queue'
+        }
+    )
+    @unpack
+    def test_differentiate_encodes_val_status(self, uncompleted_encodes,
+                                              expected_encodes, video_props, expected_val_status):
+        """
+        Tests that the val status changes as expected based on encode list.
+        """
+        video_instance = Video.objects.create(inst_class=self.course_object, **video_props)
+        self.heal_instance.differentiate_encodes(
+            uncompleted_encodes,
+            expected_encodes,
+            video_instance
+        )
+        self.assertEqual(self.heal_instance.val_status, expected_val_status)
 
     @data(
         {

control/veda_heal.py

@@ -8,14 +8,19 @@ Roll through videos, check for completion
 
 """
 import datetime
-import uuid
 from datetime import timedelta
-
+import os
+import sys
+import uuid
 import yaml
+
 from django.utils.timezone import utc
 
+from VEDA_OS01.models import Encode, URL, Video
+from VEDA_OS01.utils import VAL_TRANSCRIPT_STATUS_MAP
+
 import celeryapp
-from control_env import *
+from control_env import WORK_DIRECTORY
 from veda_encode import VedaEncode
 from veda_val import VALAPICall
 
@@ -73,6 +78,11 @@ class VedaHeal(object):
             encode_list = self.determine_fault(video_object=v)
             # Using the 'Video Proto' Model
             if self.val_status is not None:
+                # Update to VAL is also happening for those videos which are already marked complete,
+                # All these retries are for the data-parity between VAL and VEDA, as calls to VAL api are
+                # unreliable and times out. For a completed Video, VEDA heal will keep doing this unless
+                # the Video is old enough and escapes from the time-span that HEAL is picking up on.
+                # cc Greg Martin
                 VAC = VALAPICall(
                     video_proto=None,
                     video_object=v,
@@ -138,8 +148,19 @@ class VedaHeal(object):
                 # These encodes don't count towards 'file_complete'
                 if e != 'mobile_high' and e != 'audio_mp3' and e != 'review':
                     check_list.append(e)
+
+        # See if VEDA's Video data model is already having transcript status which corresponds
+        # to any of Val's Video transcript statuses. If its True, set `val_status` to that status
+        # instead of `file_complete` as transcription phase comes after encoding phase of a Video,
+        # and `file_complete` shows that a Video's encodes are complete, while there may be possibility
+        # that the Video has gone through transcription phase as well after the encodes were ready.
+        val_transcription_status = VAL_TRANSCRIPT_STATUS_MAP.get(video_object.transcript_status, None)
         if check_list is None or len(check_list) == 0:
-            self.val_status = 'file_complete'
+            if val_transcription_status:
+                self.val_status = val_transcription_status
+            else:
+                self.val_status = 'file_complete'
+
             # File is complete!
             # Check for data parity, and call done
             if video_object.video_trans_status != 'Complete':
@@ -156,7 +177,11 @@ class VedaHeal(object):
             if self.determine_longterm_corrupt(uncompleted_encodes, expected_encodes, video_object):
                 return []
 
-        if self.val_status != 'file_complete':
+        complete_statuses = ['file_complete']
+        if val_transcription_status:
+            complete_statuses.append(val_transcription_status)
+
+        if self.val_status not in complete_statuses:
             self.val_status = 'transcode_queue'
         return uncompleted_encodes