Keep encoding from overwriting transcription status except for youtube/review profiles.

VEDA_OS01/tests/factories.py

+"""
+VEDA model factories.
+"""
+from factory import Sequence, SubFactory
+from factory.django import DjangoModelFactory
+
+from VEDA_OS01.models import Course, Destination, Encode, TranscriptStatus, URL, Video, VideoStatus
+
+
+class CourseFactory(DjangoModelFactory):
+    """
+    Course data model factory.
+    """
+    class Meta(object):
+        model = Course
+
+    course_name = Sequence('Test Course {0}'.format)
+    institution = Sequence('INST-{0}'.format)
+    edx_classid = Sequence('CLASS-{0}'.format)
+    semesterid = Sequence('2018-{0}'.format)
+    proc_loc = False
+    review_proc = False
+    yt_proc = False
+    s3_proc = False
+    local_storedir = None
+
+
+class VideoFactory(DjangoModelFactory):
+    """
+    Video data model factory.
+    """
+    class Meta(object):
+        model = Video
+
+    inst_class = SubFactory(CourseFactory)
+    client_title = Sequence('Video {0}'.format)
+    edx_id = Sequence('ABC-CDE-EFG-{0}'.format)
+    studio_id = Sequence('61bd0526{0}'.format)
+    video_trans_status = VideoStatus.SI
+    transcript_status = TranscriptStatus.NOT_APPLICABLE
+    process_transcription = False
+    provider = None
+    three_play_turnaround = None
+    cielo24_turnaround = None
+    cielo24_fidelity = None
+    source_language = None
+    preferred_languages = []
+
+
+class DestinationFactory(DjangoModelFactory):
+    """
+    Destination data model factory.
+    """
+    class Meta(object):
+        model = Destination
+
+    destination_name = Sequence('Dest-{0}'.format)
+    destination_active = False
+    destination_nick = Sequence('D{0}'.format)
+
+
+class EncodeFactory(DjangoModelFactory):
+    """
+    Encode data model factory.
+    """
+    class Meta(object):
+        model = Encode
+
+    encode_destination = SubFactory(DestinationFactory)
+    encode_name = Sequence('Encode-{0}'.format)
+    profile_active = False
+    encode_suffix = ''
+    encode_filetype = 'mp4'
+    encode_bitdepth = None
+    encode_resolution = None
+    product_spec = None
+
+
+class UrlFactory(DjangoModelFactory):
+    """
+    URL data model factory.
+    """
+    class Meta(object):
+        model = URL
+
+    encode_profile = SubFactory(EncodeFactory)
+    videoID = SubFactory(VideoFactory)
+    encode_url = Sequence('https://www.querty.com/{0}'.format)

VEDA_OS01/tests/test_utils.py

@@ -5,12 +5,13 @@ from unittest import TestCase
 
 from ddt import data, ddt, unpack
 from django.conf import settings
-from django.test import override_settings
+from django.test import override_settings, TransactionTestCase
 from mock import MagicMock, Mock
 
 from VEDA_OS01 import utils
 from VEDA_OS01.models import TranscriptCredentials
-
+from VEDA_OS01.tests.factories import CourseFactory, DestinationFactory, EncodeFactory, VideoFactory, UrlFactory
+from VEDA_OS01.utils import get_incomplete_encodes, is_video_ready
 
 OLD_FERNET_KEYS_LIST = ['test-ferent-key']
 
@@ -61,3 +62,68 @@ class UtilTests(TestCase):
         with override_settings(FERNET_KEYS=new_keys_set):
             self.assertEqual(settings.FERNET_KEYS, new_keys_set)
             verify_model_field_keys(TranscriptCredentials, 'api_key', new_keys_set)
+
+
+class EncodeUtilsTest(TransactionTestCase):
+    """
+    Tests for video encode utils
+    """
+
+    def setUp(self):
+        # Setup test courses
+        course1 = CourseFactory(review_proc=True, yt_proc=True, s3_proc=True)
+        course2 = CourseFactory(review_proc=False, yt_proc=True, s3_proc=True)
+        course3 = CourseFactory(review_proc=True, yt_proc=True, s3_proc=False)
+
+        # Setup test encode profiles
+        destination = DestinationFactory(destination_active=True)
+        encode1 = EncodeFactory(encode_destination=destination, product_spec='desktop_mp4', profile_active=True)
+        encode2 = EncodeFactory(encode_destination=destination, product_spec='review', profile_active=True)
+        encode3 = EncodeFactory(encode_destination=destination, product_spec='mobile_low', profile_active=True)
+        encode4 = EncodeFactory(encode_destination=destination, product_spec='audio_mp3', profile_active=True)
+        encode5 = EncodeFactory(encode_destination=destination, product_spec='hls', profile_active=False)
+        encode6 = EncodeFactory(encode_destination=destination, product_spec='youtube', profile_active=True)
+
+        # Setup videos
+        self.video1 = VideoFactory(inst_class=course1)
+        self.video2 = VideoFactory(inst_class=course2)
+        self.video3 = VideoFactory(inst_class=course3)
+
+        # Setup urls for video1
+        UrlFactory(encode_profile=encode1, videoID=self.video1)
+        UrlFactory(encode_profile=encode2, videoID=self.video1)
+        UrlFactory(encode_profile=encode3, videoID=self.video1)
+        UrlFactory(encode_profile=encode4, videoID=self.video1)
+        UrlFactory(encode_profile=encode5, videoID=self.video1)
+        UrlFactory(encode_profile=encode6, videoID=self.video1)
+
+        # Setup urls for video2
+        UrlFactory(encode_profile=encode1, videoID=self.video2)
+        UrlFactory(encode_profile=encode3, videoID=self.video2)
+        UrlFactory(encode_profile=encode6, videoID=self.video2)
+
+        # Setup urls for video3
+        UrlFactory(encode_profile=encode6, videoID=self.video3)
+
+    def test_get_incomplete_encodes_invalid_video(self):
+        """
+        Tests that `get_incomplete_encodes` returns an empty list with non existent video id.
+        """
+        self.assertEqual(get_incomplete_encodes(u'non-existent-id'), [])
+
+    def test_get_incomplete_encodes(self):
+        """
+        Tests that `get_incomplete_encodes` works as expected.
+        """
+        self.assertEqual(get_incomplete_encodes(self.video1.edx_id), [])
+        self.assertEqual(get_incomplete_encodes(self.video2.edx_id), ['audio_mp3'])
+        self.assertEqual(get_incomplete_encodes(self.video3.edx_id), ['review'])
+
+    def test_is_video_ready(self):
+        """
+        Tests that `is_video_ready` works as expected.
+        """
+        self.assertTrue(is_video_ready(self.video1.edx_id))
+        self.assertFalse(is_video_ready(self.video2.edx_id))
+        self.assertTrue(is_video_ready(self.video2.edx_id, ignore_encodes=['audio_mp3']))
+        self.assertTrue(is_video_ready(self.video3.edx_id, ignore_encodes=['review', 'abc_encode']))

VEDA_OS01/utils.py

 """
 Common utils.
 """
-
-from VEDA_OS01.models import TranscriptStatus
+from VEDA.utils import get_config
+from VEDA_OS01.models import Encode, TranscriptStatus, URL, Video
 
 
 class ValTranscriptStatus(object):
@@ -57,3 +57,53 @@ def invalidate_fernet_cached_properties(model, fields):
             del field.fernet
         except AttributeError:
             pass
+
+
+def get_incomplete_encodes(edx_id):
+    """
+    Get incomplete encodes for the given video.
+
+    Arguments:
+        edx_id(unicode): an ID identifying the VEDA video.
+    """
+    encode_list = []
+    try:
+        video = Video.objects.filter(edx_id=edx_id).latest()
+    except Video.DoesNotExist:
+        return encode_list
+
+    course = video.inst_class
+    # Pick the encodes map from the settings.
+    encodes_map = get_config().get('encode_dict', {})
+    # Active encodes according to course instance.
+    for attr, encodes in encodes_map.iteritems():
+        if getattr(course, attr, False):
+            encode_list += [encode.strip() for encode in encodes]
+
+    # Filter active encodes further according to their corresponding encode profiles activation.
+    for encode in list(encode_list):
+        encode_profile = Encode.objects.filter(product_spec=encode).first()
+        if not encode_profile or (encode_profile and not encode_profile.profile_active):
+            encode_list.remove(encode)
+
+    # Filter encodes based on their successful encoding for the specified video.
+    for encode in list(encode_list):
+        completed_encode_profile = URL.objects.filter(
+            videoID=video,
+            encode_profile__product_spec=encode
+        )
+        if completed_encode_profile.exists():
+            encode_list.remove(encode)
+
+    return encode_list
+
+
+def is_video_ready(edx_id, ignore_encodes=list()):
+    """
+    Check whether a video should be considered ready.
+
+    Arguments:
+        edx_id(unicode): An ID identifying the VEDA video.
+        ignore_encodes(list): A list containing the profiles that should not be considered.
+    """
+    return set(get_incomplete_encodes(edx_id)).issubset(set(ignore_encodes))

control/tests/test_deliver.py

@@ -23,7 +23,6 @@ class VedaDeliverRunTest(TestCase):
     def setUp(self):
         self.veda_id = 'XXXXXXXX2014-V00TES1'
         self.encode_profile = 'hls'
-        self.encode_profile_mp4 = 'desktop_mp4'
         self.course = Course.objects.create(
             institution='XXX',
             edx_classid='XXXXX',
@@ -47,13 +46,6 @@ class VedaDeliverRunTest(TestCase):
             product_spec=self.encode_profile,
             encode_filetype='HLS'
         )
-        self.encode_mp4 = Encode.objects.create(
-            encode_destination=self.destination,
-            profile_active=True,
-            encode_suffix='DTH',
-            product_spec=self.encode_profile_mp4,
-            encode_filetype='mpeg-4'
-        )
         self.deliver_instance = VedaDelivery(
             veda_id=self.veda_id,
             encode_profile=self.encode_profile,
@@ -149,11 +141,6 @@ class VedaDeliverRunTest(TestCase):
             videoID=self.video,
             encode_url='Test_URL'
         )
-        URL.objects.create(
-            encode_profile=self.encode_mp4,
-            videoID=self.video,
-            encode_url='Test_URL'
-        )
 
         self.assertEqual(self.deliver_instance._DETERMINE_STATUS(), 'Complete')
 

control/veda_deliver.py

@@ -101,10 +101,14 @@ class VedaDelivery:
         """
         if self.encode_profile == 'youtube':
             self._CLEANUP()
-            self.start_transcription(self._DETERMINE_STATUS())
+            # We only want to generate transcripts when all the encodings(except for YT and Review) are done.
+            if utils.is_video_ready(self.video_query.edx_id, ignore_encodes=['review', 'youtube']):
+                self.start_transcription()
             return None
 
         if self.encode_profile == 'review':
+            # No need to start transcription here separately as the `self.encode_profile == 'youtube'`
+            # will take care for this encode profile as well.
             return None
 
         if self.auth_dict['edx_cloudfront_prefix'] is not None:
@@ -129,25 +133,25 @@ class VedaDelivery:
         self._UPDATE_DATA()
         self._CLEANUP()
 
-        self.start_transcription(self.status)
+        # We only want to generate transcripts when all the encodings(except for YT and Review) are done.
+        if utils.is_video_ready(self.video_query.edx_id, ignore_encodes=['review', 'youtube']):
+            self.start_transcription()
 
-    def start_transcription(self, status):
+    def start_transcription(self):
         """
-        Kick off the transcription process
+        Kick off the transcription process.
 
-        Arguments:
-            status (str): `Complete` or `Progress`
+        NOTE: Transcription should be started without waiting for YT/Review encodings.
         """
-        # We only want to generate transcripts when all encodings are completed
-        if status == 'Complete' and self.video_query.process_transcription:
+        if self.video_query.process_transcription:
             encode_query = Encode.objects.get(
                 product_spec='desktop_mp4'
             )
 
-            encoded_file = '%s_%s.%s' % (
-                self.veda_id,
-                encode_query.encode_suffix,
-                encode_query.encode_filetype
+            encoded_file = u'{video_id}_{suffix}.{ext}'.format(
+                video_id=self.veda_id,
+                suffix=encode_query.encode_suffix,
+                ext=encode_query.encode_filetype
             )
 
             # 3PlayMedia

test_requirements.txt

 ## NOTE:  Test requirements.
 codecov==2.0.9
-pep8==1.7.0
 coverage==4.2
-isort==4.2.15
 ddt==1.1.1
+edx-lint==0.5.4
+factory_boy==2.10.0
+isort==4.2.15
 moto==1.0.1
-responses==0.6.1
+Paver==1.2.4
+pep8==1.7.0
 pytest==3.0.6
 pytest-django==3.1.2
 pytest-django-ordering==1.0.1
-Paver==1.2.4
 pylint==1.7.1
 pylint-celery==0.3
 pylint-django==0.7.2
-edx-lint==0.5.4
+responses==0.6.1