Skip to content

E
edx-video-pipeline
Commit bbe3c6d1 by Qubad786 Committed by muzaffaryousaf
Options

Heal fixes: To avoid transcript status override in VAL 

- Fix - HEAL overwrite transcript statuses for completed videos in VAL.
- Add source_language and transcript_status to VideoSerializer and Admin.
- Add tests coverage for Model Serializers.
parent acae00b4
Showing with 294 additions and 27 deletions
VEDA_OS01/admin.py
......@@ -34,8 +34,10 @@ class VideoAdmin(admin.ModelAdmin):
'studio_id',
'video_trans_start',
'video_trans_status',
'transcript_status',
'video_active',
'process_transcription',
'source_language',
'provider',
'three_play_turnaround',
'cielo24_turnaround',
......
VEDA_OS01/serializers.py
......@@ -72,9 +72,11 @@ class VideoSerializer(serializers.ModelSerializer):
'video_trans_start',
'video_trans_end',
'video_trans_status',
'transcript_status',
'video_glacierid',
'course_ids',
'process_transcription',
'source_language',
'provider',
'three_play_turnaround',
'cielo24_turnaround',
......@@ -130,6 +132,10 @@ class VideoSerializer(serializers.ModelSerializer):
'video_trans_status',
instance.video_trans_status
)
instance.transcript_status = validated_data.get(
'transcript_status',
instance.transcript_status
)
instance.video_glacierid = validated_data.get(
'video_glacierid',
instance.video_glacierid
......@@ -138,6 +144,10 @@ class VideoSerializer(serializers.ModelSerializer):
'process_transcription',
instance.process_transcription
)
instance.source_language = validated_data.get(
'source_language',
instance.source_language
)
instance.provider = validated_data.get(
'provider',
instance.provider
......
VEDA_OS01/tests/test_serializers.py 0 → 100644
from django.test import TestCase
from VEDA_OS01.models import Course, Destination, Encode, URL, Video
from VEDA_OS01.serializers import CourseSerializer, EncodeSerializer, URLSerializer, VideoSerializer
class TestCourseSerializer(TestCase):
"""
Tests for `CourseSerializer`.
"""
def setUp(self):
self.course_props = dict(
course_name=u'Intro to VEDA',
institution=u'MAx',
edx_classid=u'123',
semesterid=u'2017',
)
def test_create_course(self):
"""
Tests that `CourseSerializer.create` works as expected.
"""
course_serializer = CourseSerializer(data=self.course_props)
course_serializer.is_valid(raise_exception=True)
course_serializer.save()
# Now, get the created course record.
serialized_course = CourseSerializer(
instance=Course.objects.get(**self.course_props)
).data
self.assertDictEqual(serialized_course, course_serializer.data)
def test_update_course(self):
"""
Tests that `CourseSerializer.update` works as expected.
"""
course = Course.objects.create(**self.course_props)
# Perform the update via serializer.
updated_course_props = dict(self.course_props, course_name=u'Intro to edx-video-pipeline')
course_serializer = CourseSerializer(instance=course, data=updated_course_props, partial=True)
course_serializer.is_valid(raise_exception=True)
course_serializer.save()
# Now, see if its updated
serialized_course = CourseSerializer(
instance=Course.objects.first()
).data
self.assertDictEqual(serialized_course, course_serializer.data)
class TestVideoSerializer(TestCase):
"""
Tests for `VideoSerializer`.
"""
def setUp(self):
self.course = Course.objects.create(
course_name=u'Intro to VEDA',
institution=u'MAx',
edx_classid=u'123',
semesterid=u'2017',
local_storedir='course_id1, course_id2',
)
self.video_props = dict(
inst_class=self.course.pk,
client_title=u'Intro to video',
edx_id=u'12345678',
studio_id=u'43211234',
video_active=True,
process_transcription=True,
source_language=u'fr',
)
def test_create_video(self):
"""
Tests that `VideoSerializer.create` works as expected.
"""
video_serializer = VideoSerializer(data=self.video_props)
video_serializer.is_valid(raise_exception=True)
video_serializer.save()
# Now, get the created video record.
serialized_video = VideoSerializer(
instance=Video.objects.get(**self.video_props)
).data
self.assertDictEqual(serialized_video, video_serializer.data)
def test_update_video(self):
"""
Tests that `VideoSerializer.update` works as expected.
"""
video = Video.objects.create(**dict(self.video_props, inst_class=self.course))
# Perform the update via serializer.
updated_video_props = dict(self.video_props, client_title=u'Intro to new Video')
video_serializer = VideoSerializer(instance=video, data=updated_video_props, partial=True)
video_serializer.is_valid(raise_exception=True)
video_serializer.save()
# Now, see if its updated
serialized_video = VideoSerializer(
instance=Video.objects.first()
).data
self.assertDictEqual(serialized_video, video_serializer.data)
class TestURLSerializer(TestCase):
"""
Tests for `URLSerializer`.
"""
def setUp(self):
# Setup an encode
destination = Destination.objects.create(
destination_name='test_destination',
destination_nick='des',
destination_active=True
)
encode = Encode.objects.create(
encode_destination=destination,
encode_name='desktop_mp4',
profile_active=True,
)
# Setup a video
course = Course.objects.create(
course_name=u'Intro to VEDA',
institution=u'MAx',
edx_classid=u'123',
semesterid=u'2017',
local_storedir='course_id1, course_id2',
)
video = Video.objects.create(
inst_class=course,
client_title=u'Intro to video',
edx_id=u'12345678',
studio_id=u'43211234'
)
# Setup URL properties
self.url_props = dict(
encode_profile=encode.pk,
videoID=video.pk,
encode_url='https://www.s3.amazon.com/123.mp4'
)
def test_create_url(self):
"""
Tests that `URLSerializer.create` works as expected.
"""
url_serializer = URLSerializer(data=self.url_props)
url_serializer.is_valid(raise_exception=True)
url_serializer.save()
# Now, get the created URL record.
serialized_url = URLSerializer(
instance=URL.objects.first()
).data
self.assertDictEqual(serialized_url, url_serializer.data)
class TestEncodeSerializer(TestCase):
"""
Tests for `EncodeSerializer`.
"""
def test_serialized_encode(self):
"""
Tests that serializing/de-serializing 'Encode' works as expected.
"""
destination = Destination.objects.create(
destination_name='test_destination',
destination_nick='des',
destination_active=True
)
encode = Encode.objects.create(
encode_destination=destination,
encode_name='desktop_mp4',
profile_active=True,
)
self.assertEqual(Encode.objects.count(), 1)
actual_serialized_encode = EncodeSerializer(encode).data
for attr, actual_value in actual_serialized_encode.iteritems():
expected_value = getattr(encode, attr)
self.assertEqual(actual_value, expected_value)
control/tests/test_heal.py
......@@ -8,14 +8,13 @@ import sys
from django.test import TestCase
from datetime import timedelta
from ddt import data, ddt, unpack
from unittest import skip
import responses
from django.utils.timezone import utc
from mock import PropertyMock, patch
from control.veda_heal import VedaHeal
from VEDA_OS01.models import URL, Course, Destination, Encode, Video
from VEDA_OS01.utils import build_url, get_config
from VEDA_OS01.models import URL, Course, Destination, Encode, Video, TranscriptStatus
from VEDA_OS01.utils import build_url, get_config, ValTranscriptStatus
sys.path.append(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
......@@ -169,50 +168,103 @@ class HealTests(TestCase):
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_object': {
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
}
},
'result': []
},
{
'uncompleted_encodes': ['test_obj'],
'expected_encodes': ['test_obj'],
'video_object': {
'video_props': {
'edx_id': '2',
'video_trans_status': 'Ingest',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
}
},
'result': ['test_obj']
}
)
@unpack
def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_props, result):
"""
Tests that differentiate_encodes list comparison works as expected. This doesn't test video states,
just the list comparison function.
"""
video_instance = Video(
edx_id=video_object['edx_id'],
video_trans_status=video_object['video_trans_status'],
video_trans_start=video_object['video_trans_start'],
video_active=video_object['video_active'],
inst_class=self.course_object
)
video_instance.save()
video_instance = Video.objects.create(inst_class=self.course_object, **video_props)
encode_list = self.heal_instance.differentiate_encodes(
uncompleted_encodes,
expected_encodes,
video_instance
)
self.assertEqual(encode_list, result)
if video_instance.edx_id == '1':
self.assertEqual(encode_list, [])
elif video_instance.edx_id == '2':
self.assertEqual(encode_list, ['test_obj'])
@data(
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.PENDING
},
'expected_val_status': 'file_complete'
},
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.IN_PROGRESS
},
'expected_val_status': ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS
},
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.READY
},
'expected_val_status': ValTranscriptStatus.TRANSCRIPT_READY
},
{
'uncompleted_encodes': ['test_obj'],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '2',
'video_trans_status': 'Ingest',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.READY
},
'expected_val_status': 'transcode_queue'
}
)
@unpack
def test_differentiate_encodes_val_status(self, uncompleted_encodes,
expected_encodes, video_props, expected_val_status):
"""
Tests that the val status changes as expected based on encode list.
"""
video_instance = Video.objects.create(inst_class=self.course_object, **video_props)
self.heal_instance.differentiate_encodes(
uncompleted_encodes,
expected_encodes,
video_instance
)
self.assertEqual(self.heal_instance.val_status, expected_val_status)
@data(
{
......
control/veda_heal.py
......@@ -8,14 +8,19 @@ Roll through videos, check for completion
"""
import datetime
import uuid
from datetime import timedelta
import os
import sys
import uuid
import yaml
from django.utils.timezone import utc
from VEDA_OS01.models import Encode, URL, Video
from VEDA_OS01.utils import VAL_TRANSCRIPT_STATUS_MAP
import celeryapp
from control_env import *
from control_env import WORK_DIRECTORY
from veda_encode import VedaEncode
from veda_val import VALAPICall
......@@ -73,6 +78,11 @@ class VedaHeal(object):
encode_list = self.determine_fault(video_object=v)
# Using the 'Video Proto' Model
if self.val_status is not None:
# Update to VAL is also happening for those videos which are already marked complete,
# All these retries are for the data-parity between VAL and VEDA, as calls to VAL api are
# unreliable and times out. For a completed Video, VEDA heal will keep doing this unless
# the Video is old enough and escapes from the time-span that HEAL is picking up on.
# cc Greg Martin
VAC = VALAPICall(
video_proto=None,
video_object=v,
......@@ -138,8 +148,19 @@ class VedaHeal(object):
# These encodes don't count towards 'file_complete'
if e != 'mobile_high' and e != 'audio_mp3' and e != 'review':
check_list.append(e)
# See if VEDA's Video data model is already having transcript status which corresponds
# to any of Val's Video transcript statuses. If its True, set `val_status` to that status
# instead of `file_complete` as transcription phase comes after encoding phase of a Video,
# and `file_complete` shows that a Video's encodes are complete, while there may be possibility
# that the Video has gone through transcription phase as well after the encodes were ready.
val_transcription_status = VAL_TRANSCRIPT_STATUS_MAP.get(video_object.transcript_status, None)
if check_list is None or len(check_list) == 0:
self.val_status = 'file_complete'
if val_transcription_status:
self.val_status = val_transcription_status
else:
self.val_status = 'file_complete'
# File is complete!
# Check for data parity, and call done
if video_object.video_trans_status != 'Complete':
......@@ -156,7 +177,11 @@ class VedaHeal(object):
if self.determine_longterm_corrupt(uncompleted_encodes, expected_encodes, video_object):
return []
if self.val_status != 'file_complete':
complete_statuses = ['file_complete']
if val_transcription_status:
complete_statuses.append(val_transcription_status)
if self.val_status not in complete_statuses:
self.val_status = 'transcode_queue'
return uncompleted_encodes
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment