Skip to content

E
edx-val
Unverified Commit 00b8ded8 by Mushtaq Ali Committed by GitHub
Options

Merge pull request #134 from edx/import_video_transcripts 

Import video transcripts
parents 6469fc26 7ef924d0
Showing with 906 additions and 182 deletions
edxval/api.py
......@@ -8,19 +8,25 @@ from enum import Enum
from uuid import uuid4
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.core.files import File
from django.core.files.base import ContentFile
from fs import open_fs
from fs.errors import ResourceNotFound
from fs.path import combine
from lxml import etree
from lxml.etree import Element, SubElement
from pysrt.srtexc import Error
from edxval.exceptions import (InvalidTranscriptFormat,
InvalidTranscriptProvider, ValCannotCreateError,
ValCannotUpdateError, ValInternalError,
ValVideoNotFoundError)
from edxval.models import (CourseVideo, EncodedVideo, Profile,
TranscriptFormat, TranscriptPreference,
from edxval.models import (CourseVideo, EncodedVideo, Profile, TranscriptPreference,
TranscriptProviderType, Video, VideoImage,
VideoTranscript, ThirdPartyTranscriptCredentialsState)
from edxval.serializers import TranscriptPreferenceSerializer, TranscriptSerializer, VideoSerializer
from edxval.utils import THIRD_PARTY_TRANSCRIPTION_PLANS
from edxval.utils import TranscriptFormat, THIRD_PARTY_TRANSCRIPTION_PLANS, create_file_in_fs, get_transcript_format
logger = logging.getLogger(__name__) # pylint: disable=C0103
......@@ -789,7 +795,7 @@ def export_to_xml(video_id, resource_fs, static_dir, course_id=None):
video_id (str): Video id of the video to export transcripts.
course_id (str): The ID of the course with which this video is associated.
static_dir (str): The Directory to store transcript file.
resource_fs (OSFS): The file system to store transcripts.
resource_fs (SubFS): Export file system.
Returns:
An lxml video_asset element containing export data
......@@ -826,7 +832,7 @@ def export_to_xml(video_id, resource_fs, static_dir, course_id=None):
return create_transcripts_xml(video_id, video_el, resource_fs, static_dir)
def create_trancript_file(video_id, language_code, file_format, resource_fs, static_dir):
def create_transcript_file(video_id, language_code, file_format, resource_fs, static_dir):
"""
Writes transcript file to file system.
......@@ -835,24 +841,17 @@ def create_trancript_file(video_id, language_code, file_format, resource_fs, sta
language_code (str): Language code of the transcript.
file_format (str): File format of the transcript file.
static_dir (str): The Directory to store transcript file.
resource_fs (OSFS): The file system to store transcripts.
resource_fs (SubFS): The file system to store transcripts.
"""
transcript_name = u'{static_dir}/{video_id}-{language_code}.{file_format}'.format(
static_dir=static_dir,
transcript_name = u'{video_id}-{language_code}.{file_format}'.format(
video_id=video_id,
language_code=language_code,
file_format=file_format
)
try:
transcript_data = get_video_transcript_data(video_id, language_code)
if transcript_data:
transcript_content = transcript_data['content']
with resource_fs.open(transcript_name, 'wb') as f:
f.write(transcript_content)
except Exception:
# Do not raise exception in case no transcript file is found for now.
# TODO: Remove this - EDUCATOR-2173
pass
transcript_data = get_video_transcript_data(video_id, language_code)
if transcript_data:
transcript_content = transcript_data['content']
create_file_in_fs(transcript_content, transcript_name, resource_fs, static_dir)
def create_transcripts_xml(video_id, video_el, resource_fs, static_dir):
......@@ -864,7 +863,7 @@ def create_transcripts_xml(video_id, video_el, resource_fs, static_dir):
video_id (str): Video id of the video.
video_el (Element): lxml Element object
static_dir (str): The Directory to store transcript file.
resource_fs (OSFS): The file system to store transcripts.
resource_fs (SubFS): The file system to store transcripts.
Returns:
lxml Element object with transcripts information
......@@ -880,13 +879,18 @@ def create_transcripts_xml(video_id, video_el, resource_fs, static_dir):
language_code = video_transcript.language_code
file_format = video_transcript.file_format
create_trancript_file(video_id, language_code, file_format, resource_fs, static_dir)
create_transcript_file(
video_id,
language_code,
file_format,
resource_fs.delegate_fs(),
combine(u'course', static_dir) # File system should not start from /draft directory.
)
SubElement(
transcripts_el,
'transcript',
{
'file_name': video_transcript.transcript.name,
'language_code': language_code,
'file_format': file_format,
'provider': video_transcript.provider,
......@@ -897,7 +901,7 @@ def create_transcripts_xml(video_id, video_el, resource_fs, static_dir):
return video_el
def import_from_xml(xml, edx_video_id, course_id=None):
def import_from_xml(xml, edx_video_id, resource_fs, static_dir, external_transcripts=dict(), course_id=None):
"""
Imports data from a video_asset element about the given video_id.
......@@ -907,20 +911,30 @@ def import_from_xml(xml, edx_video_id, course_id=None):
Arguments:
xml (Element): An lxml video_asset element containing import data
edx_video_id (str): val video id
resource_fs (OSFS): Import file system.
static_dir (str): The Directory to retrieve transcript file.
external_transcripts (dict): A dict containing the list of names of the external transcripts.
Example:
{
'en': ['The_Flash.srt', 'Harry_Potter.srt'],
'es': ['Green_Arrow.srt']
}
course_id (str): The ID of a course to associate the video with
Raises:
ValCannotCreateError: if there is an error importing the video
Returns:
edx_video_id (str): val video id.
"""
if xml.tag != 'video_asset':
raise ValCannotCreateError('Invalid XML')
# TODO this will be moved as a part of EDUCATOR-2173
if not edx_video_id:
return
# If video with edx_video_id already exists, associate it with the given course_id.
try:
if not edx_video_id:
raise Video.DoesNotExist
video = Video.objects.get(edx_video_id=edx_video_id)
logger.info(
"edx_video_id '%s' present in course '%s' not imported because it exists in VAL.",
......@@ -934,60 +948,163 @@ def import_from_xml(xml, edx_video_id, course_id=None):
if image_file_name:
VideoImage.create_or_update(course_video, image_file_name)
return
return edx_video_id
except ValidationError as err:
logger.exception(err.message)
raise ValCannotCreateError(err.message_dict)
except Video.DoesNotExist:
pass
# Video with edx_video_id did not exist, so create one from xml data.
data = {
'edx_video_id': edx_video_id,
'client_video_id': xml.get('client_video_id'),
'duration': xml.get('duration'),
'status': 'imported',
'encoded_videos': [],
'courses': [{course_id: xml.get('image')}] if course_id else [],
}
for encoded_video_el in xml.iterfind('encoded_video'):
profile_name = encoded_video_el.get('profile')
if edx_video_id:
# Video with edx_video_id did not exist, so create one from xml data.
data = {
'edx_video_id': edx_video_id,
'client_video_id': xml.get('client_video_id'),
'duration': xml.get('duration'),
'status': 'imported',
'encoded_videos': [],
'courses': [{course_id: xml.get('image')}] if course_id else [],
}
for encoded_video_el in xml.iterfind('encoded_video'):
profile_name = encoded_video_el.get('profile')
try:
Profile.objects.get(profile_name=profile_name)
except Profile.DoesNotExist:
logger.info(
"Imported edx_video_id '%s' contains unknown profile '%s'.",
edx_video_id,
profile_name
)
continue
data['encoded_videos'].append({
'profile': profile_name,
'url': encoded_video_el.get('url'),
'file_size': encoded_video_el.get('file_size'),
'bitrate': encoded_video_el.get('bitrate'),
})
# Create external video if no edx_video_id.
edx_video_id = create_video(data)
else:
edx_video_id = create_external_video('External Video')
create_transcript_objects(xml, edx_video_id, resource_fs, static_dir, external_transcripts)
return edx_video_id
def import_transcript_from_fs(edx_video_id, language_code, file_name, provider, resource_fs, static_dir):
"""
Imports transcript file from file system and creates transcript record in DS.
Arguments:
edx_video_id (str): Video id of the video.
language_code (unicode): Language code of the requested transcript.
file_name (unicode): File name of the transcript file.
provider (unicode): Transcript provider.
resource_fs (OSFS): Import file system.
static_dir (str): The Directory to retrieve transcript file.
"""
file_format = None
transcript_data = get_video_transcript_data(edx_video_id, language_code)
# First check if transcript record does not exist.
if not transcript_data:
# Read file from import file system and attach it to transcript record in DS.
try:
Profile.objects.get(profile_name=profile_name)
except Profile.DoesNotExist:
logger.info(
"Imported edx_video_id '%s' contains unknown profile '%s'.",
with resource_fs.open(combine(static_dir, file_name), 'rb') as f:
file_content = f.read()
file_content = file_content.decode('utf-8-sig')
except ResourceNotFound as exc:
# Don't raise exception in case transcript file is not found in course OLX.
logger.warn(
'[edx-val] "%s" transcript "%s" for video "%s" is not found.',
language_code,
file_name,
edx_video_id
)
return
except UnicodeDecodeError:
# Don't raise exception in case transcript contains non-utf8 content.
logger.warn(
'[edx-val] "%s" transcript "%s" for video "%s" contains a non-utf8 file content.',
language_code,
file_name,
edx_video_id
)
return
# Get file format from transcript content.
try:
file_format = get_transcript_format(file_content)
except Error as ex:
# Don't raise exception, just don't create transcript record.
logger.warn(
'[edx-val] Error while getting transcript format for video=%s -- language_code=%s --file_name=%s',
edx_video_id,
profile_name
language_code,
file_name
)
continue
data['encoded_videos'].append({
'profile': profile_name,
'url': encoded_video_el.get('url'),
'file_size': encoded_video_el.get('file_size'),
'bitrate': encoded_video_el.get('bitrate'),
})
create_video(data)
create_transcript_objects(xml)
return
# Create transcript record.
create_video_transcript(
video_id=edx_video_id,
language_code=language_code,
file_format=file_format,
content=ContentFile(file_content),
provider=provider
)
def create_transcript_objects(xml):
def create_transcript_objects(xml, edx_video_id, resource_fs, static_dir, external_transcripts):
"""
Create VideoTranscript objects.
Arguments:
xml (Element): lxml Element object
xml (Element): lxml Element object.
edx_video_id (str): Video id of the video.
resource_fs (OSFS): Import file system.
static_dir (str): The Directory to retrieve transcript file.
external_transcripts (dict): A dict containing the list of names of the external transcripts.
Example:
{
'en': ['The_Flash.srt', 'Harry_Potter.srt'],
'es': ['Green_Arrow.srt']
}
"""
for transcript in xml.findall('.//transcripts/transcript'):
try:
create_or_update_video_transcript(
transcript.attrib['video_id'],
transcript.attrib['language_code'],
metadata=dict(
# File system should not start from /draft directory.
with open_fs(resource_fs.root_path.split('/drafts')[0]) as file_system:
# First import VAL transcripts.
for transcript in xml.findall('.//transcripts/transcript'):
try:
file_format = transcript.attrib['file_format']
language_code = transcript.attrib['language_code']
transcript_file_name = u'{edx_video_id}-{language_code}.{file_format}'.format(
edx_video_id=edx_video_id,
language_code=language_code,
file_format=file_format
)
import_transcript_from_fs(
edx_video_id=edx_video_id,
language_code=transcript.attrib['language_code'],
file_name=transcript_file_name,
provider=transcript.attrib['provider'],
file_name=transcript.attrib['file_name'],
file_format=transcript.attrib['file_format'],
resource_fs=file_system,
static_dir=static_dir
)
except KeyError:
logger.warn("VAL: Required attributes are missing from xml, xml=[%s]", etree.tostring(transcript).strip())
# This won't overwrite transcript for a language which is already present for the video.
for language_code, transcript_file_names in external_transcripts.iteritems():
for transcript_file_name in transcript_file_names:
import_transcript_from_fs(
edx_video_id=edx_video_id,
language_code=language_code,
file_name=transcript_file_name,
provider=TranscriptProviderType.CUSTOM,
resource_fs=file_system,
static_dir=static_dir
)
)
except KeyError:
logger.warn("VAL: Required attributes are missing from xml, xml=[%s]", etree.tostring(transcript).strip())
edxval/models.py
......@@ -25,7 +25,7 @@ from django.dispatch import receiver
from django.utils.six import python_2_unicode_compatible
from model_utils.models import TimeStampedModel
from edxval.utils import (get_video_image_storage,
from edxval.utils import (TranscriptFormat, get_video_image_storage,
get_video_transcript_storage, video_image_path,
video_transcript_path)
......@@ -373,16 +373,6 @@ class TranscriptProviderType(object):
)
class TranscriptFormat(object):
SRT = 'srt'
SJSON = 'sjson'
CHOICES = (
(SRT, 'SubRip'),
(SJSON, 'SRT JSON')
)
class CustomizableFileField(models.FileField):
"""
Subclass of FileField that allows custom settings to not
......
edxval/tests/constants.py
......@@ -4,14 +4,19 @@
Constants used for tests.
"""
from edxval.models import (
TranscriptFormat,
TranscriptProviderType,
Cielo24Fidelity,
Cielo24Turnaround,
ThreePlayTurnaround
)
from edxval.utils import TranscriptFormat
EDX_VIDEO_ID = "itchyjacket"
EXPORT_IMPORT_COURSE_DIR = u'course'
EXPORT_IMPORT_STATIC_DIR = u'static'
"""
Generic Profiles for manually creating profile objects
"""
......@@ -363,12 +368,46 @@ VIDEO_DICT_UPDATE_ANIMAL = dict(
encoded_videos=[],
)
TRANSCRIPT_DATA = {
"overwatch": """
1
00:00:14,370 --> 00:00:16,530
I am overwatch.
2
00:00:16,500 --> 00:00:18,600
可以用“我不太懂艺术 但我知道我喜欢什么”做比喻.""",
"flash": """
1
00:00:07,180 --> 00:00:08,460
This is Flash line 1.""",
"wow": """{\n "start": [10],\n "end": [100],\n "text": ["Hi, welcome to edxval."]\n}\n"""
}
VIDEO_TRANSCRIPT_CUSTOM_SRT = dict(
language_code='en',
transcript='edxval/tests/data/The_Flash.srt',
provider=TranscriptProviderType.CUSTOM,
file_format=TranscriptFormat.SRT,
file_data=TRANSCRIPT_DATA['flash']
)
VIDEO_TRANSCRIPT_CUSTOM_SJSON = dict(
language_code='en',
transcript='edxval/tests/data/wow.sjson',
provider=TranscriptProviderType.CUSTOM,
file_format=TranscriptFormat.SJSON,
file_data=TRANSCRIPT_DATA['wow']
)
VIDEO_TRANSCRIPT_CIELO24 = dict(
video_id='super-soaker',
language_code='en',
transcript='edxval/tests/data/The_Flash.srt',
provider=TranscriptProviderType.CIELO24,
file_format=TranscriptFormat.SRT,
file_data=TRANSCRIPT_DATA['flash']
)
VIDEO_TRANSCRIPT_3PLAY = dict(
......@@ -377,6 +416,7 @@ VIDEO_TRANSCRIPT_3PLAY = dict(
transcript='edxval/tests/data/wow.sjson',
provider=TranscriptProviderType.THREE_PLAY_MEDIA,
file_format=TranscriptFormat.SJSON,
file_data=TRANSCRIPT_DATA['wow']
)
TRANSCRIPT_PREFERENCES_CIELO24 = dict(
......
edxval/tests/data/wow.sjson
{
"start": [10],
"end": [100],
"text": ["Hi, welcome to edxval."],
}
\ No newline at end of file
"text": ["Hi, welcome to edxval."]
}
edxval/tests/test_api.py
......@@ -30,26 +30,18 @@ from edxval.api import (InvalidTranscriptFormat, InvalidTranscriptProvider,
ValCannotUpdateError, ValVideoNotFoundError,
VideoSortField)
from edxval.models import (LIST_MAX_ITEMS, CourseVideo, EncodedVideo, Profile,
ThirdPartyTranscriptCredentialsState,
TranscriptFormat, TranscriptPreference,
ThirdPartyTranscriptCredentialsState, TranscriptPreference,
TranscriptProviderType, Video, VideoImage,
VideoTranscript)
from edxval.serializers import VideoSerializer
from edxval.tests import APIAuthTestCase, constants
STATIC_DIR = u'static'
STATIC_PATH = u'/static'
FILE_DATA = """
1
00:00:14,370 --> 00:00:16,530
I am overwatch.
2
00:00:16,500 --> 00:00:18,600
可以用“我不太懂艺术 但我知道我喜欢什么”做比喻.
"""
def omit_attrs(dict, attrs_to_omit=[]):
"""
Omits provided attributes from the dict.
"""
return {attr: value for attr, value in dict.iteritems() if attr not in attrs_to_omit}
class SortedVideoTestMixin(object):
......@@ -938,15 +930,16 @@ class ExportTest(TestCase):
# create internal video transcripts
transcript_data = dict(constants.VIDEO_TRANSCRIPT_CIELO24, video=video)
transcript_data.pop('video_id')
transcript_data = omit_attrs(transcript_data, ['video_id', 'file_data'])
VideoTranscript.objects.create(**transcript_data)
transcript_data = dict(constants.VIDEO_TRANSCRIPT_3PLAY, video=video)
transcript_data.pop('video_id')
transcript_data = omit_attrs(transcript_data, ['video_id', 'file_data'])
VideoTranscript.objects.create(**transcript_data)
self.temp_dir = mkdtemp()
self.file_system = OSFS(self.temp_dir)
self.file_system.makedir(STATIC_DIR, recreate=True)
delegate_fs = OSFS(self.temp_dir)
self.file_system = delegate_fs.makedir(constants.EXPORT_IMPORT_COURSE_DIR, recreate=True)
self.file_system.makedir(constants.EXPORT_IMPORT_STATIC_DIR, recreate=True)
self.addCleanup(shutil.rmtree, self.temp_dir)
def assert_xml_equal(self, left, right):
......@@ -981,7 +974,7 @@ class ExportTest(TestCase):
<video_asset client_video_id="TWINKLE TWINKLE" duration="122.0" image=""/>
""")
self.assert_xml_equal(
api.export_to_xml(constants.VIDEO_DICT_STAR['edx_video_id'], self.file_system, STATIC_DIR),
api.export_to_xml(constants.VIDEO_DICT_STAR['edx_video_id'], self.file_system, constants.EXPORT_IMPORT_STATIC_DIR),
expected
)
......@@ -993,7 +986,11 @@ class ExportTest(TestCase):
<video_asset client_video_id="TWINKLE TWINKLE" duration="122.0" image=""/>
""")
exported_xml = api.export_to_xml(constants.VIDEO_DICT_STAR['edx_video_id'], self.file_system, STATIC_DIR)
exported_xml = api.export_to_xml(
constants.VIDEO_DICT_STAR['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
self.assert_xml_equal(exported_xml, expected)
# Verify that no transcript is present in the XML.
......@@ -1014,14 +1011,19 @@ class ExportTest(TestCase):
<encoded_video url="http://www.meowmagic.com" file_size="33" bitrate="44" profile="desktop"/>
<encoded_video url="https://www.tmnt.com/tmnt101.m3u8" file_size="100" bitrate="0" profile="hls"/>
<transcripts>
<transcript file_format="sjson" file_name="edxval/tests/data/wow.sjson" language_code="de" provider="3PlayMedia" />
<transcript file_format="srt" file_name="edxval/tests/data/The_Flash.srt" language_code="en" provider="Cielo24" />
<transcript file_format="sjson" language_code="de" provider="3PlayMedia" />
<transcript file_format="srt" language_code="en" provider="Cielo24" />
</transcripts>
</video_asset>
""".format(image=image))
self.assert_xml_equal(
api.export_to_xml(constants.VIDEO_DICT_FISH['edx_video_id'], self.file_system, STATIC_DIR, course_id),
api.export_to_xml(
constants.VIDEO_DICT_FISH['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
course_id
),
expected
)
......@@ -1032,7 +1034,10 @@ class ExportTest(TestCase):
language_code = 'en'
video_id = constants.VIDEO_DICT_FISH['edx_video_id']
transcript_files = {'de': u'super-soaker-de.sjson', 'en': u'super-soaker-en.srt'}
expected_transcript_path = combine(self.temp_dir, STATIC_PATH)
expected_transcript_path = combine(
self.temp_dir,
combine(constants.EXPORT_IMPORT_COURSE_DIR, constants.EXPORT_IMPORT_STATIC_DIR)
)
expected_xml = self.parse_xml("""
<video_asset client_video_id="Shallow Swordfish" duration="122.0" image="image.jpg">
......@@ -1040,19 +1045,19 @@ class ExportTest(TestCase):
<encoded_video url="http://www.meowmagic.com" file_size="33" bitrate="44" profile="desktop"/>
<encoded_video url="https://www.tmnt.com/tmnt101.m3u8" file_size="100" bitrate="0" profile="hls"/>
<transcripts>
<transcript file_format="sjson" file_name="edxval/tests/data/wow.sjson" language_code="de" provider="3PlayMedia" />
<transcript file_format="srt" file_name="edxval/tests/data/The_Flash.srt" language_code="en" provider="Cielo24" />
<transcript file_format="sjson" language_code="de" provider="3PlayMedia" />
<transcript file_format="srt" language_code="en" provider="Cielo24" />
</transcripts>
</video_asset>
""")
exported_xml = api.export_to_xml(video_id, self.file_system, STATIC_DIR, 'test-course')
exported_xml = api.export_to_xml(video_id, self.file_system, constants.EXPORT_IMPORT_STATIC_DIR, 'test-course')
# Assert video and transcript xml is exported correctly.
self.assert_xml_equal(exported_xml, expected_xml)
# Verify transcript file is created.
self.assertItemsEqual(transcript_files.values(), self.file_system.listdir(STATIC_PATH))
self.assertItemsEqual(transcript_files.values(), self.file_system.listdir(constants.EXPORT_IMPORT_STATIC_DIR))
# Also verify the content of created transcript file.
for language_code in transcript_files.keys():
......@@ -1068,12 +1073,15 @@ class ExportTest(TestCase):
Test export with invalid video id.
"""
with self.assertRaises(ValVideoNotFoundError):
api.export_to_xml('unknown_video', self.file_system, STATIC_DIR)
api.export_to_xml('unknown_video', self.file_system, constants.EXPORT_IMPORT_STATIC_DIR)
@ddt
class ImportTest(TestCase):
"""Tests import_from_xml"""
"""
Tests import_from_xml
"""
def setUp(self):
self.image_name = 'image.jpg'
mobile_profile = Profile.objects.create(profile_name=constants.PROFILE_MOBILE)
......@@ -1090,6 +1098,16 @@ class ImportTest(TestCase):
self.transcript_data2 = dict(constants.VIDEO_TRANSCRIPT_3PLAY, video_id='little-star')
self.transcript_data3 = dict(self.transcript_data2, video_id='super-soaker')
self.temp_dir = mkdtemp()
self.file_system = OSFS(self.temp_dir)
self.file_system.makedir(constants.EXPORT_IMPORT_COURSE_DIR, recreate=True)
self.file_system.makedir(
constants.EXPORT_IMPORT_STATIC_DIR,
recreate=True
)
self.addCleanup(shutil.rmtree, self.temp_dir)
def make_import_xml(self, video_dict, encoded_video_dicts=None, image=None, video_transcripts=None):
import_xml = etree.Element(
"video_asset",
......@@ -1115,18 +1133,32 @@ class ImportTest(TestCase):
if video_transcripts:
transcripts_el = etree.SubElement(import_xml, 'transcripts')
for video_transcript in video_transcripts:
file_format = video_transcript['file_format']
language_code = video_transcript['language_code']
etree.SubElement(
transcripts_el,
'transcript',
{
'video_id': video_transcript['video_id'],
'file_name': video_transcript['transcript'],
'language_code': video_transcript['language_code'],
'file_format': video_transcript['file_format'],
'language_code': language_code,
'file_format': file_format,
'provider': video_transcript['provider'],
}
)
# Create transcript files
transcript_file_name = u'{edx_video_id}-{language_code}.{file_format}'.format(
edx_video_id=video_dict['edx_video_id'],
language_code=language_code,
file_format=file_format
)
utils.create_file_in_fs(
video_transcript['file_data'],
transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
return import_xml
def assert_obj_matches_dict_for_keys(self, obj, dict_, keys):
......@@ -1150,29 +1182,41 @@ class ImportTest(TestCase):
def assert_invalid_import(self, xml, course_id=None):
edx_video_id = "test_edx_video_id"
with self.assertRaises(ValCannotCreateError):
api.import_from_xml(xml, edx_video_id, course_id)
api.import_from_xml(
xml,
edx_video_id,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
course_id
)
self.assertFalse(Video.objects.filter(edx_video_id=edx_video_id).exists())
def assert_transcripts(self, video_id, expected_transcripts):
"""
Compare `received` with `expected` and assert if not equal
Compare `received` with `expected` and assert if not equal.
"""
# Verify total number of expected transcripts for a video
# Verify total number of expected transcripts for a video.
video_transcripts = VideoTranscript.objects.filter(video__edx_video_id=video_id)
self.assertEqual(video_transcripts.count(), len(expected_transcripts))
# Verify data for each transcript
# Verify data for each transcript.
for expected_transcript in expected_transcripts:
language_code = expected_transcript['language_code']
expected_transcript['name'] = expected_transcript.pop('transcript')
# get the imported transcript and rename `url` key
received = api.TranscriptSerializer(
# Get the imported transcript and remove `url` key.
received_transcript = api.TranscriptSerializer(
VideoTranscript.objects.get(video__edx_video_id=video_id, language_code=language_code)
).data
received['name'] = received.pop('url')
self.assertDictEqual(received, expected_transcript)
# Assert transcript content
received_transcript['file_data'] = api.get_video_transcript_data(video_id, language_code)['content']
# Omit not needed attrs.
expected_transcript = omit_attrs(expected_transcript, ['transcript'])
received_transcript = omit_attrs(received_transcript, ['url'])
self.assertDictEqual(received_transcript, expected_transcript)
def test_new_video_full(self):
new_course_id = 'new_course_id'
......@@ -1184,11 +1228,18 @@ class ImportTest(TestCase):
video_transcripts=[self.transcript_data1, self.transcript_data2]
)
# there must not be any transcript before import
with self.assertRaises(VideoTranscript.DoesNotExist):
VideoTranscript.objects.get(video__edx_video_id=constants.VIDEO_DICT_STAR['edx_video_id'])
# There must not be any transcript before import.
self.assert_transcripts(constants.VIDEO_DICT_STAR['edx_video_id'], [])
api.import_from_xml(xml, constants.VIDEO_DICT_STAR['edx_video_id'], new_course_id)
edx_video_id = api.import_from_xml(
xml,
constants.VIDEO_DICT_STAR['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
new_course_id
)
self.assertEqual(edx_video_id, constants.VIDEO_DICT_STAR['edx_video_id'])
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_STAR['edx_video_id'])
self.assert_video_matches_dict(video, constants.VIDEO_DICT_STAR)
......@@ -1217,7 +1268,7 @@ class ImportTest(TestCase):
"duration": "0",
}
)
api.import_from_xml(xml, edx_video_id)
api.import_from_xml(xml, edx_video_id, self.file_system, constants.EXPORT_IMPORT_STATIC_DIR)
video = Video.objects.get(edx_video_id=edx_video_id)
self.assertFalse(video.encoded_videos.all().exists())
......@@ -1234,29 +1285,37 @@ class ImportTest(TestCase):
transcript_data = dict(self.transcript_data3, language_code=language_code)
xml = self.make_import_xml(
video_dict={
"client_video_id": "new_client_video_id",
"duration": 0,
'edx_video_id': constants.VIDEO_DICT_FISH['edx_video_id'],
'client_video_id': 'new_client_video_id',
'duration': 0,
},
encoded_video_dicts=[
constants.ENCODED_VIDEO_DICT_FISH_DESKTOP,
{
"url": "http://example.com/new_url",
"file_size": 2733256,
"bitrate": 1597804,
"profile": "mobile",
'url': 'http://example.com/new_url',
'file_size': 2733256,
'bitrate': 1597804,
'profile': 'mobile',
},
],
image=self.image_name,
video_transcripts=[transcript_data]
)
# there must not be any transcript before import
with self.assertRaises(VideoTranscript.DoesNotExist):
VideoTranscript.objects.get(video__edx_video_id=constants.VIDEO_DICT_FISH["edx_video_id"])
# There must not be any transcript before import.
self.assert_transcripts(constants.VIDEO_DICT_FISH['edx_video_id'], [])
api.import_from_xml(xml, constants.VIDEO_DICT_FISH["edx_video_id"], course_id)
edx_video_id = api.import_from_xml(
xml,
constants.VIDEO_DICT_FISH['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
course_id
)
self.assertEqual(edx_video_id, constants.VIDEO_DICT_FISH['edx_video_id'])
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_FISH["edx_video_id"])
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_FISH['edx_video_id'])
self.assert_video_matches_dict(video, constants.VIDEO_DICT_FISH)
self.assert_encoded_video_matches_dict(
video.encoded_videos.get(profile__profile_name=constants.PROFILE_MOBILE),
......@@ -1270,14 +1329,21 @@ class ImportTest(TestCase):
self.assertTrue(course_video.video_image.image.name, self.image_name)
self.assert_transcripts(
constants.VIDEO_DICT_FISH["edx_video_id"],
constants.VIDEO_DICT_FISH['edx_video_id'],
[]
)
def test_existing_video_with_invalid_course_id(self):
xml = self.make_import_xml(video_dict=constants.VIDEO_DICT_FISH)
with self.assertRaises(ValCannotCreateError):
api.import_from_xml(xml, edx_video_id=constants.VIDEO_DICT_FISH["edx_video_id"], course_id="x" * 300)
api.import_from_xml(
xml,
constants.VIDEO_DICT_FISH['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
course_id='x' * 300
)
def test_unknown_profile(self):
profile = "unknown_profile"
......@@ -1293,9 +1359,14 @@ class ImportTest(TestCase):
}
]
)
api.import_from_xml(xml, constants.VIDEO_DICT_STAR["edx_video_id"])
api.import_from_xml(
xml,
constants.VIDEO_DICT_STAR['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_STAR["edx_video_id"])
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_STAR['edx_video_id'])
self.assertFalse(video.encoded_videos.filter(profile__profile_name=profile).exists())
def test_invalid_tag(self):
......@@ -1337,33 +1408,487 @@ class ImportTest(TestCase):
"""
Verify that transcript import for external video working as expected when there is no transcript.
"""
api.import_from_xml(etree.fromstring('<video_asset/>'), '')
api.import_from_xml(
etree.fromstring('<video_asset/>'),
'',
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
self.assertEqual(
VideoTranscript.objects.count(),
0
)
@data(
('external-transcript.srt', constants.VIDEO_TRANSCRIPT_CUSTOM_SRT),
('external-transcript.sjson', constants.VIDEO_TRANSCRIPT_CUSTOM_SJSON)
)
@unpack
def test_external_video_transcript(self, transcript_file_name, transcript_data):
"""
Verify that transcript import for external video working as expected when there is transcript present.
"""
# First create external transcript.
utils.create_file_in_fs(
transcript_data['file_data'],
transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
# Verify that one video is present before import.
self.assertEqual(Video.objects.count(), 1)
# Verify that no transript was present before import.
self.assertEqual(VideoTranscript.objects.count(), 0)
# Import xml with empty edx video id.
edx_video_id = api.import_from_xml(
etree.fromstring('<video_asset/>'),
'',
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{
'en': [transcript_file_name]
}
)
# Verify that a new video is created.
self.assertIsNotNone(edx_video_id)
# Verify transcript record is created with correct data.
self.assert_transcripts(
edx_video_id,
[dict(transcript_data, video_id=edx_video_id)]
)
def test_multiple_external_transcripts_different_langauges(self):
"""
Verify that transcript import for external video working as expected when multiple transcripts are imported.
"""
# First create external transcripts.
en_transcript_file_name = 'external-transcript-en.srt'
utils.create_file_in_fs(
constants.VIDEO_TRANSCRIPT_CUSTOM_SRT['file_data'],
en_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
es_transcript_file_name = 'external-transcript-es.srt'
utils.create_file_in_fs(
constants.VIDEO_TRANSCRIPT_CUSTOM_SRT['file_data'],
es_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
# Verify that one video is present before import.
self.assertEqual(Video.objects.count(), 1)
# Verify that no transript was present before import.
self.assertEqual(VideoTranscript.objects.count(), 0)
# Import xml with empty edx video id.
edx_video_id = api.import_from_xml(
etree.fromstring('<video_asset/>'),
'',
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{
'en': [en_transcript_file_name],
'es': [es_transcript_file_name]
}
)
# Verify that new video is created.
self.assertIsNotNone(edx_video_id)
# Verify transcript records are created with correct data.
expected_transcripts = [
dict(constants.VIDEO_TRANSCRIPT_CUSTOM_SRT, video_id=edx_video_id, language_code='en'),
dict(constants.VIDEO_TRANSCRIPT_CUSTOM_SRT, video_id=edx_video_id, language_code='es')
]
self.assert_transcripts(
edx_video_id,
expected_transcripts
)
def test_multiple_external_transcripts_for_language(self):
"""
Verify that transcript import for external video working as expected when multiple transcripts present against
a language e.g. external english transcript is imported through sub and transcripts field.
"""
# First create external transcripts.
sub_transcript_file_name = 'external-transcript-sub.srt'
utils.create_file_in_fs(
constants.VIDEO_TRANSCRIPT_CUSTOM_SRT['file_data'],
sub_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
ext_transcript_file_name = 'external-transcript-ext.sjson'
utils.create_file_in_fs(
constants.VIDEO_TRANSCRIPT_CUSTOM_SJSON['file_data'],
ext_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
# Verify that one video is present before import.
self.assertEqual(Video.objects.count(), 1)
# Verify that no transript was present before import.
self.assertEqual(VideoTranscript.objects.count(), 0)
# Import xml with empty edx video id.
edx_video_id = api.import_from_xml(
etree.fromstring('<video_asset/>'),
'',
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{
'en': [sub_transcript_file_name, ext_transcript_file_name]
}
)
# Verify that new video is created.
self.assertIsNotNone(edx_video_id)
# Verify transcript record is created with correct data i.e sub field transcript.
expected_transcripts = [
dict(constants.VIDEO_TRANSCRIPT_CUSTOM_SRT, video_id=edx_video_id, language_code='en')
]
self.assert_transcripts(
edx_video_id,
expected_transcripts
)
def test_external_internal_transcripts_conflict(self):
"""
Tests that when importing both external and internal (VAL) transcripts, internal transcript is imported.
"""
# First create external transcript in sjson format.
en_transcript_file_name = 'external-transcript-en.sjson'
utils.create_file_in_fs(
constants.VIDEO_TRANSCRIPT_CUSTOM_SJSON['file_data'],
en_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
# Let's create internal transcript in srt format.
expected_val_transcript = [self.transcript_data1]
import_xml = self.make_import_xml(
video_dict=constants.VIDEO_DICT_STAR,
video_transcripts=expected_val_transcript
)
# Verify that one video is present before import.
self.assertEqual(Video.objects.count(), 1)
# Verify that no transript was present before import.
self.assertEqual(VideoTranscript.objects.count(), 0)
# Note that we have an external en transcript as well as internal en transcript.
edx_video_id = api.import_from_xml(
import_xml,
constants.VIDEO_DICT_STAR['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{
'en': [en_transcript_file_name]
}
)
# Verify that new video is created.
self.assertIsNotNone(edx_video_id)
# Verify transcript record is created with internal transcript data.
self.assert_transcripts(
constants.VIDEO_DICT_STAR['edx_video_id'],
[self.transcript_data1]
)
def test_external_internal_transcripts_different_languages(self):
"""
Tests that when importing both external and internal (VAL) transcripts for different langauges, all transcripts
are imported correctly.
"""
edx_video_id = constants.VIDEO_DICT_STAR['edx_video_id']
# First create external es transcript.
es_transcript_file_name = 'external-transcript-es.sjson'
es_external_transcript = dict(
constants.VIDEO_TRANSCRIPT_CUSTOM_SJSON,
video_id=edx_video_id,
language_code='es'
)
utils.create_file_in_fs(
es_external_transcript['file_data'],
es_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
# Let's create en internal transcript.
import_xml = self.make_import_xml(
video_dict=constants.VIDEO_DICT_STAR,
video_transcripts=[self.transcript_data1]
)
# Verify that one video is present before import.
self.assertEqual(Video.objects.count(), 1)
# Verify that no transript was present before import.
self.assertEqual(VideoTranscript.objects.count(), 0)
# Note that we have an external 'es' language transcript as well as an internal 'es' language transcript.
edx_video_id = api.import_from_xml(
import_xml,
edx_video_id,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{
'es': [es_transcript_file_name]
}
)
# Verify all transcript records are created correctly.
self.assert_transcripts(
constants.VIDEO_DICT_STAR['edx_video_id'],
[self.transcript_data1, es_external_transcript]
)
@patch('edxval.api.logger')
def test_import_transcript_from_fs_resource_not_found(self, mock_logger):
"""
Test that `import_transcript_from_fs` correctly logs if transcript file is not found in file system.
"""
language_code = 'en'
edx_video_id = 'test-edx-video-id'
file_name = 'file-not-found.srt'
api.import_transcript_from_fs(
edx_video_id=edx_video_id,
language_code=language_code,
file_name=file_name,
provider=TranscriptProviderType.CUSTOM,
resource_fs=self.file_system,
static_dir=constants.EXPORT_IMPORT_STATIC_DIR
)
mock_logger.warn.assert_called_with(
'[edx-val] "%s" transcript "%s" for video "%s" is not found.',
language_code,
file_name,
edx_video_id
)
@patch('edxval.api.logger')
def test_import_transcript_from_fs_invalid_format(self, mock_logger):
"""
Test that `import_transcript_from_fs` correctly logs if we get error while retrieving transcript file format.
"""
language_code = 'en'
edx_video_id = constants.VIDEO_DICT_FISH['edx_video_id']
# First create transcript file.
invalid_transcript_file_name = 'invalid-transcript.txt'
invalid_transcript = dict(
constants.VIDEO_TRANSCRIPT_CUSTOM_SJSON,
video_id=edx_video_id,
file_data='This is an invalid transcript file data.'
)
utils.create_file_in_fs(
invalid_transcript['file_data'],
invalid_transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
api.import_transcript_from_fs(
edx_video_id=edx_video_id,
language_code=language_code,
file_name=invalid_transcript_file_name,
provider=TranscriptProviderType.CUSTOM,
resource_fs=self.file_system,
static_dir=constants.EXPORT_IMPORT_STATIC_DIR
)
mock_logger.warn.assert_called_with(
'[edx-val] Error while getting transcript format for video=%s -- language_code=%s --file_name=%s',
edx_video_id,
language_code,
invalid_transcript_file_name
)
@patch('edxval.api.logger')
def test_import_transcript_from_fs_bad_content(self, mock_logger):
"""
Test that `import_transcript_from_fs` correctly logs if we get error while decoding transcript content.
"""
language_code = 'en'
edx_video_id = constants.VIDEO_DICT_FISH['edx_video_id']
# First create transcript file.
transcript_file_name = 'invalid-transcript.txt'
invalid_transcript = dict(
constants.VIDEO_TRANSCRIPT_CUSTOM_SJSON,
video_id=edx_video_id,
file_data=u'Привіт, edX вітає вас.'.encode('cp1251')
)
utils.create_file_in_fs(
invalid_transcript['file_data'],
transcript_file_name,
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
api.import_transcript_from_fs(
edx_video_id=edx_video_id,
language_code=language_code,
file_name=transcript_file_name,
provider=TranscriptProviderType.CUSTOM,
resource_fs=self.file_system,
static_dir=constants.EXPORT_IMPORT_STATIC_DIR
)
mock_logger.warn.assert_called_with(
'[edx-val] "%s" transcript "%s" for video "%s" contains a non-utf8 file content.',
language_code,
transcript_file_name,
edx_video_id
)
def test_import_transcript_attached_existing_video(self):
"""
Verify that transcript import for existing video with transcript attached is working as expected.
"""
expected_video_transcripts = [self.transcript_data3]
import_xml = self.make_import_xml(
video_dict=constants.VIDEO_DICT_FISH,
video_transcripts=expected_video_transcripts
)
# Verify video is present before.
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_FISH['edx_video_id'])
self.assertIsNotNone(video)
# Create internal video transcripts
transcript_data = dict(constants.VIDEO_TRANSCRIPT_3PLAY, video=video)
transcript_data = omit_attrs(transcript_data, ['video_id', 'file_data'])
transcript = VideoTranscript.objects.create(**transcript_data)
# Verify that video has expected transcripts before import.
self.assert_transcripts(
constants.VIDEO_DICT_FISH['edx_video_id'],
expected_video_transcripts
)
api.import_from_xml(
import_xml,
constants.VIDEO_DICT_FISH['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
'test_course_id'
)
# Verify that video has expected transcripts after import.
self.assert_transcripts(
constants.VIDEO_DICT_FISH['edx_video_id'],
expected_video_transcripts
)
def test_import_no_transcript_attached_existing_video(self):
"""
Verify that transcript import for existing video with no transcript attached is working as expected.
"""
exported_video_transcripts = [self.transcript_data1, self.transcript_data2]
# Verify video is present before.
video = Video.objects.get(edx_video_id=constants.VIDEO_DICT_FISH['edx_video_id'])
self.assertIsNotNone(video)
import_xml = self.make_import_xml(
video_dict=constants.VIDEO_DICT_FISH,
video_transcripts=exported_video_transcripts
)
# There must not be any transcript before import.
self.assert_transcripts(constants.VIDEO_DICT_FISH['edx_video_id'], [])
api.import_from_xml(
import_xml,
constants.VIDEO_DICT_FISH['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
'test_course_id'
)
# Verify that no transcript record is created.
self.assert_transcripts(constants.VIDEO_DICT_FISH['edx_video_id'], [])
def test_import_transcript_new_video(self):
"""
Verify that transcript import for new video is working as expected when transcript is present in XML.
"""
expected_video_transcripts = [self.transcript_data1, self.transcript_data2]
import_xml = self.make_import_xml(
video_dict=constants.VIDEO_DICT_STAR,
video_transcripts=expected_video_transcripts
)
# Verify video is not present before.
with self.assertRaises(Video.DoesNotExist):
Video.objects.get(edx_video_id=constants.VIDEO_DICT_STAR['edx_video_id'])
# There must not be any transcript before import.
self.assert_transcripts(constants.VIDEO_DICT_STAR['edx_video_id'], [])
api.import_from_xml(
import_xml,
constants.VIDEO_DICT_STAR['edx_video_id'],
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR,
{},
'test_course_id'
)
# Verify that transcript record is created with correct data.
self.assert_transcripts(
constants.VIDEO_DICT_STAR['edx_video_id'],
expected_video_transcripts
)
@patch('edxval.api.logger')
def test_video_transcript_missing_attribute(self, mock_logger):
"""
Verify that video transcript import working as expected if transcript xml data is missing.
"""
video_id = 'super-soaker'
transcript_xml = '<transcript file_name="wow.srt" language_code="en" file_format="srt" provider="Cielo24"/>'
transcript_xml = '<transcript file_format="srt" provider="Cielo24"/>'
xml = etree.fromstring("""
<video_asset>
<transcripts>
{transcript_xml}
<transcript file_name="edxval/tests/data/wow.sjson" language_code="de" file_format="sjson" provider='3PlayMedia' video_id="{video_id}"/>
<transcript language_code="de" file_format="sjson" provider='3PlayMedia'/>
</transcripts>
</video_asset>
""".format(transcript_xml=transcript_xml, video_id=video_id))
""".format(transcript_xml=transcript_xml))
# there should be no video transcript before import
# There should be no video transcript before import
with self.assertRaises(VideoTranscript.DoesNotExist):
VideoTranscript.objects.get(video__edx_video_id=video_id)
api.create_transcript_objects(xml)
# Create transcript files
utils.create_file_in_fs(
constants.TRANSCRIPT_DATA['wow'],
u'super-soaker-de.sjson',
self.file_system,
constants.EXPORT_IMPORT_STATIC_DIR
)
api.create_transcript_objects(xml, video_id, self.file_system, constants.EXPORT_IMPORT_STATIC_DIR, {})
mock_logger.warn.assert_called_with(
"VAL: Required attributes are missing from xml, xml=[%s]",
......@@ -1710,15 +2235,15 @@ class TranscriptTest(TestCase):
'language_code': 'en',
'provider': TranscriptProviderType.THREE_PLAY_MEDIA,
'file_name': None,
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'file_data': File(open(self.flash_transcript_path))
},
{
'language_code': 'fr',
'provider': TranscriptProviderType.CIELO24,
'file_name': None,
'file_format': TranscriptFormat.SRT,
'file_data': ContentFile(FILE_DATA)
'file_format': utils.TranscriptFormat.SRT,
'file_data': ContentFile(constants.TRANSCRIPT_DATA['overwatch'])
}
]
)
......@@ -1734,14 +2259,14 @@ class TranscriptTest(TestCase):
'language_code': 'de',
'provider': TranscriptProviderType.CUSTOM,
'file_name': None,
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'file_data': File(open(self.arrow_transcript_path))
},
{
'language_code': 'zh',
'provider': TranscriptProviderType.CUSTOM,
'file_name': 'non/existent/transcript/path',
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'file_data': None
}
]
......@@ -1818,7 +2343,7 @@ class TranscriptTest(TestCase):
expectation = {
'video_id': u'super-soaker',
'url': self.v1_transcript2.url(),
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'provider': TranscriptProviderType.CIELO24,
'language_code': u'fr'
}
......@@ -1874,14 +2399,14 @@ class TranscriptTest(TestCase):
{
'file_data': None,
'file_name': 'overwatch.sjson',
'file_format': TranscriptFormat.SJSON,
'file_format': utils.TranscriptFormat.SJSON,
'language_code': 'da',
'provider': TranscriptProviderType.CIELO24
},
{
'file_data': ContentFile(FILE_DATA),
'file_data': ContentFile(constants.TRANSCRIPT_DATA['overwatch']),
'file_name': None,
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'language_code': 'es',
'provider': TranscriptProviderType.THREE_PLAY_MEDIA
},
......@@ -1922,7 +2447,7 @@ class TranscriptTest(TestCase):
self.assertTrue(transcript_url.startswith(settings.VIDEO_TRANSCRIPTS_SETTINGS['DIRECTORY_PREFIX']))
self.assertEqual(video_transcript.transcript.name, transcript_url)
with open(video_transcript.transcript.name) as saved_transcript:
self.assertEqual(saved_transcript.read(), FILE_DATA)
self.assertEqual(saved_transcript.read(), constants.TRANSCRIPT_DATA['overwatch'])
else:
self.assertEqual(video_transcript.transcript.name, file_name)
......@@ -1936,7 +2461,7 @@ class TranscriptTest(TestCase):
},
{
'video_id': 'medium-soaker',
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'provider': 123,
'exception': InvalidTranscriptProvider,
'exception_message': '123 transcript provider is not supported',
......@@ -1965,8 +2490,8 @@ class TranscriptTest(TestCase):
video_id=edx_video_id,
language_code=language_code,
provider=TranscriptProviderType.THREE_PLAY_MEDIA,
file_format=TranscriptFormat.SRT,
content=ContentFile(FILE_DATA)
file_format=utils.TranscriptFormat.SRT,
content=ContentFile(constants.TRANSCRIPT_DATA['overwatch'])
)
# setup video with the `edx_video_id` above.
......@@ -1988,7 +2513,7 @@ class TranscriptTest(TestCase):
self.assertEqual(video_transcript.file_format, transcript_props['file_format'])
self.assertEqual(video_transcript.provider, transcript_props['provider'])
with open(video_transcript.transcript.name) as created_transcript:
self.assertEqual(created_transcript.read(), FILE_DATA)
self.assertEqual(created_transcript.read(), constants.TRANSCRIPT_DATA['overwatch'])
@data(
{
......@@ -2001,7 +2526,7 @@ class TranscriptTest(TestCase):
{
'video_id': 'medium-soaker',
'language_code': 'en',
'file_format': TranscriptFormat.SRT,
'file_format': utils.TranscriptFormat.SRT,
'provider': 'unknown provider',
'exception_msg': '"unknown provider" is not a valid choice.'
}
......@@ -2012,7 +2537,7 @@ class TranscriptTest(TestCase):
Verify that `create_video_transcript` api function raise exceptions on invalid values.
"""
with self.assertRaises(ValCannotCreateError) as transcript_exception:
api.create_video_transcript(video_id, language_code, file_format, ContentFile(FILE_DATA), provider)
api.create_video_transcript(video_id, language_code, file_format, ContentFile(constants.TRANSCRIPT_DATA['overwatch']), provider)
self.assertIn(exception_msg, unicode(transcript_exception.exception.message))
......@@ -2030,7 +2555,7 @@ class TranscriptTest(TestCase):
video_id=edx_video_id,
language_code='en',
metadata=dict(provider=TranscriptProviderType.CIELO24),
file_data=ContentFile(FILE_DATA)
file_data=ContentFile(constants.TRANSCRIPT_DATA['overwatch'])
)
# Verify that new transcript is set to video
......@@ -2039,7 +2564,7 @@ class TranscriptTest(TestCase):
# verify that new data is written correctly
with open(video_transcript.transcript.name) as saved_transcript:
self.assertEqual(saved_transcript.read(), FILE_DATA)
self.assertEqual(saved_transcript.read(), constants.TRANSCRIPT_DATA['overwatch'])
# Verify that an exception is raised if we try to open a deleted transcript file
with self.assertRaises(IOError) as file_open_exception:
......@@ -2088,22 +2613,26 @@ class TranscriptTest(TestCase):
language_code = 'en'
video_id = constants.VIDEO_DICT_FISH['edx_video_id']
transcript_file_name = u'super-soaker-en.srt'
expected_transcript_path = combine(self.temp_dir, combine(STATIC_PATH, transcript_file_name))
expected_transcript_path = combine(
combine(self.temp_dir, constants.EXPORT_IMPORT_COURSE_DIR),
combine(constants.EXPORT_IMPORT_STATIC_DIR, transcript_file_name)
)
file_system = OSFS(self.temp_dir)
file_system.makedir(STATIC_DIR, recreate=True)
delegate_fs = OSFS(self.temp_dir)
file_system = delegate_fs.makedir(constants.EXPORT_IMPORT_COURSE_DIR, recreate=True)
file_system.makedir(constants.EXPORT_IMPORT_STATIC_DIR, recreate=True)
# Create transcript file now.
api.create_trancript_file(
api.create_transcript_file(
video_id=video_id,
language_code=language_code,
file_format=TranscriptFormat.SRT,
static_dir=STATIC_DIR,
file_format=utils.TranscriptFormat.SRT,
static_dir=constants.EXPORT_IMPORT_STATIC_DIR,
resource_fs=file_system
)
# Verify transcript file is created.
self.assertTrue(transcript_file_name in file_system.listdir(STATIC_PATH))
self.assertTrue(transcript_file_name in file_system.listdir(constants.EXPORT_IMPORT_STATIC_DIR))
# Also verify the content of created transcript file.
expected_transcript_content = File(open(expected_transcript_path)).read()
......@@ -2120,19 +2649,19 @@ class TranscriptTest(TestCase):
Tests that no transcript file is created in case of invalid scenario.
"""
file_system = OSFS(self.temp_dir)
file_system.makedir(STATIC_DIR, recreate=True)
file_system.makedir(constants.EXPORT_IMPORT_STATIC_DIR, recreate=True)
# Try to create transcript file now.
api.create_trancript_file(
api.create_transcript_file(
video_id=video_id,
language_code=language_code,
file_format=TranscriptFormat.SRT,
static_dir=STATIC_DIR,
file_format=utils.TranscriptFormat.SRT,
static_dir=constants.EXPORT_IMPORT_STATIC_DIR,
resource_fs=file_system
)
# Verify no file is created.
self.assertEqual(file_system.listdir(STATIC_PATH), [])
self.assertEqual(file_system.listdir(constants.EXPORT_IMPORT_STATIC_DIR), [])
@ddt
......
edxval/tests/test_views.py
......@@ -8,10 +8,11 @@ from ddt import data, ddt, unpack
from django.core.urlresolvers import reverse
from rest_framework import status
from edxval.models import (CourseVideo, Profile, TranscriptFormat,
from edxval.models import (CourseVideo, Profile,
TranscriptProviderType, Video, VideoTranscript)
from edxval.serializers import TranscriptSerializer
from edxval.tests import APIAuthTestCase, constants
from edxval.utils import TranscriptFormat
class VideoDetail(APIAuthTestCase):
......@@ -813,6 +814,7 @@ class VideoTranscriptViewTest(APIAuthTestCase):
Tests POSTing transcript successfully.
"""
post_transcript_data = dict(self.transcript_data)
post_transcript_data.pop('file_data')
post_transcript_data['name'] = post_transcript_data.pop('transcript')
response = self.client.post(self.url, post_transcript_data, format='json')
......
edxval/utils.py
......@@ -2,8 +2,22 @@
Util methods to be used in api and models.
"""
import json
from django.conf import settings
from django.core.files.storage import get_storage_class
from fs.path import combine
from pysrt import SubRipFile
class TranscriptFormat(object):
SRT = 'srt'
SJSON = 'sjson'
CHOICES = (
(SRT, 'SubRip'),
(SJSON, 'SRT JSON')
)
# 3rd Party Transcription Plans
THIRD_PARTY_TRANSCRIPTION_PLANS = {
......@@ -169,3 +183,35 @@ def get_video_transcript_storage():
# during edx-platform loading this method gets called but settings are not ready yet
# so in that case we will return default(FileSystemStorage) storage class instance
return get_storage_class()()
def create_file_in_fs(file_data, file_name, file_system, static_dir):
"""
Writes file in specific file system.
Arguments:
file_data (str): Data to store into the file.
file_name (str): File name of the file to be created.
resource_fs (OSFS): Import file system.
static_dir (str): The Directory to retrieve transcript file.
"""
with file_system.open(combine(static_dir, file_name), 'wb') as f:
f.write(file_data)
def get_transcript_format(transcript_content):
"""
Returns transcript format.
Arguments:
transcript_content (str): Transcript file content.
"""
try:
sjson_obj = json.loads(transcript_content)
except ValueError:
# With error handling (set to 'ERROR_RAISE'), we will be getting
# the exception if something went wrong in parsing the transcript.
srt_subs = SubRipFile.from_string(transcript_content, error_handling=SubRipFile.ERROR_RAISE)
if len(srt_subs) > 0:
return TranscriptFormat.SRT
return TranscriptFormat.SJSON
edxval/views.py
......@@ -15,13 +15,13 @@ from rest_framework_oauth.authentication import OAuth2Authentication
from edxval.api import create_or_update_video_transcript
from edxval.models import (
CourseVideo,
TranscriptFormat,
TranscriptProviderType,
Video,
VideoImage,
VideoTranscript
)
from edxval.serializers import VideoSerializer
from edxval.utils import TranscriptFormat
LOGGER = logging.getLogger(__name__) # pylint: disable=C0103
......
requirements/base.in
......@@ -10,3 +10,4 @@ django-storages
enum34
lxml
pillow
pysrt==0.4.7
setup.py
......@@ -41,7 +41,7 @@ def load_requirements(*requirements_paths):
setup(
name='edxval',
version='0.1.12',
version='0.1.13',
author='edX',
url='http://github.com/edx/edx-val',
description='edx-val',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment