Skip to content

E
edx-video-pipeline
Commit 79a90655 by Muzaffar yousaf Committed by GitHub
Options

Merge pull request #22 from edx/transcripts-3rd-party-integration 

Transcripts 3rd party integration
parents 5f53d133 a3a193b9
Showing with 1809 additions and 266 deletions
.coveragerc
# .coveragerc for edx-video-pipeline
[run]
data_file = reports/.coverage
data_file = .coverage
source =
VEDA
VEDA_OS01
control
frontend
youtube_callback
scripts
# do not calculate coverage for these for now
# youtube_callback
# scripts
omit =
templates/*
frontend/tests/*
dependencies/*
control/*
control/tests/*
VEDA/tests/*
VEDA_OS01/tests/*
VEDA_OS01/migrations/*
VEDA_OS01/admin.py
concurrency=multiprocessing
......@@ -28,5 +35,3 @@ exclude_lines =
title = edx-video-worker Python Test Coverage Report
directory = reports/cover
[xml]
output = reports/coverage.xml
.gitignore
......@@ -3,10 +3,16 @@
*.pyc
static/admin/
static/
sandbox.db
.coverage
coverage/
reports/
.cache/
VEDA/private.py
.travis.yml
......@@ -5,13 +5,14 @@ python:
sudo: required
before_install:
- export BOTO_CONFIG=/dev/null
install:
- pip install -r requirements.txt
- pip install -r test_requirements.txt
- make requirements
# build tests
script:
- make validate
- make validate
after_success:
- pip install -U codecov
......
AUTHORS 0 → 100644
Mushtaq Ali <mushtaak@gmail.com>
Muhammad Ammar <mammar@gmail.com>
Makefile
PACKAGES = VEDA VEDA_OS01 control frontend youtube_callback scripts
requirements:
pip install -r requirements.txt
pip install -r test_requirements.txt
validate: test ## Run tests and quality checks
test: clean
nosetests --with-coverage --cover-inclusive --cover-branches \
--cover-html --cover-html-dir=build/coverage/html/ \
--cover-xml --cover-xml-file=build/coverage/coverage.xml --verbosity=2 \
$(foreach package,$(PACKAGES),--cover-package=$(package)) \
$(PACKAGES)
coverage run -m pytest --durations=10
coverage combine
coverage report
clean:
coverage erase
......
VEDA/django_secrets.py
......@@ -21,5 +21,5 @@ with open(read_yaml, 'r') as stream:
DJANGO_SECRET_KEY = return_dict['django_secret_key'] or 'test_secret_key'
DJANGO_ADMIN = ('', '')
DJANGO_DEBUG = return_dict['debug']
DJANGO_DEBUG = return_dict['debug'] if 'debug' in return_dict else False
DATABASES = return_dict['DATABASES']
VEDA/settings.py
......@@ -2,6 +2,8 @@
Settings
"""
from os.path import join, dirname, abspath
DATABASES = None
import os
......@@ -138,6 +140,7 @@ INSTALLED_APPS = (
'rest_framework.authtoken',
'oauth2_provider',
'rest_framework',
'django_filters',
'corsheaders',
'frontend',
'VEDA_OS01',
......@@ -175,3 +178,7 @@ LOGGING = {
},
}
}
# See if the developer has any local overrides.
if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):
from .private import * # pylint: disable=import-error, wildcard-import
VEDA/test_settings.py 0 → 100644
"""
Test Settings
"""
from settings import *
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': 'pipeline.db',
}
}
VEDA/urls.py
......@@ -10,7 +10,7 @@ from rest_framework import routers
from django.conf.urls import patterns, include, url
from django.contrib import admin
from VEDA_OS01 import views
from VEDA_OS01 import views, transcripts
router = routers.DefaultRouter()
admin.autodiscover()
......@@ -33,5 +33,16 @@ urlpatterns = [
url(r'^api/', include(router.urls)),
url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),
# Cheap auth server
url(r'^veda_auth/', views.token_auth)
url(r'^veda_auth/', views.token_auth),
url(
regex=r'^cielo24/transcript_completed/(?P<token>[\w]+)$',
view=transcripts.Cielo24CallbackHandlerView.as_view(),
name='cielo24_transcript_completed'
),
# 3PlayMedia callback handler view
url(
regex=r'^3playmedia/transcripts/handle/(?P<token>[\w]+)$',
view=transcripts.ThreePlayMediaCallbackHandlerView.as_view(),
name='3play_media_callback'
)
]
VEDA_OS01/admin.py
from django.contrib import admin
from VEDA_OS01.models import Course, Video, Encode, URL, Destination, Institution, VedaUpload
from VEDA_OS01.models import (
Course, Video, Encode, URL, Destination, Institution, VedaUpload,
TranscriptCredentials, TranscriptProcessMetadata
)
class CourseAdmin(admin.ModelAdmin):
......@@ -30,7 +34,15 @@ class VideoAdmin(admin.ModelAdmin):
'studio_id',
'video_trans_start',
'video_trans_status',
'video_active'
'transcript_status',
'video_active',
'process_transcription',
'source_language',
'provider',
'three_play_turnaround',
'cielo24_turnaround',
'cielo24_fidelity',
'preferred_languages',
]
list_filter = ['inst_class__institution']
search_fields = ['edx_id', 'client_title', 'studio_id']
......@@ -101,6 +113,14 @@ class VideoUploadAdmin(admin.ModelAdmin):
]
class TranscriptCredentialsAdmin(admin.ModelAdmin):
model = TranscriptCredentials
class TranscriptProcessMetadataAdmin(admin.ModelAdmin):
model = TranscriptProcessMetadata
admin.site.register(Course, CourseAdmin)
admin.site.register(Video, VideoAdmin)
admin.site.register(Encode, EncodeAdmin)
......@@ -108,3 +128,5 @@ admin.site.register(URL, URLAdmin)
admin.site.register(Destination, DestinationAdmin)
admin.site.register(Institution, InstitutionAdmin)
admin.site.register(VedaUpload, VideoUploadAdmin)
admin.site.register(TranscriptCredentials, TranscriptCredentialsAdmin)
admin.site.register(TranscriptProcessMetadata, TranscriptProcessMetadataAdmin)
VEDA_OS01/migrations/0002_auto_20171016_1211.py 0 → 100644
# -*- coding: utf-8 -*-
# Generated by Django 1.9 on 2017-10-16 12:11
from __future__ import unicode_literals
import VEDA_OS01.models
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import model_utils.fields
class Migration(migrations.Migration):
dependencies = [
('VEDA_OS01', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='TranscriptCredentials',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
('org', models.CharField(help_text=b'This value must match the value of organization in studio/edx-platform.', max_length=50, verbose_name=b'Organization')),
('provider', models.CharField(choices=[(b'3PlayMedia', b'3PlayMedia'), (b'Cielo24', b'Cielo24')], max_length=50, verbose_name=b'Transcript provider')),
('api_key', models.CharField(max_length=255, verbose_name=b'API key')),
('api_secret', models.CharField(blank=True, max_length=255, null=True, verbose_name=b'API secret')),
],
options={
'verbose_name_plural': 'Transcript Credentials',
},
),
migrations.CreateModel(
name='TranscriptProcessMetadata',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
('provider', models.CharField(choices=[(b'3PlayMedia', b'3PlayMedia'), (b'Cielo24', b'Cielo24')], max_length=50, verbose_name=b'Transcript provider')),
('process_id', models.CharField(max_length=255, verbose_name=b'Process id')),
('translation_id', models.CharField(blank=True, help_text=b'Keeps track of 3Play Translations', max_length=255, null=True, verbose_name=b'Translation id')),
('lang_code', models.CharField(max_length=50, verbose_name=b'Language code')),
('status', models.CharField(choices=[(b'N/A', b'N/A'), (b'PENDING', b'PENDING'), (b'IN PROGRESS', b'IN PROGRESS'), (b'FAILED', b'FAILED'), (b'READY', b'READY')], default=b'PENDING', max_length=50, verbose_name=b'Transcript status')),
],
options={
'get_latest_by': 'modified',
'verbose_name_plural': 'Transcript process metadata',
},
),
migrations.AddField(
model_name='video',
name='cielo24_fidelity',
field=models.CharField(blank=True, choices=[(b'MECHANICAL', b'Mechanical, 75% Accuracy'), (b'PREMIUM', b'Premium, 95% Accuracy'), (b'PROFESSIONAL', b'Professional, 99% Accuracy')], max_length=20, null=True, verbose_name=b'Cielo24 Fidelity'),
),
migrations.AddField(
model_name='video',
name='cielo24_turnaround',
field=models.CharField(blank=True, choices=[(b'STANDARD', b'Standard, 48h'), (b'PRIORITY', b'Priority, 24h')], max_length=20, null=True, verbose_name=b'Cielo24 Turnaround'),
),
migrations.AddField(
model_name='video',
name='preferred_languages',
field=VEDA_OS01.models.ListField(blank=True, default=[]),
),
migrations.AddField(
model_name='video',
name='process_transcription',
field=models.BooleanField(default=False, verbose_name=b'Process transcripts from Cielo24/3PlayMedia'),
),
migrations.AddField(
model_name='video',
name='provider',
field=models.CharField(blank=True, choices=[(b'3PlayMedia', b'3PlayMedia'), (b'Cielo24', b'Cielo24')], max_length=20, null=True, verbose_name=b'Transcription provider'),
),
migrations.AddField(
model_name='video',
name='source_language',
field=models.CharField(blank=True, help_text=b'This is video speech language.', max_length=50, null=True, verbose_name=b'video source language'),
),
migrations.AddField(
model_name='video',
name='three_play_turnaround',
field=models.CharField(blank=True, choices=[(b'extended_service', b'10-Day/Extended'), (b'default', b'4-Day/Default'), (b'expedited_service', b'2-Day/Expedited'), (b'rush_service', b'24 hour/Rush'), (b'same_day_service', b'Same Day')], max_length=20, null=True, verbose_name=b'3PlayMedia Turnaround'),
),
migrations.AddField(
model_name='video',
name='transcript_status',
field=models.CharField(choices=[(b'N/A', b'N/A'), (b'PENDING', b'PENDING'), (b'IN PROGRESS', b'IN PROGRESS'), (b'FAILED', b'FAILED'), (b'READY', b'READY')], default=b'N/A', max_length=100, verbose_name=b'Transcription Status'),
),
migrations.AddField(
model_name='transcriptprocessmetadata',
name='video',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='transcript_processes', to='VEDA_OS01.Video'),
),
migrations.AlterUniqueTogether(
name='transcriptcredentials',
unique_together=set([('org', 'provider')]),
),
]
VEDA_OS01/serializers.py
......@@ -72,8 +72,16 @@ class VideoSerializer(serializers.ModelSerializer):
'video_trans_start',
'video_trans_end',
'video_trans_status',
'transcript_status',
'video_glacierid',
'course_ids'
'course_ids',
'process_transcription',
'source_language',
'provider',
'three_play_turnaround',
'cielo24_turnaround',
'cielo24_fidelity',
'preferred_languages',
)
def get_course_ids(self, video):
......@@ -124,10 +132,42 @@ class VideoSerializer(serializers.ModelSerializer):
'video_trans_status',
instance.video_trans_status
)
instance.transcript_status = validated_data.get(
'transcript_status',
instance.transcript_status
)
instance.video_glacierid = validated_data.get(
'video_glacierid',
instance.video_glacierid
)
instance.process_transcription = validated_data.get(
'process_transcription',
instance.process_transcription
)
instance.source_language = validated_data.get(
'source_language',
instance.source_language
)
instance.provider = validated_data.get(
'provider',
instance.provider
)
instance.three_play_turnaround = validated_data.get(
'three_play_turnaround',
instance.three_play_turnaround
)
instance.cielo24_turnaround = validated_data.get(
'cielo24_turnaround',
instance.cielo24_turnaround
)
instance.cielo24_fidelity = validated_data.get(
'cielo24_fidelity',
instance.cielo24_fidelity
)
instance.preferred_languages = validated_data.get(
'preferred_languages',
instance.preferred_languages
)
instance.save()
return instance
......
VEDA_OS01/tests/test_serializers.py 0 → 100644
from django.test import TestCase
from VEDA_OS01.models import Course, Destination, Encode, URL, Video
from VEDA_OS01.serializers import CourseSerializer, EncodeSerializer, URLSerializer, VideoSerializer
class TestCourseSerializer(TestCase):
"""
Tests for `CourseSerializer`.
"""
def setUp(self):
self.course_props = dict(
course_name=u'Intro to VEDA',
institution=u'MAx',
edx_classid=u'123',
semesterid=u'2017',
)
def test_create_course(self):
"""
Tests that `CourseSerializer.create` works as expected.
"""
course_serializer = CourseSerializer(data=self.course_props)
course_serializer.is_valid(raise_exception=True)
course_serializer.save()
# Now, get the created course record.
serialized_course = CourseSerializer(
instance=Course.objects.get(**self.course_props)
).data
self.assertDictEqual(serialized_course, course_serializer.data)
def test_update_course(self):
"""
Tests that `CourseSerializer.update` works as expected.
"""
course = Course.objects.create(**self.course_props)
# Perform the update via serializer.
updated_course_props = dict(self.course_props, course_name=u'Intro to edx-video-pipeline')
course_serializer = CourseSerializer(instance=course, data=updated_course_props, partial=True)
course_serializer.is_valid(raise_exception=True)
course_serializer.save()
# Now, see if its updated
serialized_course = CourseSerializer(
instance=Course.objects.first()
).data
self.assertDictEqual(serialized_course, course_serializer.data)
class TestVideoSerializer(TestCase):
"""
Tests for `VideoSerializer`.
"""
def setUp(self):
self.course = Course.objects.create(
course_name=u'Intro to VEDA',
institution=u'MAx',
edx_classid=u'123',
semesterid=u'2017',
local_storedir='course_id1, course_id2',
)
self.video_props = dict(
inst_class=self.course.pk,
client_title=u'Intro to video',
edx_id=u'12345678',
studio_id=u'43211234',
video_active=True,
process_transcription=True,
source_language=u'fr',
)
def test_create_video(self):
"""
Tests that `VideoSerializer.create` works as expected.
"""
video_serializer = VideoSerializer(data=self.video_props)
video_serializer.is_valid(raise_exception=True)
video_serializer.save()
# Now, get the created video record.
serialized_video = VideoSerializer(
instance=Video.objects.get(**self.video_props)
).data
self.assertDictEqual(serialized_video, video_serializer.data)
def test_update_video(self):
"""
Tests that `VideoSerializer.update` works as expected.
"""
video = Video.objects.create(**dict(self.video_props, inst_class=self.course))
# Perform the update via serializer.
updated_video_props = dict(self.video_props, client_title=u'Intro to new Video')
video_serializer = VideoSerializer(instance=video, data=updated_video_props, partial=True)
video_serializer.is_valid(raise_exception=True)
video_serializer.save()
# Now, see if its updated
serialized_video = VideoSerializer(
instance=Video.objects.first()
).data
self.assertDictEqual(serialized_video, video_serializer.data)
class TestURLSerializer(TestCase):
"""
Tests for `URLSerializer`.
"""
def setUp(self):
# Setup an encode
destination = Destination.objects.create(
destination_name='test_destination',
destination_nick='des',
destination_active=True
)
encode = Encode.objects.create(
encode_destination=destination,
encode_name='desktop_mp4',
profile_active=True,
)
# Setup a video
course = Course.objects.create(
course_name=u'Intro to VEDA',
institution=u'MAx',
edx_classid=u'123',
semesterid=u'2017',
local_storedir='course_id1, course_id2',
)
video = Video.objects.create(
inst_class=course,
client_title=u'Intro to video',
edx_id=u'12345678',
studio_id=u'43211234'
)
# Setup URL properties
self.url_props = dict(
encode_profile=encode.pk,
videoID=video.pk,
encode_url='https://www.s3.amazon.com/123.mp4'
)
def test_create_url(self):
"""
Tests that `URLSerializer.create` works as expected.
"""
url_serializer = URLSerializer(data=self.url_props)
url_serializer.is_valid(raise_exception=True)
url_serializer.save()
# Now, get the created URL record.
serialized_url = URLSerializer(
instance=URL.objects.first()
).data
self.assertDictEqual(serialized_url, url_serializer.data)
class TestEncodeSerializer(TestCase):
"""
Tests for `EncodeSerializer`.
"""
def test_serialized_encode(self):
"""
Tests that serializing/de-serializing 'Encode' works as expected.
"""
destination = Destination.objects.create(
destination_name='test_destination',
destination_nick='des',
destination_active=True
)
encode = Encode.objects.create(
encode_destination=destination,
encode_name='desktop_mp4',
profile_active=True,
)
self.assertEqual(Encode.objects.count(), 1)
actual_serialized_encode = EncodeSerializer(encode).data
for attr, actual_value in actual_serialized_encode.iteritems():
expected_value = getattr(encode, attr)
self.assertEqual(actual_value, expected_value)
VEDA_OS01/tests/test_utils.py 0 → 100644
"""
Tests common utils
"""
from ddt import data, ddt, unpack
from mock import MagicMock, Mock
from unittest import TestCase
from VEDA_OS01 import utils
@ddt
class UtilTests(TestCase):
"""
Common util tests.
"""
@data(
{
'urls': ('http://api.cielo24/', '/add/job'),
'params': {},
'expected_url': 'http://api.cielo24/add/job'
},
{
'urls': ('http://api.cielo24', '/add/job'),
'params': {'a': 1, 'b': 2},
'expected_url': 'http://api.cielo24/add/job?a=1&b=2'
},
{
'urls': ('http://api.cielo24/', 'add/job'),
'params': {'c': 3, 'd': 4},
'expected_url': 'http://api.cielo24/add/job?c=3&d=4'
},
{
'urls': ('http://api.cielo24','add/job'),
'params': {'p': 100},
'expected_url': 'http://api.cielo24/add/job?p=100'
},
{
'urls': ('http://api.cielo24', 'add/job', 'media'),
'params': {'p': 100},
'expected_url': 'http://api.cielo24/add/job/media?p=100'
}
)
@unpack
def test_build_url(self, urls, params, expected_url):
"""
Tests that utils.build_url works as expected.
"""
url = utils.build_url(
*urls,
**params
)
self.assertEqual(
url,
expected_url
)
@data(
{
'course_id': 'course-v1:MITx+4.605x+3T2017',
'expected_org': 'MITx'
},
{
'course_id': 'WestonHS/PFLC1x/3T2015',
'expected_org': 'WestonHS'
},
{
'course_id': '',
'expected_org': None
},
)
@unpack
def test_extract_course_org(self, course_id, expected_org):
"""
Tests that utils.extract_course_org works as expected.
"""
org = utils.extract_course_org(course_id)
self.assertEqual(
org,
expected_org
)
def test_get_config(self):
"""
Tests that utils.get_config works as expected.
"""
config = utils.get_config()
self.assertNotEqual(config, {})
@data(
('IN PROGRESS', True),
('FAILED', False)
)
@unpack
def test_video_status_update(self, status, update_val_status):
"""
Tests that utils.video_status_update works as expected.
"""
val_api_client = MagicMock()
video = Mock(studio_id='1234', transcript_status='earlier status')
# Make call to update_video_status.
utils.update_video_status(val_api_client=val_api_client, video=video, status=status)
# Assert the status and call to edx-val api method.
self.assertEqual(val_api_client.update_video_status.called, update_val_status)
self.assertEqual(video.transcript_status, status)
VEDA_OS01/utils.py 0 → 100644
"""
Common utils.
"""
import os
import urllib
import yaml
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from VEDA_OS01.models import TranscriptStatus
class ValTranscriptStatus(object):
"""
VAL supported video transcript statuses.
"""
TRANSCRIPTION_IN_PROGRESS = 'transcription_in_progress'
TRANSCRIPT_READY = 'transcript_ready'
# Maps the edx-video-pipeline video transcript statuses to edx-val statuses.
VAL_TRANSCRIPT_STATUS_MAP = {
TranscriptStatus.IN_PROGRESS: ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS,
TranscriptStatus.READY: ValTranscriptStatus.TRANSCRIPT_READY
}
def get_config(yaml_config_file='instance_config.yaml'):
"""
Read yaml config file.
Arguments:
yaml_config_file (str): yaml config file name
Returns:
dict: yaml conifg
"""
config_dict = {}
yaml_config_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
yaml_config_file
)
with open(yaml_config_file, 'r') as config:
try:
config_dict = yaml.load(config)
except yaml.YAMLError:
pass
return config_dict
def extract_course_org(course_id):
"""
Extract video organization from course url.
"""
org = None
try:
org = CourseKey.from_string(course_id).org
except InvalidKeyError:
pass
return org
def build_url(*urls, **query_params):
"""
Build a url from specified params.
Arguments:
base_url (str): base url
relative_url (str): endpoint
query_params (dict): query params
Returns:
absolute url
"""
url = '/'.join(item.strip('/') for item in urls)
if query_params:
url = '{}?{}'.format(url, urllib.urlencode(query_params))
return url
def update_video_status(val_api_client, video, status):
"""
Updates video status both in edx-val and edx-video-pipeline.
Arguments:
video(Video): Video data model object
status(Str): Video status to be updated
"""
# update edx-val's video status
try:
val_status = VAL_TRANSCRIPT_STATUS_MAP[status]
val_api_client.update_video_status(video.studio_id, val_status)
except KeyError:
# Don't update edx-val's video status.
pass
# update edx-video-pipeline's video status
video.transcript_status = status
video.save()
bin/heal
......@@ -12,7 +12,8 @@ if project_path not in sys.path:
from control.celeryapp import maintainer_healer
from control.veda_heal import VedaHeal
from VEDA_OS01.models import Video
from VEDA_OS01.models import Course, Video
from VEDA_OS01.transcripts import retrieve_three_play_translations
"""
Deliver
......@@ -79,6 +80,11 @@ def main():
VH = VedaHeal()
VH.discovery()
VH.purge()
# Kicks off a round of retrieving successful
# translations from 3Play Media
retrieve_three_play_translations()
HC = HealCli()
HC.schedule()
return None
......
bin/loop.py
......@@ -15,6 +15,10 @@ if project_path not in sys.path:
This is a cheapo way to get a pager (using SES)
"""
import django
django.setup()
from control.veda_file_discovery import FileDiscovery
from youtube_callback.daemon import generate_course_list
from youtube_callback.sftp_id_retrieve import callfunction
......
control/celeryapp.py
"""
Start Celery Worker
"""
from __future__ import absolute_import
import os
import sys
from celery import Celery
import yaml
"""
Start Celery Worker
"""
try:
from control.control_env import *
except:
from control_env import *
try:
from control.veda_deliver import VedaDelivery
except:
except ImportError:
from veda_deliver import VedaDelivery
auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
......@@ -59,12 +54,14 @@ def worker_task_fire(veda_id, encode_profile, jobid):
@app.task(name='supervisor_deliver')
def deliverable_route(veda_id, encode_profile):
VD = VedaDelivery(
"""
Task for deliverable route.
"""
veda_deliver = VedaDelivery(
veda_id=veda_id,
encode_profile=encode_profile
)
VD.run()
veda_deliver.run()
@app.task
......
control/tests/test_deliver_cielo.py 0 → 100644
"""
Cielo24 transcription testing
"""
from django.test import TestCase
import responses
from ddt import ddt
from mock import patch
from control.veda_deliver_cielo import Cielo24Transcript
from VEDA_OS01.models import (Cielo24Fidelity, Cielo24Turnaround, Course,
TranscriptProcessMetadata, TranscriptStatus,
Video)
from VEDA_OS01.utils import build_url
VIDEO_DATA = {
'studio_id': '12345',
'source_language': 'en'
}
@ddt
class Cielo24TranscriptTests(TestCase):
"""
Cielo24 transcription tests
"""
def setUp(self):
"""
Tests setup
"""
self.course = Course.objects.create(
course_name='Intro to VEDA',
institution='MAx',
edx_classid='123'
)
self.video = Video.objects.create(
inst_class=self.course,
**VIDEO_DATA
)
self.video_transcript_preferences = {
'org': 'MAx',
'api_key': 'cielo24_api_key',
'turnaround': Cielo24Turnaround.PRIORITY,
'fidelity': Cielo24Fidelity.PROFESSIONAL,
'preferred_languages': ['en', 'ur'],
's3_video_url': 'https://s3.amazonaws.com/bkt/video.mp4',
'callback_base_url': 'https://veda.edx.org/cielo24/transcript_completed/1234567890',
'cielo24_api_base_url': 'https://sandbox.cielo24.com/api',
}
def tearDown(self):
"""
Test cleanup
"""
TranscriptProcessMetadata.objects.all().delete()
def cielo24_url(self, cielo24, endpoint):
"""
Return absolute url
Arguments:
cielo24 (Cielo24Transcript), object
endpoint (srt): url endpoint
Returns:
absolute url
"""
return build_url(cielo24.cielo24_api_base_url, endpoint)
def assert_request(self, received_request, expected_request):
"""
Verify that `received_request` matches `expected_request`
"""
self.assertEqual(received_request.method, expected_request['method'])
self.assertEqual(received_request.url, expected_request['url'])
self.assertEqual(received_request.body, expected_request['body'])
@responses.activate
def test_transcript_flow(self):
"""
Verify cielo24 transcription flow
"""
job_id = '000-111-222'
cielo24 = Cielo24Transcript(
video=self.video,
**self.video_transcript_preferences
)
responses.add(
responses.GET,
self.cielo24_url(cielo24, cielo24.cielo24_new_job),
body={'JobId': job_id},
status=200
)
responses.add(
responses.GET,
self.cielo24_url(cielo24, cielo24.cielo24_add_media),
body={'TaskId': '000-000-111'},
status=200
)
responses.add(
responses.GET,
self.cielo24_url(cielo24, cielo24.cielo24_perform_transcription),
body={'TaskId': '000-000-000'},
status=200
)
cielo24.start_transcription_flow()
# Total of 6 HTTP requests are made
# 3 cielo24 requests for first language(en)
# 3 cielo24 requests for second language(ur)
self.assertEqual(len(responses.calls), 6)
# pylint: disable=line-too-long
expected_data = [
{
'url': build_url(
'https://sandbox.cielo24.com/api/job/new',
v=1,
job_name='12345',
language='en', # A job's language.
api_token='cielo24_api_key',
),
'body': None,
'method': 'GET'
},
{
'url': build_url(
'https://sandbox.cielo24.com/api/job/add_media',
v=1,
job_id='000-111-222',
api_token='cielo24_api_key',
media_url='https://s3.amazonaws.com/bkt/video.mp4',
),
'body': None,
'method': 'GET',
},
{
'url': build_url(
'https://sandbox.cielo24.com/api/job/perform_transcription',
v=1,
job_id='000-111-222',
target_language='TARGET_LANG',
callback_url=build_url(
'https://veda.edx.org/cielo24/transcript_completed/1234567890',
lang_code='TARGET_LANG',
video_id='12345',
job_id='000-111-222',
iwp_name='{iwp_name}',
org='MAx',
),
api_token='cielo24_api_key',
priority='PRIORITY',
transcription_fidelity='PROFESSIONAL',
options='{"return_iwp": ["FINAL"]}'
),
'body': None,
'method': 'GET'
}
]
received_request_index = 0
for preferred_language in self.video_transcript_preferences['preferred_languages']:
for request_data in expected_data:
# replace target language with appropriate value
if 'api/job/perform_transcription' in request_data['url']:
request_data = dict(request_data)
request_data['url'] = request_data['url'].replace('TARGET_LANG', preferred_language)
self.assert_request(
responses.calls[received_request_index].request,
request_data
)
received_request_index += 1
@patch('control.veda_deliver_cielo.LOGGER')
@responses.activate
def test_transcript_flow_exceptions(self, mock_logger):
"""
Verify that cielo24 transcription flow works as expected in case of bad response from cielo24
"""
job_id = '010-010-010'
bad_request_message = 'Bad request data'
preferences = dict(self.video_transcript_preferences)
preferences['preferred_languages'] = ['en']
cielo24 = Cielo24Transcript(
video=self.video,
**preferences
)
responses.add(
responses.GET,
self.cielo24_url(cielo24, cielo24.cielo24_new_job),
body={'JobId': job_id},
status=200
)
responses.add(
responses.GET,
self.cielo24_url(cielo24, cielo24.cielo24_add_media),
body=bad_request_message,
status=400
)
cielo24.start_transcription_flow()
mock_logger.exception.assert_called_with(
'[CIELO24] Request failed for video=%s -- lang=%s -- job_id=%s',
self.video.studio_id,
preferences['preferred_languages'][0],
job_id
)
# Total of 2 HTTP requests are made for2 cielo24
self.assertEqual(len(responses.calls), 2)
process_metadata = TranscriptProcessMetadata.objects.all()
self.assertEqual(process_metadata.count(), 1)
self.assertEqual(process_metadata.first().status, TranscriptStatus.FAILED)
control/tests/test_delivery_cielo.py deleted 100644 → 0
from ..veda_deliver_cielo import Cielo24Transcript
'''
TEST
list_of_ids = [
'XXXC93BC2016-V000100'
]
for l in list_of_ids:
x = Cielo24Transcript(
veda_id = l
)
output = x.perform_transcription()
print output
'''
control/tests/test_encode.py
......@@ -2,6 +2,10 @@
import os
import sys
import unittest
from django.test import TestCase
from control.veda_encode import VedaEncode
from VEDA_OS01.models import URL, Course, Destination, Encode, Video
"""
Test encode profiler
......@@ -12,18 +16,39 @@ sys.path.append(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
)))
from control.veda_encode import VedaEncode
from VEDA_OS01.models import Course, URL, Video, Encode
VIDEO_DATA = {
'studio_id': '12345'
}
class TestEncode(unittest.TestCase):
class TestEncode(TestCase):
def setUp(self):
self.course_object = Course.objects.get(
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.course_object = Course.objects.create(
institution='XXX',
edx_classid='XXXXX'
)
self.veda_id = 'XXXXXXXX2016-V00TEST'
self.video = Video.objects.create(
inst_class=self.course_object,
studio_id='12345',
edx_id=self.veda_id,
)
Encode.objects.create(
product_spec='mobile_low',
encode_destination=Destination.objects.create(destination_name='destination_name'),
profile_active=True
)
Encode.objects.create(
product_spec='desktop_mp4',
encode_destination=Destination.objects.create(destination_name='destination_name'),
profile_active=True
)
self.E = VedaEncode(
course_object=self.course_object,
veda_id=self.veda_id
......@@ -32,27 +57,25 @@ class TestEncode(unittest.TestCase):
def test_encode_url(self):
"""
gen baseline, gen a url, test against baseline
"""
URL.objects.filter(
videoID=Video.objects.filter(edx_id=self.veda_id).latest()
).delete()
encode_list = self.E.determine_encodes()
baseline = len(encode_list)
self.assertTrue(isinstance(encode_list, list))
self.assertTrue(isinstance(encode_list, set))
self.E.encode_list = []
U = URL(
self.E.encode_list = set()
url = URL(
videoID=Video.objects.filter(edx_id=self.veda_id).latest(),
encode_profile=Encode.objects.get(product_spec='mobile_low'),
encode_url='THIS Is A TEST'
)
U.save()
url.save()
encode_list = self.E.determine_encodes()
self.assertTrue(len(encode_list) == baseline - 1)
self.E.encode_list = []
self.E.encode_list = set()
URL.objects.filter(
videoID=Video.objects.filter(edx_id=self.veda_id).latest(),
).delete()
......
control/tests/test_file_discovery.py
import os
import sys
import unittest
from django.test import TestCase
"""
Test VEDA API
......@@ -12,7 +13,7 @@ sys.path.append(os.path.dirname(os.path.dirname(
from control.veda_file_discovery import FileDiscovery
class TestValidation(unittest.TestCase):
class TestValidation(TestCase):
def setUp(self):
......
control/tests/test_file_ingest.py
......@@ -4,6 +4,7 @@ import os
import sys
import unittest
from django.test import TestCase
import requests
import yaml
......@@ -19,7 +20,7 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__)))
requests.packages.urllib3.disable_warnings()
class TestIngest(unittest.TestCase):
class TestIngest(TestCase):
def setUp(self):
self.VP = VideoProto(
......
control/tests/test_heal.py
"""
Tests HEAL process
Test heal processor
"""
import datetime
import json
import os
import sys
from django.test import TestCase
from datetime import timedelta
from unittest import TestCase, skip
import yaml
from ddt import data, ddt, unpack
import responses
from django.utils.timezone import utc
from mock import PropertyMock, patch
from control.veda_heal import VedaHeal
from VEDA_OS01.models import Course, Video
from VEDA_OS01.models import URL, Course, Destination, Encode, Video, TranscriptStatus
from VEDA_OS01.utils import build_url, get_config, ValTranscriptStatus
sys.path.append(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
)))
CONFIG_DATA = get_config('test_config.yaml')
@ddt
......@@ -22,15 +31,78 @@ class HealTests(TestCase):
def setUp(self):
self.heal_instance = VedaHeal()
self.auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'instance_config.yaml'
)
self.encode_list = set()
with open(self.auth_yaml, 'r') as stream:
for key, entry in yaml.load(stream)['encode_dict'].items():
for e in entry:
self.encode_list.add(e)
for key, entry in CONFIG_DATA['encode_dict'].items():
for e in entry:
self.encode_list.add(e)
self.video_id = '12345'
self.course_object = Course.objects.create(
institution='XXX',
edx_classid='XXXXX',
local_storedir='WestonHS/PFLC1x/3T2015'
)
self.video = Video.objects.create(
inst_class=self.course_object,
studio_id=self.video_id,
edx_id='XXXXXXXX2014-V00TES1',
video_trans_start=datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(
hours=CONFIG_DATA['heal_start']
),
video_trans_end=datetime.datetime.utcnow().replace(tzinfo=utc),
)
self.encode = Encode.objects.create(
product_spec='mobile_low',
encode_destination=Destination.objects.create(destination_name='destination_name')
)
self.hls_encode = Encode.objects.create(
product_spec='hls',
encode_destination=Destination.objects.create(destination_name='destination_name')
)
url = URL(
videoID=self.video,
encode_profile=self.encode,
encode_bitdepth='22',
encode_url='http://veda.edx.org/encode')
url.save()
@patch('control.veda_heal.VALAPICall._AUTH', PropertyMock(return_value=lambda: CONFIG_DATA))
@responses.activate
def test_heal(self):
val_response = {
'courses': [{u'WestonHS/PFLC1x/3T2015': None}],
'encoded_videos': [{
'url': 'https://testurl.mp4',
'file_size': 8499040,
'bitrate': 131,
'profile': 'mobile_low',
}]
}
responses.add(
responses.POST,
CONFIG_DATA['val_token_url'],
'{"access_token": "1234567890"}',
status=200
)
responses.add(
responses.GET,
build_url(CONFIG_DATA['val_api_url'], self.video_id),
body=json.dumps(val_response),
content_type='application/json',
status=200
)
responses.add(
responses.PUT,
build_url(CONFIG_DATA['val_api_url'], self.video_id),
status=200
)
heal = VedaHeal()
heal.discovery()
@data(
{
......@@ -71,7 +143,6 @@ class HealTests(TestCase):
},
)
@unpack
@skip("Failing from day 1 https://github.com/edx/edx-video-pipeline/pull/26")
def test_determine_fault(self, edx_id, video_trans_status, video_trans_start, video_active):
"""
Tests that determine_fault works in various video states.
......@@ -81,8 +152,10 @@ class HealTests(TestCase):
video_trans_status=video_trans_status,
video_trans_start=video_trans_start,
video_active=video_active,
inst_class=Course()
inst_class=self.course_object
)
video_instance.save()
encode_list = self.heal_instance.determine_fault(video_instance)
if video_instance.edx_id == '1':
......@@ -95,48 +168,103 @@ class HealTests(TestCase):
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_object': {
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
}
},
'result': []
},
{
'uncompleted_encodes': ['test_obj'],
'expected_encodes': ['test_obj'],
'video_object': {
'video_props': {
'edx_id': '2',
'video_trans_status': 'Ingest',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
}
},
'result': ['test_obj']
}
)
@unpack
def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
def test_differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_props, result):
"""
Tests that differentiate_encodes list comparison works as expected. This doesn't test video states,
just the list comparison function.
"""
video_instance = Video(
edx_id=video_object['edx_id'],
video_trans_status=video_object['video_trans_status'],
video_trans_start=video_object['video_trans_start'],
video_active=video_object['video_active'],
inst_class=Course()
)
video_instance = Video.objects.create(inst_class=self.course_object, **video_props)
encode_list = self.heal_instance.differentiate_encodes(
uncompleted_encodes,
expected_encodes,
video_instance
)
self.assertEqual(encode_list, result)
if video_instance.edx_id == '1':
self.assertEqual(encode_list, [])
elif video_instance.edx_id == '2':
self.assertEqual(encode_list, ['test_obj'])
@data(
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.PENDING
},
'expected_val_status': 'file_complete'
},
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.IN_PROGRESS
},
'expected_val_status': ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS
},
{
'uncompleted_encodes': [],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '1',
'video_trans_status': 'Complete',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.READY
},
'expected_val_status': ValTranscriptStatus.TRANSCRIPT_READY
},
{
'uncompleted_encodes': ['test_obj'],
'expected_encodes': ['test_obj'],
'video_props': {
'edx_id': '2',
'video_trans_status': 'Ingest',
'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
'video_active': True,
'transcript_status': TranscriptStatus.READY
},
'expected_val_status': 'transcode_queue'
}
)
@unpack
def test_differentiate_encodes_val_status(self, uncompleted_encodes,
expected_encodes, video_props, expected_val_status):
"""
Tests that the val status changes as expected based on encode list.
"""
video_instance = Video.objects.create(inst_class=self.course_object, **video_props)
self.heal_instance.differentiate_encodes(
uncompleted_encodes,
expected_encodes,
video_instance
)
self.assertEqual(self.heal_instance.val_status, expected_val_status)
@data(
{
......@@ -171,16 +299,17 @@ class HealTests(TestCase):
}
)
@unpack
@skip("Failing from day 1 https://github.com/edx/edx-video-pipeline/pull/26")
def test_determine_longterm_corrupt(self, uncompleted_encodes, expected_encodes, video_object):
video_instance = Video(
edx_id=video_object['edx_id'],
video_trans_status=video_object['video_trans_status'],
video_trans_start=video_object['video_trans_start'],
video_active=video_object['video_active'],
inst_class=Course()
inst_class=self.course_object
)
video_instance.save()
longterm_corrupt = self.heal_instance.determine_longterm_corrupt(
uncompleted_encodes,
expected_encodes,
......
control/tests/test_hotstore.py
"""
Test upload processes
"""
import os
import sys
import unittest
from django.test import TestCase
"""
Test upload processes
from boto.s3.connection import S3Connection
from mock import PropertyMock, patch
from moto import mock_s3_deprecated
from VEDA_OS01 import utils
from control.veda_file_ingest import VideoProto
from control.veda_hotstore import Hotstore
"""
sys.path.append(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
)))
from control.veda_hotstore import Hotstore
from control.veda_file_ingest import VideoProto
CONFIG_DATA = utils.get_config('test_config.yaml')
class TestHotstore(unittest.TestCase):
class TestHotstore(TestCase):
def setUp(self):
VP = VideoProto()
VP.veda_id = 'XXXXXXXX2014-V00TEST'
video_proto = VideoProto()
video_proto.veda_id = 'XXXXXXXX2014-V00TEST'
self.upload_filepath = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'test_files',
'OVTESTFILE_01.mp4'
)
self.H1 = Hotstore(
video_object=VP,
upload_filepath=self.upload_filepath,
video_proto=VP
)
with patch.object(Hotstore, '_READ_AUTH', PropertyMock(return_value=lambda: CONFIG_DATA)):
self.hotstore = Hotstore(
video_object=video_proto,
upload_filepath=self.upload_filepath,
video_proto=video_proto
)
# do s3 mocking
mock = mock_s3_deprecated()
mock.start()
conn = S3Connection()
conn.create_bucket(CONFIG_DATA['veda_s3_hotstore_bucket'])
self.addCleanup(mock.stop)
def test_single_upload(self):
if self.H1.auth_dict is None:
self.assertTrue(self.H1.upload() is False)
"""
Verify S3 single part upload.
"""
if self.hotstore.auth_dict is None:
self.assertTrue(self.hotstore.upload() is False)
return None
self.assertTrue(self.H1.upload())
self.hotstore.auth_dict['multi_upload_barrier'] = os.stat(self.upload_filepath).st_size + 1
self.assertTrue(self.hotstore.upload())
def test_multi_upload(self):
if self.H1.auth_dict is None:
self.assertTrue(self.H1.upload() is None)
"""
Verify S3 single multi-part upload.
"""
if self.hotstore.auth_dict is None:
self.assertTrue(self.hotstore.upload() is None)
return None
self.H1.auth_dict['multi_upload_barrier'] = 0
self.assertTrue(self.H1.upload())
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
self.hotstore.auth_dict['multi_upload_barrier'] = 0
self.assertTrue(self.hotstore.upload())
control/tests/test_reporting.py
......@@ -2,6 +2,7 @@
import os
import sys
import unittest
from django.test import TestCase
"""
A basic unittest for the "Course Addition Tool"
......@@ -14,7 +15,7 @@ sys.path.append(
from veda_utils import Report
class TestReporting(unittest.TestCase):
class TestReporting(TestCase):
def setUp(self):
self.R = Report(
......
control/tests/test_val.py
......@@ -2,25 +2,30 @@
import ast
import os
import sys
import unittest
from django.test import TestCase
from mock import PropertyMock, patch
import requests
import yaml
import responses
from control.veda_val import VALAPICall
from veda_file_ingest import VideoProto
from veda_val import VALAPICall
from VEDA_OS01 import utils
requests.packages.urllib3.disable_warnings()
"""
This is an API connection test
set to pass if instance_config.yaml is missing
"""
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
CONFIG_DATA = utils.get_config('test_config.yaml')
class TestVALAPI(unittest.TestCase):
class TestVALAPI(TestCase):
def setUp(self):
self.VP = VideoProto(
......@@ -28,20 +33,17 @@ class TestVALAPI(unittest.TestCase):
veda_id='TESTID'
)
self.VAC = VALAPICall(
video_proto=self.VP,
val_status='complete'
)
with patch.object(VALAPICall, '_AUTH', PropertyMock(return_value=lambda: CONFIG_DATA)):
self.VAC = VALAPICall(
video_proto=self.VP,
val_status='complete'
)
self.auth_yaml = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'instance_config.yaml'
)
self.auth_yaml = CONFIG_DATA
def test_val_setup(self):
if not os.path.exists(self.auth_yaml):
self.assertTrue(self.VAC.auth_dict is None)
return None
# register val url to send api response
responses.add(responses.POST, CONFIG_DATA['val_token_url'], '{"access_token": "1234567890"}', status=200)
salient_variables = [
'val_api_url',
......@@ -51,30 +53,23 @@ class TestVALAPI(unittest.TestCase):
'val_username',
'val_token_url',
]
for s in salient_variables:
self.assertTrue(len(self.VAC.auth_dict[s]) > 0)
for salient_variable in salient_variables:
self.assertTrue(len(self.VAC.auth_dict[salient_variable]) > 0)
@responses.activate
def test_val_connection(self):
if not os.path.exists(self.auth_yaml):
self.assertTrue(self.VAC.auth_dict is None)
return None
# register val url to send api response
responses.add(responses.POST, CONFIG_DATA['val_token_url'], '{"access_token": "1234567890"}', status=200)
responses.add(responses.GET, CONFIG_DATA['val_api_url'], status=200)
self.VAC.val_tokengen()
self.assertFalse(self.VAC.val_token is None)
s = requests.get(
response = requests.get(
self.VAC.auth_dict['val_api_url'],
headers=self.VAC.headers,
timeout=20
)
self.assertFalse(s.status_code == 404)
self.assertFalse(s.status_code > 299)
def main():
unittest.main()
if __name__ == '__main__':
sys.exit(main())
self.assertFalse(response.status_code == 404)
self.assertFalse(response.status_code > 299)
control/tests/test_video_validation.py
import os
import sys
import unittest
from django.test import TestCase
"""
Test VEDA API
......@@ -12,7 +13,7 @@ sys.path.append(os.path.dirname(os.path.dirname(
from control.veda_video_validation import Validation
class TestValidation(unittest.TestCase):
class TestValidation(TestCase):
"""
Test class for Validation
"""
......@@ -28,6 +29,7 @@ class TestValidation(unittest.TestCase):
videofile=self.videofile
)
@unittest.skip('Skipping this test due to unavailability of required ffprobe version.')
def test_validation(self):
"""
Check a known file for validity
......
control/veda_deliver.py
import os
import sys
import yaml
import datetime
import logging
import shutil
from os.path import expanduser
import boto
import boto.s3
from boto.s3.key import Key
from boto.exception import S3ResponseError
from os.path import expanduser
import requests
import datetime
import ftplib
import shutil
import yaml
from boto.exception import S3ResponseError
from boto.s3.key import Key
from django.core.urlresolvers import reverse
import veda_deliver_xuetang
from control_env import *
from veda_deliver_cielo import Cielo24Transcript
from veda_deliver_youtube import DeliverYoutube
from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
TranscriptStatus)
from VEDA_OS01.utils import build_url
from veda_utils import ErrorObject, Metadata, Output, VideoProto
from veda_val import VALAPICall
from veda_video_validation import Validation
from watchdog import Watchdog
try:
from control.veda_deliver_3play import ThreePlayMediaClient
except ImportError:
from veda_deliver_3play import ThreePlayMediaClient
LOGGER = logging.getLogger(__name__)
try:
......@@ -28,14 +50,6 @@ and upload to the appropriate endpoint via the approp. methods
"""
homedir = expanduser("~")
from control_env import *
from veda_utils import ErrorObject, Output, Metadata, VideoProto
from veda_video_validation import Validation
from veda_val import VALAPICall
from veda_deliver_cielo import Cielo24Transcript
import veda_deliver_xuetang
from veda_deliver_youtube import DeliverYoutube
from watchdog import Watchdog
watchdog_time = 10.0
......@@ -170,8 +184,7 @@ class VedaDelivery:
"""
Transcript, Xuetang
"""
self._THREEPLAY_UPLOAD()
self._CIELO24_UPLOAD()
self._XUETANG_ROUTE()
self.status = self._DETERMINE_STATUS()
......@@ -180,6 +193,18 @@ class VedaDelivery:
self._CLEANUP()
# Transcription Process
# We only want to generate transcripts for `desktop_mp4` profile.
if self.encode_profile == 'desktop_mp4' and self.video_query.process_transcription:
# 3PlayMedia
if self.video_query.provider == TranscriptProvider.THREE_PLAY:
self.start_3play_transcription_process()
# Cielo24
if self.video_query.provider == TranscriptProvider.CIELO24:
self.cielo24_transcription_flow()
def _INFORM_INTAKE(self):
"""
Collect all salient metadata and
......@@ -507,63 +532,106 @@ class VedaDelivery:
os.chdir(homedir)
return True
def _CIELO24_UPLOAD(self):
if self.video_query.inst_class.c24_proc is False:
return None
def cielo24_transcription_flow(self):
"""
Cielo24 transcription flow.
"""
org = utils.extract_course_org(self.video_proto.platform_course_url[0])
if self.video_query.inst_class.mobile_override is False:
if self.encode_profile != 'desktop_mp4':
return None
try:
api_key = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider).api_key
except TranscriptCredentials.DoesNotExist:
LOGGER.warn('[cielo24] Unable to find api_key for org=%s', org)
return None
C24 = Cielo24Transcript(
veda_id=self.video_query.edx_id
)
output = C24.perform_transcription()
print '[ %s ] : %s' % (
'Cielo24 JOB', self.video_query.edx_id
s3_video_url = build_url(
self.auth_dict['s3_base_url'],
self.auth_dict['edx_s3_endpoint_bucket'],
self.encoded_file
)
def _THREEPLAY_UPLOAD(self):
callback_base_url = build_url(
self.auth_dict['veda_base_url'],
reverse(
'cielo24_transcript_completed',
args=[self.auth_dict['transcript_provider_request_token']]
)
)
if self.video_query.inst_class.tp_proc is False:
return None
if self.video_query.inst_class.mobile_override is False:
if self.encode_profile != 'desktop_mp4':
return None
# update transcript status for video.
val_api_client = VALAPICall(video_proto=None, val_status=None)
utils.update_video_status(
val_api_client=val_api_client,
video=self.video_query,
status=TranscriptStatus.IN_PROGRESS
)
ftp1 = ftplib.FTP(
self.auth_dict['threeplay_ftphost']
cielo24 = Cielo24Transcript(
self.video_query,
org,
api_key,
self.video_query.cielo24_turnaround,
self.video_query.cielo24_fidelity,
self.video_query.preferred_languages,
s3_video_url,
callback_base_url,
self.auth_dict['cielo24_api_base_url'],
)
user = self.video_query.inst_class.tp_username.strip()
passwd = self.video_query.inst_class.tp_password.strip()
cielo24.start_transcription_flow()
def start_3play_transcription_process(self):
"""
3PlayMedia Transcription Flow
"""
try:
ftp1.login(user, passwd)
except:
ErrorObject.print_error(
message='3Play Authentication Failure'
# Picks the first course from the list as there may be multiple
# course runs in that list (i.e. all having the same org).
org = utils.extract_course_org(self.video_proto.platform_course_url[0])
transcript_secrets = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider)
# update transcript status for video.
val_api_client = VALAPICall(video_proto=None, val_status=None)
utils.update_video_status(
val_api_client=val_api_client,
video=self.video_query,
status=TranscriptStatus.IN_PROGRESS
)
try:
ftp1.cwd(
self.video_query.inst_class.tp_speed
# Initialize 3playMedia client and start transcription process
s3_video_url = build_url(
self.auth_dict['s3_base_url'],
self.auth_dict['edx_s3_endpoint_bucket'],
self.encoded_file
)
except:
ftp1.mkd(
self.video_query.inst_class.tp_speed
callback_url = build_url(
self.auth_dict['veda_base_url'],
reverse(
'3play_media_callback',
args=[self.auth_dict['transcript_provider_request_token']]
),
# Additional attributes that'll come back with the callback
org=org,
edx_video_id=self.video_query.studio_id,
lang_code=self.video_query.source_language,
)
ftp1.cwd(
self.video_query.inst_class.tp_speed
three_play_media = ThreePlayMediaClient(
org=org,
video=self.video_query,
media_url=s3_video_url,
api_key=transcript_secrets.api_key,
api_secret=transcript_secrets.api_secret,
callback_url=callback_url,
turnaround_level=self.video_query.three_play_turnaround,
three_play_api_base_url=self.auth_dict['three_play_api_base_url'],
)
os.chdir(self.node_work_directory)
ftp1.storbinary(
'STOR ' + self.encoded_file,
open(os.path.join(
self.node_work_directory,
self.encoded_file
), 'rb')
)
three_play_media.generate_transcripts()
os.chdir(homedir)
except TranscriptCredentials.DoesNotExist:
LOGGER.warning(
'Transcript preference is not found for provider=%s, video=%s',
self.video_query.provider,
self.video_query.studio_id,
)
def _XUETANG_ROUTE(self):
if self.video_query.inst_class.xuetang_proc is False:
......
control/veda_deliver_3play.py 0 → 100644
"""
3PlayMedia Transcription Client
"""
import json
import logging
import requests
import sys
from requests.packages.urllib3.exceptions import InsecurePlatformWarning
from VEDA_OS01.models import TranscriptProcessMetadata, TranscriptProvider, TranscriptStatus
from VEDA_OS01.utils import build_url
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
LOGGER = logging.getLogger(__name__)
class ThreePlayMediaError(Exception):
"""
An error that occurs during 3PlayMedia actions.
"""
pass
class ThreePlayMediaLanguageNotFoundError(ThreePlayMediaError):
"""
An error when language is not found in available 3playMedia languages.
"""
pass
class ThreePlayMediaPerformTranscriptionError(ThreePlayMediaError):
"""
An error occurred while adding media for transcription.
"""
pass
class ThreePlayMediaUrlError(ThreePlayMediaError):
"""
Occurs when the media url is either inaccessible or of invalid content type.
"""
pass
class ThreePlayMediaLanguagesRetrievalError(ThreePlayMediaError):
"""
An error Occurred while retrieving available 3PlayMedia languages.
"""
pass
class ThreePlayMediaClient(object):
def __init__(
self,
org,
video,
media_url,
api_key,
api_secret,
callback_url,
turnaround_level,
three_play_api_base_url
):
"""
Initialize 3play media client
"""
self.org = org
self.video = video
self.media_url = media_url
self.api_key = api_key
self.api_secret = api_secret
self.callback_url = callback_url
self.turnaround_level = turnaround_level
# default attributes
self.base_url = three_play_api_base_url
self.upload_media_file_url = u'files/'
self.available_languages_url = u'caption_imports/available_languages/'
self.allowed_content_type = u'video/mp4'
def validate_media_url(self):
"""
Validates the media URL
Raises:
3PlayMediaUrlError: on invalid media url or content type
"""
if not self.media_url:
raise ThreePlayMediaUrlError('Invalid media URL "{media_url}".'.format(media_url=self.media_url))
response = requests.head(url=self.media_url)
if not response.ok:
raise ThreePlayMediaUrlError('The URL "{media_url}" is not Accessible.'.format(media_url=self.media_url))
elif response.headers['Content-Type'] != self.allowed_content_type:
raise ThreePlayMediaUrlError(
'Media content-type should be "{allowed_type}". URL was "{media_url}", content-type was "{type}"'.format(
allowed_type=self.allowed_content_type,
media_url=self.media_url,
type=response.headers['Content-Type'],
)
)
def get_available_languages(self):
"""
Gets all the 3Play Media supported languages
"""
response = requests.get(url=build_url(self.base_url, self.available_languages_url, apikey=self.api_key))
if not response.ok:
raise ThreePlayMediaLanguagesRetrievalError(
'Error while retrieving available languages: {response} -- {status}'.format(
response=response.text, status=response.status_code
)
)
# A normal response should be a list containing 3Play Media supported languages and if we're getting a dict,
# there must be an error: https://support.3playmedia.com/hc/en-us/articles/227729968-Captions-Imports-API
available_languages = json.loads(response.text)
if isinstance(available_languages, dict):
raise ThreePlayMediaLanguagesRetrievalError(
'Expected 3Play Media Supported languages but got: {response}'.format(response=response.text)
)
return available_languages
def get_source_language_id(self, languages, source_language_code):
"""
Extracts language id for a language that matches `source_language_code`
from the given 3Play Media languages.
Arguments:
languages(list): 3PlayMedia supported languages.
source_language_code(unicode): A video source language code whose 3Play language id is required.
"""
for language in languages:
if language['iso_639_1_code'] == source_language_code:
return language['language_id']
def submit_media(self):
"""
Submits the media to perform transcription.
Raises:
ThreePlayMediaPerformTranscriptionError: error while transcription process
"""
self.validate_media_url()
# Prepare requests payload
payload = dict(
# Mandatory attributes required for transcription
link=self.media_url,
apikey=self.api_key,
api_secret_key=self.api_secret,
turnaround_level=self.turnaround_level,
callback_url=self.callback_url,
)
available_languages = self.get_available_languages()
source_language_id = self.get_source_language_id(available_languages, self.video.source_language)
if source_language_id:
payload['language_id'] = source_language_id
upload_url = build_url(self.base_url, self.upload_media_file_url)
response = requests.post(url=upload_url, json=payload)
if not response.ok:
raise ThreePlayMediaPerformTranscriptionError(
'Upload file request failed with: {response} -- {status}'.format(
response=response.text, status=response.status_code
)
)
# A normal response should be a text containing file id and if we're getting a deserializable dict, there
# must be an error: http://support.3playmedia.com/hc/en-us/articles/227729828-Files-API-Methods
if isinstance(json.loads(response.text), dict):
raise ThreePlayMediaPerformTranscriptionError(
'Expected file id but got: {response}'.format(response=response.text)
)
return response.text
def generate_transcripts(self):
"""
Kicks off transcription process for default language.
"""
try:
file_id = self.submit_media()
# Track progress of transcription process
TranscriptProcessMetadata.objects.create(
video=self.video,
process_id=file_id,
lang_code=self.video.source_language,
provider=TranscriptProvider.THREE_PLAY,
status=TranscriptStatus.IN_PROGRESS,
)
# Successfully kicked off transcription process for a video with the given language.
LOGGER.info(
'[3PlayMedia] Transcription process has been started for video=%s, source_language=%s.',
self.video.studio_id,
self.video.source_language,
)
except ThreePlayMediaError:
LOGGER.exception(
'[3PlayMedia] Could not process transcripts for video=%s source_language=%s.',
self.video.studio_id,
self.video.source_language,
)
except Exception:
LOGGER.exception(
'[3PlayMedia] Unexpected error while transcription for video=%s source_language=%s.',
self.video.studio_id,
self.video.source_language,
)
raise
control/veda_file_discovery.py
import json
import logging
import os.path
import boto
import boto.s3
from boto.exception import S3ResponseError, S3DataError
import yaml
from VEDA_OS01.models import TranscriptCredentials
from VEDA_OS01.utils import extract_course_org
try:
boto.config.add_section('Boto')
except:
......@@ -26,6 +31,8 @@ from veda_utils import ErrorObject
from veda_file_ingest import VideoProto, VedaIngest
from veda_val import VALAPICall
LOGGER = logging.getLogger(__name__)
class FileDiscovery(object):
......@@ -159,6 +166,7 @@ class FileDiscovery(object):
client_title = meta.get_metadata('client_video_id')
course_hex = meta.get_metadata('course_video_upload_token')
course_url = meta.get_metadata('course_key')
transcript_preferences = meta.get_metadata('transcript_preferences')
edx_filename = key.name[::-1].split('/')[0][::-1]
if len(course_hex) == 0:
......@@ -226,24 +234,48 @@ class FileDiscovery(object):
key.delete()
return
"""
Trigger Ingest Process
"""
V = VideoProto(
# Make decision if this video needs the transcription as well.
try:
transcript_preferences = json.loads(transcript_preferences)
TranscriptCredentials.objects.get(
org=extract_course_org(course_url),
provider=transcript_preferences.get('provider')
)
process_transcription = True
except (TypeError, TranscriptCredentials.DoesNotExist):
# when the preferences are not set OR these are set to some data in invalid format OR these don't
# have associated 3rd party transcription provider API keys.
process_transcription = False
except ValueError:
LOGGER.error('[VIDEO-PIPELINE] File Discovery - Invalid transcripts preferences=%s', transcript_preferences)
process_transcription = False
# Trigger Ingest Process
video_metadata = dict(
s3_filename=edx_filename,
client_title=client_title,
file_extension=file_extension,
platform_course_url=course_url
platform_course_url=course_url,
)
I = VedaIngest(
if process_transcription:
video_metadata.update({
'process_transcription': process_transcription,
'provider': transcript_preferences.get('provider'),
'three_play_turnaround': transcript_preferences.get('three_play_turnaround'),
'cielo24_turnaround': transcript_preferences.get('cielo24_turnaround'),
'cielo24_fidelity': transcript_preferences.get('cielo24_fidelity'),
'preferred_languages': transcript_preferences.get('preferred_languages'),
'source_language': transcript_preferences.get('video_source_language'),
})
ingest = VedaIngest(
course_object=course_query[0],
video_proto=V,
video_proto=VideoProto(**video_metadata),
node_work_directory=self.node_work_directory
)
I.insert()
ingest.insert()
if I.complete is False:
if ingest.complete is False:
return
"""
......
control/veda_file_ingest.py
import logging
import os
import sys
import subprocess
......@@ -7,6 +7,7 @@ from datetime import timedelta
import time
import fnmatch
import django
from django.db.utils import DatabaseError
from django.utils.timezone import utc
from django.db import reset_queries
import uuid
......@@ -32,6 +33,10 @@ from veda_val import VALAPICall
from veda_encode import VedaEncode
import celeryapp
from VEDA_OS01.models import TranscriptStatus
LOGGER = logging.getLogger(__name__)
'''
V = VideoProto(
s3_filename=edx_filename,
......@@ -59,9 +64,17 @@ class VideoProto():
self.file_extension = kwargs.get('file_extension', None)
self.platform_course_url = kwargs.get('platform_course_url', None)
self.abvid_serial = kwargs.get('abvid_serial', None)
"""
Determined Attrib
"""
# Transcription Process related Attributes
self.process_transcription = kwargs.get('process_transcription', False)
self.provider = kwargs.get('provider', None)
self.three_play_turnaround = kwargs.get('three_play_turnaround', None)
self.cielo24_turnaround = kwargs.get('cielo24_turnaround', None)
self.cielo24_fidelity = kwargs.get('cielo24_fidelity', None)
self.preferred_languages = kwargs.get('preferred_languages', [])
self.source_language = kwargs.get('source_language', None)
# Determined Attributes
self.valid = False
self.filesize = 0
self.duration = 0
......@@ -325,6 +338,17 @@ class VedaIngest:
self.complete = True
return None
# Update transcription preferences for the Video
if self.video_proto.process_transcription:
v1.process_transcription = self.video_proto.process_transcription
v1.transcript_status = TranscriptStatus.PENDING
v1.provider = self.video_proto.provider
v1.three_play_turnaround = self.video_proto.three_play_turnaround
v1.cielo24_turnaround = self.video_proto.cielo24_turnaround
v1.cielo24_fidelity = self.video_proto.cielo24_fidelity
v1.preferred_languages = self.video_proto.preferred_languages
v1.source_language = self.video_proto.source_language
"""
Files Below are all valid
"""
......@@ -345,7 +369,8 @@ class VedaIngest:
"""
try:
v1.save()
except:
except DatabaseError:
# in case if the client title's length is too long
char_string = self.video_proto.client_title
string_len = len(char_string)
s1 = 0
......@@ -360,6 +385,11 @@ class VedaIngest:
v1.client_title = final_string
v1.save()
except Exception:
# Log the exception and raise.
LOGGER.exception('[VIDEO-PIPELINE] File Ingest - Cataloging of video=%s failed.', self.video_proto.veda_id)
raise
def val_insert(self):
if self.video_proto.abvid_serial is not None:
return None
......
control/veda_heal.py
......@@ -8,14 +8,19 @@ Roll through videos, check for completion
"""
import datetime
import uuid
from datetime import timedelta
import os
import sys
import uuid
import yaml
from django.utils.timezone import utc
from VEDA_OS01.models import Encode, URL, Video
from VEDA_OS01.utils import VAL_TRANSCRIPT_STATUS_MAP
import celeryapp
from control_env import *
from control_env import WORK_DIRECTORY
from veda_encode import VedaEncode
from veda_val import VALAPICall
......@@ -73,6 +78,11 @@ class VedaHeal(object):
encode_list = self.determine_fault(video_object=v)
# Using the 'Video Proto' Model
if self.val_status is not None:
# Update to VAL is also happening for those videos which are already marked complete,
# All these retries are for the data-parity between VAL and VEDA, as calls to VAL api are
# unreliable and times out. For a completed Video, VEDA heal will keep doing this unless
# the Video is old enough and escapes from the time-span that HEAL is picking up on.
# cc Greg Martin
VAC = VALAPICall(
video_proto=None,
video_object=v,
......@@ -82,14 +92,15 @@ class VedaHeal(object):
self.val_status = None
# Enqueue
for e in encode_list:
veda_id = v.edx_id
encode_profile = e
jobid = uuid.uuid1().hex[0:10]
celeryapp.worker_task_fire.apply_async(
(veda_id, encode_profile, jobid),
queue=self.auth_dict['celery_worker_queue']
)
if self.auth_dict['rabbitmq_broker'] is not None:
for e in encode_list:
veda_id = v.edx_id
encode_profile = e
jobid = uuid.uuid1().hex[0:10]
celeryapp.worker_task_fire.apply_async(
(veda_id, encode_profile, jobid),
queue=self.auth_dict['celery_worker_queue']
)
def determine_fault(self, video_object):
"""
......@@ -119,8 +130,9 @@ class VedaHeal(object):
course_object=video_object.inst_class,
).determine_encodes()
try:
uncompleted_encodes.remove('review')
except ValueError:
if uncompleted_encodes:
uncompleted_encodes.remove('review')
except KeyError:
pass
# list comparison
......@@ -137,8 +149,19 @@ class VedaHeal(object):
# These encodes don't count towards 'file_complete'
if e != 'mobile_high' and e != 'audio_mp3' and e != 'review':
check_list.append(e)
# See if VEDA's Video data model is already having transcript status which corresponds
# to any of Val's Video transcript statuses. If its True, set `val_status` to that status
# instead of `file_complete` as transcription phase comes after encoding phase of a Video,
# and `file_complete` shows that a Video's encodes are complete, while there may be possibility
# that the Video has gone through transcription phase as well after the encodes were ready.
val_transcription_status = VAL_TRANSCRIPT_STATUS_MAP.get(video_object.transcript_status, None)
if check_list is None or len(check_list) == 0:
self.val_status = 'file_complete'
if val_transcription_status:
self.val_status = val_transcription_status
else:
self.val_status = 'file_complete'
# File is complete!
# Check for data parity, and call done
if video_object.video_trans_status != 'Complete':
......@@ -155,7 +178,11 @@ class VedaHeal(object):
if self.determine_longterm_corrupt(uncompleted_encodes, expected_encodes, video_object):
return []
if self.val_status != 'file_complete':
complete_statuses = ['file_complete']
if val_transcription_status:
complete_statuses.append(val_transcription_status)
if self.val_status not in complete_statuses:
self.val_status = 'transcode_queue'
return uncompleted_encodes
......
control/veda_val.py
import logging
import os
import sys
import requests
......@@ -7,6 +8,8 @@ import json
import datetime
import yaml
LOGGER = logging.getLogger(__name__)
requests.packages.urllib3.disable_warnings()
......@@ -386,6 +389,64 @@ class VALAPICall():
)
)
def update_val_transcript(self, video_id, lang_code, name, transcript_format, provider):
"""
Update status for a completed transcript.
"""
if self.val_token is None:
self.val_tokengen()
post_data = {
'video_id': video_id,
'name': name,
'provider': provider,
'language_code': lang_code,
'file_format': transcript_format,
}
response = requests.post(
self.auth_dict['val_transcript_create_url'],
json=post_data,
headers=self.headers,
timeout=20
)
if not response.ok:
LOGGER.error(
'update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s',
video_id,
provider,
response.status_code,
response.content,
)
def update_video_status(self, video_id, status):
"""
Update video transcript status.
"""
if self.val_token is None:
self.val_tokengen()
val_data = {
'edx_video_id': video_id,
'status': status
}
response = requests.patch(
self.auth_dict['val_video_transcript_status_url'],
json=val_data,
headers=self.headers,
timeout=20
)
if not response.ok:
LOGGER.error(
'update_video_status failed -- video_id=%s -- status=%s -- text=%s',
video_id,
response.status_code,
response.text
)
def main():
pass
......
control/veda_video_validation.py
......@@ -5,6 +5,9 @@ import subprocess
import fnmatch
import django
from control.control_env import FFPROBE
from VEDA_OS01.models import Video
"""
VEDA Intake/Product Final Testing Suite
......@@ -15,10 +18,9 @@ image files (which read as 0:00 duration or N/A)
Mismatched Durations (within 5 sec)
"""
from control_env import *
class Validation():
class Validation(object):
"""
Expects a full filepath
"""
......@@ -43,7 +45,6 @@ class Validation():
FFPROBE,
"\"" + self.videofile + "\""
))
"""
Test if size is zero
"""
......@@ -61,6 +62,10 @@ class Validation():
if "multiple edit list entries, a/v desync might occur, patch welcome" in line:
return False
if "command not found" in line:
print line
return False
if "Duration: " in line:
if "Duration: 00:00:00.0" in line:
return False
......
frontend/tests/test_abvid_reporting.py
......@@ -2,6 +2,7 @@
import os
import sys
import unittest
from django.test import TestCase
"""
A basic unittest for the "Course Addition Tool"
......@@ -14,7 +15,7 @@ sys.path.append(
import abvid_reporting
from frontend.course_validate import VEDACat
class TestVariables(unittest.TestCase):
class TestVariables(TestCase):
def setUp(self):
self.VCT = VEDACat()
......
frontend/tests/test_cat_tool.py
......@@ -2,6 +2,7 @@
import os
import sys
import unittest
from django.test import TestCase
"""
A basic unittest for the "Course Addition Tool"
......@@ -14,7 +15,7 @@ sys.path.append(
from course_validate import VEDACat
class TestVariables(unittest.TestCase):
class TestVariables(TestCase):
def setUp(self):
self.VCT = VEDACat()
......
instance_config.yaml
......@@ -32,17 +32,38 @@ debug: false
edx_s3_ingest_prefix:
edx_s3_ingest_bucket:
edx_s3_endpoint_bucket:
# CF
edx_cloudfront_prefix:
# Images
aws_video_images_bucket:
aws_video_images_prefix: "video-images/"
# VEDA Internal
veda_s3_upload_bucket:
veda_s3_hotstore_bucket:
veda_deliverable_bucket:
# Settings
multi_upload_barrier: 2000000000
veda_base_url:
s3_base_url: https://s3.amazonaws.com
# Transcripts
aws_video_transcripts_bucket:
aws_video_transcripts_prefix: video-transcripts/
# cielo24 api urls
cielo24_api_base_url: https://sandbox.cielo24.com/api
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
# a token identifying a valid request from transcript provider
transcript_provider_request_token: testtoken
# Ingest Secret
# TODO: Elminate access key after AWS Support ticket 08/20/17 regarding cross-account IAM role access.
......@@ -76,6 +97,8 @@ val_client_id:
val_secret_key:
val_password:
val_username:
val_transcript_create_url:
val_video_transcript_status_url:
# ---
# Celery Info
......
pytest.ini 0 → 100644
[pytest]
DJANGO_SETTINGS_MODULE = VEDA.test_settings
requirements.txt
## NOTE: This is not a working req file -- merely a collection for notes.
##
django==1.9
djangorestframework==3.6.4
django-cors-headers
django-oauth-toolkit==0.11.0
django-model-utils==3.0.0
django-filter==1.0.4
newrelic
uwsgi
......@@ -11,3 +12,5 @@ boto
pyyaml
requests==2.18.1
celery==3.1.18
pysrt==1.1.1
edx-opaque-keys==0.4
\ No newline at end of file
test_config.yaml 0 → 100644
---
veda_s3_hotstore_bucket: s3_hotstore_bucket
multi_upload_barrier: 2000000000
veda_base_url: https://veda.edx.org
s3_base_url: https://s3.amazonaws.com
# transcript bucket config
aws_video_transcripts_bucket: bucket_name
aws_video_transcripts_prefix: video-transcripts/
# cielo24 api urls
cielo24_api_base_url: https://sandbox.cielo24.com/api
# 3playmedia api urls
three_play_api_base_url: https://api.3playmedia.com/
three_play_api_transcript_url: https://static.3playmedia.com/
# a token identifying a valid request from transcript provider
transcript_provider_request_token: 1234a5a67cr890
# ---
# VAL
# ---
val_api_url: http://val.edx.org/api
val_token_url: http://val.edx.org/token
val_video_images_url:
# Credentials
val_client_id: client
val_secret_key: secret
val_password: password
val_username: username
val_transcript_create_url: http://val.edx.org/transcript/create
val_video_transcript_status_url: http://val.edx.org/video/status
celery_worker_queue: encode_worker
celery_deliver_queue: deliver_worker
# ----------
##---
# This is a list of encodes and their respective course
# boolean matches
encode_dict:
review_proc:
- review
mobile_override:
- override
s3_proc:
- mobile_high
- mobile_low
- audio_mp3
- desktop_webm
- desktop_mp4
- hls
yt_proc:
- youtube
# This is a list of encode profiles and their val profile matches
# boolean matches
val_profile_dict:
mobile_low:
- mobile_low
desktop_mp4:
- desktop_mp4
override:
- desktop_mp4
- mobile_low
- mobile_high
mobile_high:
- mobile_high
audio_mp3:
- audio_mp3
desktop_webm:
- desktop_webm
youtube:
- youtube
review:
hls:
- hls
# Heal settings
heal_start: 2
heal_end: 50
global_timeout: 40
test_requirements.txt
## NOTE: Test requirements.
codecov==2.0.9
pep8==1.7.0
coverage==3.7.1
coverage==4.2
isort==4.2.15
ddt==1.1.1
moto==1.0.1
responses==0.6.1
pytest==3.0.6
pytest-django==3.1.2
pytest-django-ordering==1.0.1
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment