Skip to content

E
edx-video-pipeline
Commit cbc5b4b5 by muhammad-ammar
Options

sandbox fixes

parent 9b01667a
Showing with 327 additions and 33 deletions
VEDA/settings/production.py
......@@ -17,7 +17,7 @@ LOGGING = get_logger_config(service_variant=CONFIG_DATA.get('SERVICE_VARIANT_NAM
# Keep track of the names of settings that represent dicts. Instead of overriding the values in base.py,
# the values read from disk should UPDATE the pre-configured dicts.
DICT_UPDATE_KEYS = ('DATABASES', 'JWT_AUTH')
DICT_UPDATE_KEYS = ('DATABASES',)
# Remove the items that should be used to update dicts, and apply them separately rather
# than pumping them into the local vars.
......@@ -28,3 +28,10 @@ for key, value in dict_updates.items():
vars()[key].update(value)
vars().update(CONFIG_DATA)
JWT_AUTH = {
'JWT_SECRET_KEY': CONFIG_DATA['val_secret_key'],
'JWT_ISSUER': '{}/oauth2'.format(CONFIG_DATA['lms_base_url'].rstrip('/')),
'JWT_AUDIENCE': CONFIG_DATA['val_client_id'],
'JWT_VERIFY_AUDIENCE': True,
}
VEDA/tests/test_utils.py
"""
Tests common utils
"""
import glob
import os
import shutil
import tempfile
from unittest import TestCase
......@@ -204,3 +206,52 @@ class UtilTests(TestCase):
utils.scrub_query_params(url, params_to_scrub),
expected_url
)
class DeleteDirectoryContentsTests(TestCase):
"""
Tests for `delete_directory_contents` util function.
"""
def setUp(self):
"""
Tests setup.
"""
# create a temp directory with temp directories and files in it
self.temp_dir = tempfile.mkdtemp()
dir_paths = map(lambda index: '{}/dir{}'.format(self.temp_dir, index), range(5))
for dir_path in dir_paths:
os.makedirs(dir_path)
__, file_path = tempfile.mkstemp(
suffix='.txt',
dir=dir_path
)
with open(file_path, 'w') as outfile:
outfile.write(str(TEST_CONFIG))
# create a temp file in root temp directory
with open('{}/{}'.format(self.temp_dir, 'temp_file.text'), 'w') as outfile:
outfile.write(str(TEST_CONFIG))
def tearDown(self):
"""
Reverse the setup
"""
shutil.rmtree(self.temp_dir)
def test_delete_directory_contents(self):
"""
Tests that utils.scrub_query_params works as expected.
"""
# Verify that directory is not empty
self.assertEqual(
len(glob.glob('{path}/*'.format(path=self.temp_dir))),
6
)
utils.delete_directory_contents(self.temp_dir)
# Verify that directory is empty
self.assertEqual(
glob.glob('{path}/*'.format(path=self.temp_dir)),
[]
)
VEDA/utils.py
"""
Common utils.
"""
import glob
import os
import shutil
import urllib
import urlparse
import yaml
......@@ -39,7 +41,7 @@ def build_url(*urls, **query_params):
Returns:
absolute url
"""
url = '/'.join(item.strip('/') for item in urls)
url = '/'.join(item.strip('/') for item in urls if item)
if query_params:
url = '{}?{}'.format(url, urllib.urlencode(query_params))
......@@ -104,3 +106,21 @@ def scrub_query_params(url, params_to_scrub):
parsed.path,
**new_query_params
)
def delete_directory_contents(path):
"""
Deletes everything inside a directory. Do nothing if path is not a directory.
Arguments:
path (str): path to a directory.
"""
if not os.path.isdir(path):
return
for file_path in glob.glob('{path}/*'.format(path=path.rstrip('/'))):
if os.path.isdir(file_path):
shutil.rmtree(file_path)
if os.path.isfile(file_path):
os.remove(file_path)
VEDA_OS01/fixtures/encodes.json 0 → 100644
[
{
"model": "VEDA_OS01.destination",
"pk": 1,
"fields": {
"destination_name": "Low Bandwidth Override (AWS S3)",
"destination_active": true,
"destination_nick": "LBO"
}
},
{
"model": "VEDA_OS01.destination",
"pk": 2,
"fields": {
"destination_name": "Youtube - Review",
"destination_active": true,
"destination_nick": "YTR"
}
},
{
"model": "VEDA_OS01.destination",
"pk": 3,
"fields": {
"destination_name": "Youtube - Primary",
"destination_active": true,
"destination_nick": "YT1"
}
},
{
"model": "VEDA_OS01.destination",
"pk": 4,
"fields": {
"destination_name": "Amazon AWS",
"destination_active": true,
"destination_nick": "S31"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 1,
"fields": {
"encode_destination": 4,
"encode_name": "Desktop - High",
"profile_active": true,
"encode_suffix": "DTH",
"encode_filetype": "mp4",
"encode_bitdepth": "27",
"encode_resolution": "720",
"product_spec": "desktop_mp4"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 2,
"fields": {
"encode_destination": 4,
"encode_name": "HLS",
"profile_active": true,
"encode_suffix": "HLS",
"encode_filetype": "HLS",
"encode_bitdepth": "0",
"encode_resolution": "0",
"product_spec": "hls"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 3,
"fields": {
"encode_destination": 1,
"encode_name": "Low Bandwidth Override",
"profile_active": true,
"encode_suffix": "LBO",
"encode_filetype": "mp4",
"encode_bitdepth": "27",
"encode_resolution": "360",
"product_spec": "override"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 4,
"fields": {
"encode_destination": 4,
"encode_name": "Mobile - Low",
"profile_active": true,
"encode_suffix": "MB2",
"encode_filetype": "mp4",
"encode_bitdepth": "27",
"encode_resolution": "360",
"product_spec": "mobile_low"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 5,
"fields": {
"encode_destination": 4,
"encode_name": "mp3",
"profile_active": true,
"encode_suffix": "AUD",
"encode_filetype": "mp3",
"encode_bitdepth": "192",
"encode_resolution": "0",
"product_spec": "audio_mp3"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 6,
"fields": {
"encode_destination": 3,
"encode_name": "Youtube - Primary",
"profile_active": true,
"encode_suffix": "100",
"encode_filetype": "mp4",
"encode_bitdepth": "18",
"encode_resolution": "1080",
"product_spec": "youtube"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 7,
"fields": {
"encode_destination": 2,
"encode_name": "Youtube - Review",
"profile_active": true,
"encode_suffix": "RVW",
"encode_filetype": "mp4",
"encode_bitdepth": "18",
"encode_resolution": "1080",
"product_spec": "review"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 8,
"fields": {
"encode_destination": 4,
"encode_name": "Mobile - High",
"profile_active": false,
"encode_suffix": "MB1",
"encode_filetype": "mp4",
"encode_bitdepth": "30",
"encode_resolution": "540",
"product_spec": "mobile_high"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 9,
"fields": {
"encode_destination": 4,
"encode_name": "WEBM - Desktop High",
"profile_active": false,
"encode_suffix": "DTH",
"encode_filetype": "webm",
"encode_bitdepth": "1000",
"encode_resolution": "720",
"product_spec": "desktop_webm"
}
}
]
VEDA_OS01/models.py
......@@ -540,7 +540,7 @@ class Destination(models.Model):
destination_nick = models.CharField('Nickname (3 Char.)', max_length=3, null=True, blank=True)
def __unicode__(self):
return u'%s'.format(self.destination_name) or u''
return u'{}'.format(self.destination_name)
class Encode(models.Model):
......
VEDA_OS01/tests/test_transcripts.py
......@@ -255,13 +255,6 @@ class Cielo24TranscriptTests(APITestCase):
video = Video.objects.get(studio_id=self.video.studio_id)
self.assertEqual(video.transcript_status, TranscriptStatus.READY)
# verify sjson data uploaded to s3
bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
key = Key(bucket)
key.key = transcript_create_request_data['name']
sjson = json.loads(key.get_contents_as_string())
self.assertEqual(sjson, TRANSCRIPT_SJSON_DATA)
@patch('VEDA_OS01.transcripts.LOGGER')
@responses.activate
def test_fetch_exception_log(self, mock_logger):
......@@ -463,8 +456,8 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
Verify sjson data uploaded to s3
"""
key = Key(connection.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket']))
key.key = '{directory}{uuid}.sjson'.format(
directory=CONFIG_DATA['aws_video_transcripts_prefix'], uuid=self.uuid_hex
key.key = '{transcript_name}.sjson'.format(
transcript_name=transcripts.construct_transcript_names(CONFIG_DATA)[1]
)
sjson_transcript = json.loads(key.get_contents_as_string())
self.assertEqual(sjson_transcript, TRANSCRIPT_SJSON_DATA)
......@@ -1565,3 +1558,26 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
self.edx_video_id,
self.file_id,
)
class TranscriptNameConstructionTests(APITestCase):
"""
Tests for `construct_transcript_names` util function
"""
def setUp(self):
"""
Tests setup.
"""
super(TranscriptNameConstructionTests, self).setUp()
def test_upload_sjson_to_s3(self):
"""
Verify that `construct_transcript_names` works as expected.
"""
edxval_name, s3_name = transcripts.construct_transcript_names(CONFIG_DATA)
self.assertTrue(
s3_name.startswith(CONFIG_DATA['instance_prefix'])
)
self.assertTrue(
s3_name.endswith(edxval_name)
)
VEDA_OS01/transcripts.py
......@@ -314,21 +314,53 @@ def convert_srt_to_sjson(srt_data):
return subs
def construct_transcript_names(config):
"""
Constructs transcript names for 'edxval' and 's3'
Arguments:
config (dict): instance configuration
Returns:
transcript names for 'edxval' and 's3'
"""
transcript_name_without_instance_prefix = build_url(
config['aws_video_transcripts_prefix'],
uuid.uuid4().hex
)
transcript_name_with_instance_prefix = build_url(
config['instance_prefix'],
transcript_name_without_instance_prefix
)
return transcript_name_without_instance_prefix, transcript_name_with_instance_prefix
def upload_sjson_to_s3(config, sjson_data):
"""
Upload sjson data to s3.
Arguments:
config (dict): instance configuration
sjson_data (list): transcript data to be uploaded to `s3`
Returns:
transcript name for 'edxval'
"""
s3_conn = boto.connect_s3()
bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
k = Key(bucket)
k.content_type = 'application/json'
k.key = '{directory}{uuid}.sjson'.format(
directory=config['aws_video_transcripts_prefix'],
uuid=uuid.uuid4().hex
)
transcript_name_without_instance_prefix, transcript_name_with_instance_prefix = construct_transcript_names(config)
k.key = '{}.sjson'.format(transcript_name_with_instance_prefix)
k.set_contents_from_string(json.dumps(sjson_data))
k.set_acl('public-read')
return k.key
# transcript path is stored in edxval without `instance_prefix`
return '{}.sjson'.format(transcript_name_without_instance_prefix)
class ThreePlayMediaCallbackHandlerView(APIView):
......
bin/loop.py
......@@ -7,6 +7,8 @@ from django.db import reset_queries
import resource
import time
from control.control_env import WORK_DIRECTORY
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path:
sys.path.append(project_path)
......@@ -69,14 +71,8 @@ class DaemonCli:
def ingest_daemon(self):
x = 0
while True:
node_work_directory = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
))),
'VEDA_WORKING'
)
FD = FileDiscovery(
node_work_directory=node_work_directory
node_work_directory=WORK_DIRECTORY
)
FD.discover_studio_ingested_videos()
......
control/control_env.py
......@@ -25,14 +25,18 @@ from VEDA_OS01.models import Destination
from VEDA_OS01.models import Encode
from VEDA_OS01.models import URL
from VEDA_OS01.models import VedaUpload
from VEDA.utils import get_config
"""
Central Config
"""
WORK_DIRECTORY = os.path.join(
CONFIG = get_config()
DEFAULT_WORK_DIRECTORY = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'VEDA_WORKING'
)
WORK_DIRECTORY = CONFIG.get('VEDA_WORKING', DEFAULT_WORK_DIRECTORY)
if not os.path.exists(WORK_DIRECTORY):
os.mkdir(WORK_DIRECTORY)
......
control/veda_deliver.py
......@@ -25,7 +25,7 @@ from veda_deliver_youtube import DeliverYoutube
from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
TranscriptStatus)
from VEDA.utils import build_url, extract_course_org, get_config
from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents
from veda_utils import ErrorObject, Metadata, Output, VideoProto
from veda_val import VALAPICall
from veda_video_validation import Validation
......@@ -77,8 +77,7 @@ class VedaDelivery:
else:
if os.path.exists(WORK_DIRECTORY):
shutil.rmtree(WORK_DIRECTORY)
os.mkdir(WORK_DIRECTORY)
delete_directory_contents(WORK_DIRECTORY)
self._INFORM_INTAKE()
......
control/veda_file_discovery.py
......@@ -250,7 +250,7 @@ class FileDiscovery(object):
connection = boto.connect_s3()
self.bucket = connection.get_bucket(self.auth_dict['edx_s3_ingest_bucket'])
for video_s3_key in self.bucket.list(self.auth_dict['edx_s3_ingest_prefix'], '/'):
if video_s3_key.name != 'prod-edx/unprocessed/':
if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']:
self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
except S3ResponseError:
ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')
......
frontend/views.py
......@@ -196,7 +196,8 @@ def upload_alpha_1(request):
'policy': policy,
'signature': signature,
'abvid_serial': abvid_serial,
'access_key': auth_dict['veda_access_key_id']
'access_key': auth_dict['veda_access_key_id'],
'upload_bucket': auth_dict['veda_upload_bucket'],
})
)
return HttpResponse(template.render(context))
......
instance_config.yaml
......@@ -98,8 +98,8 @@ val_api_url:
val_token_url:
val_video_images_url:
# Credentials
val_client_id:
val_secret_key:
val_client_id: 'clientkey'
val_secret_key: 'secretkey'
val_password:
val_username:
val_transcript_create_url:
......@@ -121,3 +121,6 @@ sg_script_key:
# Endpoints
# ---
threeplay_ftphost:
lms_base_url: ''
instance_prefix: ''
templates/upload_video.html
......@@ -37,7 +37,7 @@ var abvid_serial = "{{abvid_serial}}"
</div></div>
<div id="uploadselect">
<form class="dropzone" id="dmz" action="https://veda-uploads.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
<form class="dropzone" id="dmz" action="https://{{ upload_bucket }}.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
<input type="hidden" name="key" value="upload/{{ abvid_serial }}">
<input type="hidden" name="AWSAccessKeyId" value="{{ access_key }}">
<input type="hidden" name="acl" value="private">
......
test_config.yaml
......@@ -87,3 +87,4 @@ heal_end: 50
global_timeout: 40
instance_prefix: '127.0.0.1'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment