Skip to content

E
edx-video-pipeline
Unverified Commit d096135b by Muhammad Ammar Committed by GitHub
Options

Merge pull request #86 from edx/revert-84-ammar/sandbox-testing-fixes 

Revert "sandbox testing fixes"
parents 192f378e 5f79e241
Showing with 36 additions and 330 deletions
VEDA/settings/production.py
......@@ -17,7 +17,7 @@ LOGGING = get_logger_config(service_variant=CONFIG_DATA.get('SERVICE_VARIANT_NAM
# Keep track of the names of settings that represent dicts. Instead of overriding the values in base.py,
# the values read from disk should UPDATE the pre-configured dicts.
DICT_UPDATE_KEYS = ('DATABASES',)
DICT_UPDATE_KEYS = ('DATABASES', 'JWT_AUTH')
# Remove the items that should be used to update dicts, and apply them separately rather
# than pumping them into the local vars.
......@@ -28,10 +28,3 @@ for key, value in dict_updates.items():
vars()[key].update(value)
vars().update(CONFIG_DATA)
JWT_AUTH = {
'JWT_SECRET_KEY': CONFIG_DATA['val_secret_key'],
'JWT_ISSUER': '{}/oauth2'.format(CONFIG_DATA['lms_base_url'].rstrip('/')),
'JWT_AUDIENCE': CONFIG_DATA['val_client_id'],
'JWT_VERIFY_AUDIENCE': True,
}
VEDA/tests/test_utils.py
"""
Tests common utils
"""
import glob
import os
import shutil
import tempfile
from unittest import TestCase
......@@ -206,52 +204,3 @@ class UtilTests(TestCase):
utils.scrub_query_params(url, params_to_scrub),
expected_url
)
class DeleteDirectoryContentsTests(TestCase):
"""
Tests for `delete_directory_contents` util function.
"""
def setUp(self):
"""
Tests setup.
"""
# create a temp directory with temp directories and files in it
self.temp_dir = tempfile.mkdtemp()
dir_paths = map(lambda index: '{}/dir{}'.format(self.temp_dir, index), range(5))
for dir_path in dir_paths:
os.makedirs(dir_path)
__, file_path = tempfile.mkstemp(
suffix='.txt',
dir=dir_path
)
with open(file_path, 'w') as outfile:
outfile.write(str(TEST_CONFIG))
# create a temp file in root temp directory
with open('{}/{}'.format(self.temp_dir, 'temp_file.text'), 'w') as outfile:
outfile.write(str(TEST_CONFIG))
def tearDown(self):
"""
Reverse the setup
"""
shutil.rmtree(self.temp_dir)
def test_delete_directory_contents(self):
"""
Tests that utils.scrub_query_params works as expected.
"""
# Verify that directory is not empty
self.assertEqual(
len(glob.glob('{path}/*'.format(path=self.temp_dir))),
6
)
utils.delete_directory_contents(self.temp_dir)
# Verify that directory is empty
self.assertEqual(
glob.glob('{path}/*'.format(path=self.temp_dir)),
[]
)
VEDA/utils.py
"""
Common utils.
"""
import glob
import os
import shutil
import urllib
import urlparse
import yaml
......@@ -41,7 +39,7 @@ def build_url(*urls, **query_params):
Returns:
absolute url
"""
url = '/'.join(item.strip('/') for item in urls if item)
url = '/'.join(item.strip('/') for item in urls)
if query_params:
url = '{}?{}'.format(url, urllib.urlencode(query_params))
......@@ -106,21 +104,3 @@ def scrub_query_params(url, params_to_scrub):
parsed.path,
**new_query_params
)
def delete_directory_contents(path):
"""
Deletes everything inside a directory. Do nothing if path is not a directory.
Arguments:
path (str): path to a directory.
"""
if not os.path.isdir(path):
return
for file_path in glob.glob('{path}/*'.format(path=path.rstrip('/'))):
if os.path.isdir(file_path):
shutil.rmtree(file_path)
if os.path.isfile(file_path):
os.remove(file_path)
VEDA_OS01/fixtures/encodes.json deleted 100644 → 0
[
{
"model": "VEDA_OS01.destination",
"pk": 1,
"fields": {
"destination_name": "Low Bandwidth Override (AWS S3)",
"destination_active": true,
"destination_nick": "LBO"
}
},
{
"model": "VEDA_OS01.destination",
"pk": 2,
"fields": {
"destination_name": "Youtube - Review",
"destination_active": true,
"destination_nick": "YTR"
}
},
{
"model": "VEDA_OS01.destination",
"pk": 3,
"fields": {
"destination_name": "Youtube - Primary",
"destination_active": true,
"destination_nick": "YT1"
}
},
{
"model": "VEDA_OS01.destination",
"pk": 4,
"fields": {
"destination_name": "Amazon AWS",
"destination_active": true,
"destination_nick": "S31"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 1,
"fields": {
"encode_destination": 4,
"encode_name": "Desktop - High",
"profile_active": true,
"encode_suffix": "DTH",
"encode_filetype": "mp4",
"encode_bitdepth": "27",
"encode_resolution": "720",
"product_spec": "desktop_mp4"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 2,
"fields": {
"encode_destination": 4,
"encode_name": "HLS",
"profile_active": true,
"encode_suffix": "HLS",
"encode_filetype": "HLS",
"encode_bitdepth": "0",
"encode_resolution": "0",
"product_spec": "hls"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 3,
"fields": {
"encode_destination": 1,
"encode_name": "Low Bandwidth Override",
"profile_active": true,
"encode_suffix": "LBO",
"encode_filetype": "mp4",
"encode_bitdepth": "27",
"encode_resolution": "360",
"product_spec": "override"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 4,
"fields": {
"encode_destination": 4,
"encode_name": "Mobile - Low",
"profile_active": true,
"encode_suffix": "MB2",
"encode_filetype": "mp4",
"encode_bitdepth": "27",
"encode_resolution": "360",
"product_spec": "mobile_low"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 5,
"fields": {
"encode_destination": 4,
"encode_name": "mp3",
"profile_active": true,
"encode_suffix": "AUD",
"encode_filetype": "mp3",
"encode_bitdepth": "192",
"encode_resolution": "0",
"product_spec": "audio_mp3"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 6,
"fields": {
"encode_destination": 3,
"encode_name": "Youtube - Primary",
"profile_active": true,
"encode_suffix": "100",
"encode_filetype": "mp4",
"encode_bitdepth": "18",
"encode_resolution": "1080",
"product_spec": "youtube"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 7,
"fields": {
"encode_destination": 2,
"encode_name": "Youtube - Review",
"profile_active": true,
"encode_suffix": "RVW",
"encode_filetype": "mp4",
"encode_bitdepth": "18",
"encode_resolution": "1080",
"product_spec": "review"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 8,
"fields": {
"encode_destination": 4,
"encode_name": "Mobile - High",
"profile_active": false,
"encode_suffix": "MB1",
"encode_filetype": "mp4",
"encode_bitdepth": "30",
"encode_resolution": "540",
"product_spec": "mobile_high"
}
},
{
"model": "VEDA_OS01.encode",
"pk": 9,
"fields": {
"encode_destination": 4,
"encode_name": "WEBM - Desktop High",
"profile_active": false,
"encode_suffix": "DTH",
"encode_filetype": "webm",
"encode_bitdepth": "1000",
"encode_resolution": "720",
"product_spec": "desktop_webm"
}
}
]
VEDA_OS01/models.py
......@@ -540,7 +540,7 @@ class Destination(models.Model):
destination_nick = models.CharField('Nickname (3 Char.)', max_length=3, null=True, blank=True)
def __unicode__(self):
return u'{}'.format(self.destination_name)
return u'%s'.format(self.destination_name) or u''
class Encode(models.Model):
......
VEDA_OS01/tests/test_transcripts.py
......@@ -255,6 +255,13 @@ class Cielo24TranscriptTests(APITestCase):
video = Video.objects.get(studio_id=self.video.studio_id)
self.assertEqual(video.transcript_status, TranscriptStatus.READY)
# verify sjson data uploaded to s3
bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
key = Key(bucket)
key.key = transcript_create_request_data['name']
sjson = json.loads(key.get_contents_as_string())
self.assertEqual(sjson, TRANSCRIPT_SJSON_DATA)
@patch('VEDA_OS01.transcripts.LOGGER')
@responses.activate
def test_fetch_exception_log(self, mock_logger):
......@@ -456,8 +463,8 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
Verify sjson data uploaded to s3
"""
key = Key(connection.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket']))
key.key = '{transcript_name}.sjson'.format(
transcript_name=transcripts.construct_transcript_names(CONFIG_DATA)[1]
key.key = '{directory}{uuid}.sjson'.format(
directory=CONFIG_DATA['aws_video_transcripts_prefix'], uuid=self.uuid_hex
)
sjson_transcript = json.loads(key.get_contents_as_string())
self.assertEqual(sjson_transcript, TRANSCRIPT_SJSON_DATA)
......@@ -1558,26 +1565,3 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
self.edx_video_id,
self.file_id,
)
class TranscriptNameConstructionTests(APITestCase):
"""
Tests for `construct_transcript_names` util function
"""
def setUp(self):
"""
Tests setup.
"""
super(TranscriptNameConstructionTests, self).setUp()
def test_upload_sjson_to_s3(self):
"""
Verify that `construct_transcript_names` works as expected.
"""
edxval_name, s3_name = transcripts.construct_transcript_names(CONFIG_DATA)
self.assertTrue(
s3_name.startswith(CONFIG_DATA['instance_prefix'])
)
self.assertTrue(
s3_name.endswith(edxval_name)
)
VEDA_OS01/transcripts.py
......@@ -314,53 +314,21 @@ def convert_srt_to_sjson(srt_data):
return subs
def construct_transcript_names(config):
"""
Constructs transcript names for 'edxval' and 's3'
Arguments:
config (dict): instance configuration
Returns:
transcript names for 'edxval' and 's3'
"""
transcript_name_without_instance_prefix = build_url(
config['aws_video_transcripts_prefix'],
uuid.uuid4().hex
)
transcript_name_with_instance_prefix = build_url(
config['instance_prefix'],
transcript_name_without_instance_prefix
)
return transcript_name_without_instance_prefix, transcript_name_with_instance_prefix
def upload_sjson_to_s3(config, sjson_data):
"""
Upload sjson data to s3.
Arguments:
config (dict): instance configuration
sjson_data (list): transcript data to be uploaded to `s3`
Returns:
transcript name for 'edxval'
"""
s3_conn = boto.connect_s3()
bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
k = Key(bucket)
k.content_type = 'application/json'
transcript_name_without_instance_prefix, transcript_name_with_instance_prefix = construct_transcript_names(config)
k.key = '{}.sjson'.format(transcript_name_with_instance_prefix)
k.key = '{directory}{uuid}.sjson'.format(
directory=config['aws_video_transcripts_prefix'],
uuid=uuid.uuid4().hex
)
k.set_contents_from_string(json.dumps(sjson_data))
k.set_acl('public-read')
# transcript path is stored in edxval without `instance_prefix`
return '{}.sjson'.format(transcript_name_without_instance_prefix)
return k.key
class ThreePlayMediaCallbackHandlerView(APIView):
......
bin/loop.py
......@@ -7,8 +7,6 @@ from django.db import reset_queries
import resource
import time
from control.control_env import WORK_DIRECTORY
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_path not in sys.path:
sys.path.append(project_path)
......@@ -71,8 +69,14 @@ class DaemonCli:
def ingest_daemon(self):
x = 0
while True:
node_work_directory = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
))),
'VEDA_WORKING'
)
FD = FileDiscovery(
node_work_directory=WORK_DIRECTORY
node_work_directory=node_work_directory
)
FD.discover_studio_ingested_videos()
......
control/control_env.py
......@@ -25,18 +25,14 @@ from VEDA_OS01.models import Destination
from VEDA_OS01.models import Encode
from VEDA_OS01.models import URL
from VEDA_OS01.models import VedaUpload
from VEDA.utils import get_config
"""
Central Config
"""
CONFIG = get_config()
DEFAULT_WORK_DIRECTORY = os.path.join(
WORK_DIRECTORY = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'VEDA_WORKING'
)
WORK_DIRECTORY = CONFIG.get('VEDA_WORKING', DEFAULT_WORK_DIRECTORY)
if not os.path.exists(WORK_DIRECTORY):
os.mkdir(WORK_DIRECTORY)
......
control/veda_deliver.py
......@@ -25,7 +25,7 @@ from veda_deliver_youtube import DeliverYoutube
from VEDA_OS01 import utils
from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
TranscriptStatus)
from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents
from VEDA.utils import build_url, extract_course_org, get_config
from veda_utils import ErrorObject, Metadata, Output, VideoProto
from veda_val import VALAPICall
from veda_video_validation import Validation
......@@ -77,7 +77,8 @@ class VedaDelivery:
else:
if os.path.exists(WORK_DIRECTORY):
delete_directory_contents(WORK_DIRECTORY)
shutil.rmtree(WORK_DIRECTORY)
os.mkdir(WORK_DIRECTORY)
self._INFORM_INTAKE()
......
control/veda_file_discovery.py
......@@ -250,7 +250,7 @@ class FileDiscovery(object):
connection = boto.connect_s3()
self.bucket = connection.get_bucket(self.auth_dict['edx_s3_ingest_bucket'])
for video_s3_key in self.bucket.list(self.auth_dict['edx_s3_ingest_prefix'], '/'):
if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']:
if video_s3_key.name != 'prod-edx/unprocessed/':
self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
except S3ResponseError:
ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')
......
frontend/views.py
......@@ -196,8 +196,7 @@ def upload_alpha_1(request):
'policy': policy,
'signature': signature,
'abvid_serial': abvid_serial,
'access_key': auth_dict['veda_access_key_id'],
'upload_bucket': auth_dict['veda_upload_bucket'],
'access_key': auth_dict['veda_access_key_id']
})
)
return HttpResponse(template.render(context))
......
instance_config.yaml
......@@ -98,8 +98,8 @@ val_api_url:
val_token_url:
val_video_images_url:
# Credentials
val_client_id: 'clientkey'
val_secret_key: 'secretkey'
val_client_id:
val_secret_key:
val_password:
val_username:
val_transcript_create_url:
......@@ -121,6 +121,3 @@ sg_script_key:
# Endpoints
# ---
threeplay_ftphost:
lms_base_url: ''
instance_prefix: ''
templates/upload_video.html
......@@ -21,11 +21,11 @@ var abvid_serial = "{{abvid_serial}}"
</head>
<body>
<body>
<div id="initial_title"><h1>
edX About Video Upload
</h1>
</div>
</div>
<div id="inst_lookup">
<h3 style="font-size: 2.0em;">File Upload Complete</h3>
<span class="advisory" style="margin-left: 49px;">Thank you, file has been received. You can close this window<br></span>
......@@ -37,7 +37,7 @@ var abvid_serial = "{{abvid_serial}}"
</div></div>
<div id="uploadselect">
<form class="dropzone" id="dmz" action="https://{{ upload_bucket }}.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
<form class="dropzone" id="dmz" action="https://veda-uploads.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
<input type="hidden" name="key" value="upload/{{ abvid_serial }}">
<input type="hidden" name="AWSAccessKeyId" value="{{ access_key }}">
<input type="hidden" name="acl" value="private">
......@@ -48,7 +48,7 @@ var abvid_serial = "{{abvid_serial}}"
</form>
<span class="advisory" style="margin-left: 30%;">Do not close or refresh this window while file is transferring<br>
Max. 1 file at once, file must be smaller than 5GB</span>
</div>
</div>
<div id="new_buttons">
<form id="reset-form" method="post" >
<input id="rstb" class="reset_button" type="reset" value="Reset"/>
......
test_config.yaml
......@@ -87,4 +87,3 @@ heal_end: 50
global_timeout: 40
instance_prefix: '127.0.0.1'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment