sandbox fixes

VEDA/settings/production.py

@@ -17,7 +17,7 @@ LOGGING = get_logger_config(service_variant=CONFIG_DATA.get('SERVICE_VARIANT_NAM
 
 # Keep track of the names of settings that represent dicts. Instead of overriding the values in base.py,
 # the values read from disk should UPDATE the pre-configured dicts.
-DICT_UPDATE_KEYS = ('DATABASES', 'JWT_AUTH')
+DICT_UPDATE_KEYS = ('DATABASES',)
 
 # Remove the items that should be used to update dicts, and apply them separately rather
 # than pumping them into the local vars.
@@ -28,3 +28,10 @@ for key, value in dict_updates.items():
         vars()[key].update(value)
 
 vars().update(CONFIG_DATA)
+
+JWT_AUTH = {
+    'JWT_SECRET_KEY': CONFIG_DATA['val_secret_key'],
+    'JWT_ISSUER': '{}/oauth2'.format(CONFIG_DATA['lms_base_url'].rstrip('/')),
+    'JWT_AUDIENCE': CONFIG_DATA['val_client_id'],
+    'JWT_VERIFY_AUDIENCE': True,
+}

VEDA/tests/test_utils.py

 """
 Tests common utils
 """
+import glob
 import os
+import shutil
 import tempfile
 from unittest import TestCase
 
@@ -204,3 +206,52 @@ class UtilTests(TestCase):
             utils.scrub_query_params(url, params_to_scrub),
             expected_url
         )
+
+
+class DeleteDirectoryContentsTests(TestCase):
+    """
+    Tests for `delete_directory_contents` util function.
+    """
+    def setUp(self):
+        """
+        Tests setup.
+        """
+        # create a temp directory with temp directories and files in it
+        self.temp_dir = tempfile.mkdtemp()
+        dir_paths = map(lambda index: '{}/dir{}'.format(self.temp_dir, index), range(5))
+        for dir_path in dir_paths:
+            os.makedirs(dir_path)
+            __, file_path = tempfile.mkstemp(
+                suffix='.txt',
+                dir=dir_path
+            )
+            with open(file_path, 'w') as outfile:
+                outfile.write(str(TEST_CONFIG))
+
+        # create a temp file in root temp directory
+        with open('{}/{}'.format(self.temp_dir, 'temp_file.text'), 'w') as outfile:
+            outfile.write(str(TEST_CONFIG))
+
+    def tearDown(self):
+        """
+        Reverse the setup
+        """
+        shutil.rmtree(self.temp_dir)
+
+    def test_delete_directory_contents(self):
+        """
+        Tests that utils.scrub_query_params works as expected.
+        """
+        # Verify that directory is not empty
+        self.assertEqual(
+            len(glob.glob('{path}/*'.format(path=self.temp_dir))),
+            6
+        )
+
+        utils.delete_directory_contents(self.temp_dir)
+
+        # Verify that directory is empty
+        self.assertEqual(
+            glob.glob('{path}/*'.format(path=self.temp_dir)),
+            []
+        )

VEDA/utils.py

 """
 Common utils.
 """
+import glob
 import os
+import shutil
 import urllib
 import urlparse
 import yaml
@@ -39,7 +41,7 @@ def build_url(*urls, **query_params):
     Returns:
         absolute url
     """
-    url = '/'.join(item.strip('/') for item in urls)
+    url = '/'.join(item.strip('/') for item in urls if item)
     if query_params:
         url = '{}?{}'.format(url, urllib.urlencode(query_params))
 
@@ -104,3 +106,21 @@ def scrub_query_params(url, params_to_scrub):
         parsed.path,
         **new_query_params
     )
+
+
+def delete_directory_contents(path):
+    """
+    Deletes everything inside a directory. Do nothing if path is not a directory.
+
+    Arguments:
+        path (str): path to a directory.
+    """
+    if not os.path.isdir(path):
+        return
+
+    for file_path in glob.glob('{path}/*'.format(path=path.rstrip('/'))):
+        if os.path.isdir(file_path):
+            shutil.rmtree(file_path)
+
+        if os.path.isfile(file_path):
+            os.remove(file_path)

VEDA_OS01/fixtures/encodes.json

+[
+    {
+        "model": "VEDA_OS01.destination",
+        "pk": 1,
+        "fields": {
+            "destination_name": "Low Bandwidth Override (AWS S3)",
+            "destination_active": true,
+            "destination_nick": "LBO"
+        }
+    },
+    {
+        "model": "VEDA_OS01.destination",
+        "pk": 2,
+        "fields": {
+            "destination_name": "Youtube - Review",
+            "destination_active": true,
+            "destination_nick": "YTR"
+        }
+    },
+        {
+        "model": "VEDA_OS01.destination",
+        "pk": 3,
+        "fields": {
+            "destination_name": "Youtube - Primary",
+            "destination_active": true,
+            "destination_nick": "YT1"
+        }
+    },
+    {
+        "model": "VEDA_OS01.destination",
+        "pk": 4,
+        "fields": {
+            "destination_name": "Amazon AWS",
+            "destination_active": true,
+            "destination_nick": "S31"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 1,
+        "fields": {
+            "encode_destination": 4,
+            "encode_name": "Desktop - High",
+            "profile_active": true,
+            "encode_suffix": "DTH",
+            "encode_filetype": "mp4",
+            "encode_bitdepth": "27",
+            "encode_resolution": "720",
+            "product_spec": "desktop_mp4"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 2,
+        "fields": {
+            "encode_destination": 4,
+            "encode_name": "HLS",
+            "profile_active": true,
+            "encode_suffix": "HLS",
+            "encode_filetype": "HLS",
+            "encode_bitdepth": "0",
+            "encode_resolution": "0",
+            "product_spec": "hls"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 3,
+        "fields": {
+            "encode_destination": 1,
+            "encode_name": "Low Bandwidth Override",
+            "profile_active": true,
+            "encode_suffix": "LBO",
+            "encode_filetype": "mp4",
+            "encode_bitdepth": "27",
+            "encode_resolution": "360",
+            "product_spec": "override"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 4,
+        "fields": {
+            "encode_destination": 4,
+            "encode_name": "Mobile - Low",
+            "profile_active": true,
+            "encode_suffix": "MB2",
+            "encode_filetype": "mp4",
+            "encode_bitdepth": "27",
+            "encode_resolution": "360",
+            "product_spec": "mobile_low"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 5,
+        "fields": {
+            "encode_destination": 4,
+            "encode_name": "mp3",
+            "profile_active": true,
+            "encode_suffix": "AUD",
+            "encode_filetype": "mp3",
+            "encode_bitdepth": "192",
+            "encode_resolution": "0",
+            "product_spec": "audio_mp3"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 6,
+        "fields": {
+            "encode_destination": 3,
+            "encode_name": "Youtube - Primary",
+            "profile_active": true,
+            "encode_suffix": "100",
+            "encode_filetype": "mp4",
+            "encode_bitdepth": "18",
+            "encode_resolution": "1080",
+            "product_spec": "youtube"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 7,
+        "fields": {
+            "encode_destination": 2,
+            "encode_name": "Youtube - Review",
+            "profile_active": true,
+            "encode_suffix": "RVW",
+            "encode_filetype": "mp4",
+            "encode_bitdepth": "18",
+            "encode_resolution": "1080",
+            "product_spec": "review"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 8,
+        "fields": {
+            "encode_destination": 4,
+            "encode_name": "Mobile - High",
+            "profile_active": false,
+            "encode_suffix": "MB1",
+            "encode_filetype": "mp4",
+            "encode_bitdepth": "30",
+            "encode_resolution": "540",
+            "product_spec": "mobile_high"
+        }
+    },
+    {
+        "model": "VEDA_OS01.encode",
+        "pk": 9,
+        "fields": {
+            "encode_destination": 4,
+            "encode_name": "WEBM - Desktop High",
+            "profile_active": false,
+            "encode_suffix": "DTH",
+            "encode_filetype": "webm",
+            "encode_bitdepth": "1000",
+            "encode_resolution": "720",
+            "product_spec": "desktop_webm"
+        }
+    }
+]

VEDA_OS01/models.py

@@ -540,7 +540,7 @@ class Destination(models.Model):
     destination_nick = models.CharField('Nickname (3 Char.)', max_length=3, null=True, blank=True)
 
     def __unicode__(self):
-        return u'%s'.format(self.destination_name) or u''
+        return u'{}'.format(self.destination_name)
 
 
 class Encode(models.Model):

VEDA_OS01/tests/test_transcripts.py

@@ -255,13 +255,6 @@ class Cielo24TranscriptTests(APITestCase):
         video = Video.objects.get(studio_id=self.video.studio_id)
         self.assertEqual(video.transcript_status, TranscriptStatus.READY)
 
-        # verify sjson data uploaded to s3
-        bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
-        key = Key(bucket)
-        key.key = transcript_create_request_data['name']
-        sjson = json.loads(key.get_contents_as_string())
-        self.assertEqual(sjson, TRANSCRIPT_SJSON_DATA)
-
     @patch('VEDA_OS01.transcripts.LOGGER')
     @responses.activate
     def test_fetch_exception_log(self, mock_logger):
@@ -463,8 +456,8 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         Verify sjson data uploaded to s3
         """
         key = Key(connection.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket']))
-        key.key = '{directory}{uuid}.sjson'.format(
-            directory=CONFIG_DATA['aws_video_transcripts_prefix'], uuid=self.uuid_hex
+        key.key = '{transcript_name}.sjson'.format(
+            transcript_name=transcripts.construct_transcript_names(CONFIG_DATA)[1]
         )
         sjson_transcript = json.loads(key.get_contents_as_string())
         self.assertEqual(sjson_transcript, TRANSCRIPT_SJSON_DATA)
@@ -1565,3 +1558,26 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             self.edx_video_id,
             self.file_id,
         )
+
+
+class TranscriptNameConstructionTests(APITestCase):
+    """
+    Tests for `construct_transcript_names` util function
+    """
+    def setUp(self):
+        """
+        Tests setup.
+        """
+        super(TranscriptNameConstructionTests, self).setUp()
+
+    def test_upload_sjson_to_s3(self):
+        """
+        Verify that `construct_transcript_names` works as expected.
+        """
+        edxval_name, s3_name = transcripts.construct_transcript_names(CONFIG_DATA)
+        self.assertTrue(
+            s3_name.startswith(CONFIG_DATA['instance_prefix'])
+        )
+        self.assertTrue(
+            s3_name.endswith(edxval_name)
+        )

VEDA_OS01/transcripts.py

@@ -314,21 +314,53 @@ def convert_srt_to_sjson(srt_data):
     return subs
 
 
+def construct_transcript_names(config):
+    """
+    Constructs transcript names for 'edxval' and 's3'
+
+    Arguments:
+        config (dict): instance configuration
+
+    Returns:
+        transcript names for 'edxval' and 's3'
+    """
+    transcript_name_without_instance_prefix = build_url(
+        config['aws_video_transcripts_prefix'],
+        uuid.uuid4().hex
+    )
+
+    transcript_name_with_instance_prefix = build_url(
+        config['instance_prefix'],
+        transcript_name_without_instance_prefix
+    )
+
+    return transcript_name_without_instance_prefix, transcript_name_with_instance_prefix
+
+
 def upload_sjson_to_s3(config, sjson_data):
     """
     Upload sjson data to s3.
+
+    Arguments:
+        config (dict): instance configuration
+        sjson_data (list): transcript data to be uploaded to `s3`
+
+    Returns:
+        transcript name for 'edxval'
     """
     s3_conn = boto.connect_s3()
     bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
     k = Key(bucket)
     k.content_type = 'application/json'
-    k.key = '{directory}{uuid}.sjson'.format(
-        directory=config['aws_video_transcripts_prefix'],
-        uuid=uuid.uuid4().hex
-    )
+
+    transcript_name_without_instance_prefix, transcript_name_with_instance_prefix = construct_transcript_names(config)
+
+    k.key = '{}.sjson'.format(transcript_name_with_instance_prefix)
     k.set_contents_from_string(json.dumps(sjson_data))
     k.set_acl('public-read')
-    return k.key
+
+    # transcript path is stored in edxval without `instance_prefix`
+    return '{}.sjson'.format(transcript_name_without_instance_prefix)
 
 
 class ThreePlayMediaCallbackHandlerView(APIView):

bin/loop.py

@@ -7,6 +7,8 @@ from django.db import reset_queries
 import resource
 import time
 
+from control.control_env import WORK_DIRECTORY
+
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
@@ -69,14 +71,8 @@ class DaemonCli:
     def ingest_daemon(self):
         x = 0
         while True:
-            node_work_directory = os.path.join(
-                os.path.dirname(os.path.dirname(os.path.dirname(
-                    os.path.abspath(__file__)
-                ))),
-                'VEDA_WORKING'
-            )
             FD = FileDiscovery(
-                node_work_directory=node_work_directory
+                node_work_directory=WORK_DIRECTORY
             )
 
             FD.discover_studio_ingested_videos()

control/control_env.py

@@ -25,14 +25,18 @@ from VEDA_OS01.models import Destination
 from VEDA_OS01.models import Encode
 from VEDA_OS01.models import URL
 from VEDA_OS01.models import VedaUpload
+from VEDA.utils import get_config
 
 """
 Central Config
 """
-WORK_DIRECTORY = os.path.join(
+CONFIG = get_config()
+
+DEFAULT_WORK_DIRECTORY = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
     'VEDA_WORKING'
 )
+WORK_DIRECTORY = CONFIG.get('VEDA_WORKING', DEFAULT_WORK_DIRECTORY)
 
 if not os.path.exists(WORK_DIRECTORY):
     os.mkdir(WORK_DIRECTORY)

control/veda_deliver.py

@@ -25,7 +25,7 @@ from veda_deliver_youtube import DeliverYoutube
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
                               TranscriptStatus)
-from VEDA.utils import build_url, extract_course_org, get_config
+from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents
 from veda_utils import ErrorObject, Metadata, Output, VideoProto
 from veda_val import VALAPICall
 from veda_video_validation import Validation
@@ -77,8 +77,7 @@ class VedaDelivery:
 
         else:
             if os.path.exists(WORK_DIRECTORY):
-                shutil.rmtree(WORK_DIRECTORY)
-                os.mkdir(WORK_DIRECTORY)
+                delete_directory_contents(WORK_DIRECTORY)
 
             self._INFORM_INTAKE()
 

control/veda_file_discovery.py

@@ -250,7 +250,7 @@ class FileDiscovery(object):
                 connection = boto.connect_s3()
                 self.bucket = connection.get_bucket(self.auth_dict['edx_s3_ingest_bucket'])
                 for video_s3_key in self.bucket.list(self.auth_dict['edx_s3_ingest_prefix'], '/'):
-                    if video_s3_key.name != 'prod-edx/unprocessed/':
+                    if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']:
                         self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
             except S3ResponseError:
                 ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')

frontend/views.py

@@ -196,7 +196,8 @@ def upload_alpha_1(request):
             'policy': policy,
             'signature': signature,
             'abvid_serial': abvid_serial,
-            'access_key': auth_dict['veda_access_key_id']
+            'access_key': auth_dict['veda_access_key_id'],
+            'upload_bucket': auth_dict['veda_upload_bucket'],
         })
     )
     return HttpResponse(template.render(context))

instance_config.yaml

@@ -98,8 +98,8 @@ val_api_url:
 val_token_url:
 val_video_images_url:
 # Credentials
-val_client_id:
-val_secret_key:
+val_client_id: 'clientkey'
+val_secret_key: 'secretkey'
 val_password:
 val_username:
 val_transcript_create_url:
@@ -121,3 +121,6 @@ sg_script_key:
 # Endpoints
 # ---
 threeplay_ftphost:
+
+lms_base_url: ''
+instance_prefix: ''

templates/upload_video.html

@@ -21,11 +21,11 @@ var abvid_serial = "{{abvid_serial}}"
 
 </head>
 
-<body> 
+<body>
   <div id="initial_title"><h1>
       edX About Video Upload
     </h1>
-  </div>  
+  </div>
   <div id="inst_lookup">
         <h3 style="font-size: 2.0em;">File Upload Complete</h3>
         <span class="advisory" style="margin-left: 49px;">Thank you, file has been received. You can close this window<br></span>
@@ -37,7 +37,7 @@ var abvid_serial = "{{abvid_serial}}"
   </div></div>
     <div id="uploadselect">
 
-      <form class="dropzone" id="dmz" action="https://veda-uploads.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
+      <form class="dropzone" id="dmz" action="https://{{ upload_bucket }}.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
         <input type="hidden" name="key" value="upload/{{ abvid_serial }}">
         <input type="hidden" name="AWSAccessKeyId" value="{{ access_key }}">
         <input type="hidden" name="acl" value="private">
@@ -48,7 +48,7 @@ var abvid_serial = "{{abvid_serial}}"
       </form>
       <span class="advisory" style="margin-left: 30%;">Do not close or refresh this window while file is transferring<br>
         Max. 1 file at once, file must be smaller than 5GB</span>
-  </div> 
+  </div>
   <div id="new_buttons">
     <form id="reset-form" method="post" >
       <input id="rstb" class="reset_button" type="reset" value="Reset"/>

test_config.yaml

@@ -87,3 +87,4 @@ heal_end: 50
 
 global_timeout: 40
 
+instance_prefix: '127.0.0.1'