Revert "sandbox testing fixes"

VEDA/settings/production.py

@@ -17,7 +17,7 @@ LOGGING = get_logger_config(service_variant=CONFIG_DATA.get('SERVICE_VARIANT_NAM
 
 # Keep track of the names of settings that represent dicts. Instead of overriding the values in base.py,
 # the values read from disk should UPDATE the pre-configured dicts.
-DICT_UPDATE_KEYS = ('DATABASES',)
+DICT_UPDATE_KEYS = ('DATABASES', 'JWT_AUTH')
 
 # Remove the items that should be used to update dicts, and apply them separately rather
 # than pumping them into the local vars.
@@ -28,10 +28,3 @@ for key, value in dict_updates.items():
         vars()[key].update(value)
 
 vars().update(CONFIG_DATA)
-
-JWT_AUTH = {
-    'JWT_SECRET_KEY': CONFIG_DATA['val_secret_key'],
-    'JWT_ISSUER': '{}/oauth2'.format(CONFIG_DATA['lms_base_url'].rstrip('/')),
-    'JWT_AUDIENCE': CONFIG_DATA['val_client_id'],
-    'JWT_VERIFY_AUDIENCE': True,
-}

VEDA/tests/test_utils.py

 """
 Tests common utils
 """
-import glob
 import os
-import shutil
 import tempfile
 from unittest import TestCase
 
@@ -206,52 +204,3 @@ class UtilTests(TestCase):
             utils.scrub_query_params(url, params_to_scrub),
             expected_url
         )
-
-
-class DeleteDirectoryContentsTests(TestCase):
-    """
-    Tests for `delete_directory_contents` util function.
-    """
-    def setUp(self):
-        """
-        Tests setup.
-        """
-        # create a temp directory with temp directories and files in it
-        self.temp_dir = tempfile.mkdtemp()
-        dir_paths = map(lambda index: '{}/dir{}'.format(self.temp_dir, index), range(5))
-        for dir_path in dir_paths:
-            os.makedirs(dir_path)
-            __, file_path = tempfile.mkstemp(
-                suffix='.txt',
-                dir=dir_path
-            )
-            with open(file_path, 'w') as outfile:
-                outfile.write(str(TEST_CONFIG))
-
-        # create a temp file in root temp directory
-        with open('{}/{}'.format(self.temp_dir, 'temp_file.text'), 'w') as outfile:
-            outfile.write(str(TEST_CONFIG))
-
-    def tearDown(self):
-        """
-        Reverse the setup
-        """
-        shutil.rmtree(self.temp_dir)
-
-    def test_delete_directory_contents(self):
-        """
-        Tests that utils.scrub_query_params works as expected.
-        """
-        # Verify that directory is not empty
-        self.assertEqual(
-            len(glob.glob('{path}/*'.format(path=self.temp_dir))),
-            6
-        )
-
-        utils.delete_directory_contents(self.temp_dir)
-
-        # Verify that directory is empty
-        self.assertEqual(
-            glob.glob('{path}/*'.format(path=self.temp_dir)),
-            []
-        )

VEDA/utils.py

 """
 Common utils.
 """
-import glob
 import os
-import shutil
 import urllib
 import urlparse
 import yaml
@@ -41,7 +39,7 @@ def build_url(*urls, **query_params):
     Returns:
         absolute url
     """
-    url = '/'.join(item.strip('/') for item in urls if item)
+    url = '/'.join(item.strip('/') for item in urls)
     if query_params:
         url = '{}?{}'.format(url, urllib.urlencode(query_params))
 
@@ -106,21 +104,3 @@ def scrub_query_params(url, params_to_scrub):
         parsed.path,
         **new_query_params
     )
-
-
-def delete_directory_contents(path):
-    """
-    Deletes everything inside a directory. Do nothing if path is not a directory.
-
-    Arguments:
-        path (str): path to a directory.
-    """
-    if not os.path.isdir(path):
-        return
-
-    for file_path in glob.glob('{path}/*'.format(path=path.rstrip('/'))):
-        if os.path.isdir(file_path):
-            shutil.rmtree(file_path)
-
-        if os.path.isfile(file_path):
-            os.remove(file_path)

VEDA_OS01/fixtures/encodes.json

-[
-    {
-        "model": "VEDA_OS01.destination",
-        "pk": 1,
-        "fields": {
-            "destination_name": "Low Bandwidth Override (AWS S3)",
-            "destination_active": true,
-            "destination_nick": "LBO"
-        }
-    },
-    {
-        "model": "VEDA_OS01.destination",
-        "pk": 2,
-        "fields": {
-            "destination_name": "Youtube - Review",
-            "destination_active": true,
-            "destination_nick": "YTR"
-        }
-    },
-        {
-        "model": "VEDA_OS01.destination",
-        "pk": 3,
-        "fields": {
-            "destination_name": "Youtube - Primary",
-            "destination_active": true,
-            "destination_nick": "YT1"
-        }
-    },
-    {
-        "model": "VEDA_OS01.destination",
-        "pk": 4,
-        "fields": {
-            "destination_name": "Amazon AWS",
-            "destination_active": true,
-            "destination_nick": "S31"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 1,
-        "fields": {
-            "encode_destination": 4,
-            "encode_name": "Desktop - High",
-            "profile_active": true,
-            "encode_suffix": "DTH",
-            "encode_filetype": "mp4",
-            "encode_bitdepth": "27",
-            "encode_resolution": "720",
-            "product_spec": "desktop_mp4"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 2,
-        "fields": {
-            "encode_destination": 4,
-            "encode_name": "HLS",
-            "profile_active": true,
-            "encode_suffix": "HLS",
-            "encode_filetype": "HLS",
-            "encode_bitdepth": "0",
-            "encode_resolution": "0",
-            "product_spec": "hls"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 3,
-        "fields": {
-            "encode_destination": 1,
-            "encode_name": "Low Bandwidth Override",
-            "profile_active": true,
-            "encode_suffix": "LBO",
-            "encode_filetype": "mp4",
-            "encode_bitdepth": "27",
-            "encode_resolution": "360",
-            "product_spec": "override"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 4,
-        "fields": {
-            "encode_destination": 4,
-            "encode_name": "Mobile - Low",
-            "profile_active": true,
-            "encode_suffix": "MB2",
-            "encode_filetype": "mp4",
-            "encode_bitdepth": "27",
-            "encode_resolution": "360",
-            "product_spec": "mobile_low"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 5,
-        "fields": {
-            "encode_destination": 4,
-            "encode_name": "mp3",
-            "profile_active": true,
-            "encode_suffix": "AUD",
-            "encode_filetype": "mp3",
-            "encode_bitdepth": "192",
-            "encode_resolution": "0",
-            "product_spec": "audio_mp3"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 6,
-        "fields": {
-            "encode_destination": 3,
-            "encode_name": "Youtube - Primary",
-            "profile_active": true,
-            "encode_suffix": "100",
-            "encode_filetype": "mp4",
-            "encode_bitdepth": "18",
-            "encode_resolution": "1080",
-            "product_spec": "youtube"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 7,
-        "fields": {
-            "encode_destination": 2,
-            "encode_name": "Youtube - Review",
-            "profile_active": true,
-            "encode_suffix": "RVW",
-            "encode_filetype": "mp4",
-            "encode_bitdepth": "18",
-            "encode_resolution": "1080",
-            "product_spec": "review"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 8,
-        "fields": {
-            "encode_destination": 4,
-            "encode_name": "Mobile - High",
-            "profile_active": false,
-            "encode_suffix": "MB1",
-            "encode_filetype": "mp4",
-            "encode_bitdepth": "30",
-            "encode_resolution": "540",
-            "product_spec": "mobile_high"
-        }
-    },
-    {
-        "model": "VEDA_OS01.encode",
-        "pk": 9,
-        "fields": {
-            "encode_destination": 4,
-            "encode_name": "WEBM - Desktop High",
-            "profile_active": false,
-            "encode_suffix": "DTH",
-            "encode_filetype": "webm",
-            "encode_bitdepth": "1000",
-            "encode_resolution": "720",
-            "product_spec": "desktop_webm"
-        }
-    }
-]

VEDA_OS01/models.py

@@ -540,7 +540,7 @@ class Destination(models.Model):
     destination_nick = models.CharField('Nickname (3 Char.)', max_length=3, null=True, blank=True)
 
     def __unicode__(self):
-        return u'{}'.format(self.destination_name)
+        return u'%s'.format(self.destination_name) or u''
 
 
 class Encode(models.Model):

VEDA_OS01/tests/test_transcripts.py

@@ -255,6 +255,13 @@ class Cielo24TranscriptTests(APITestCase):
         video = Video.objects.get(studio_id=self.video.studio_id)
         self.assertEqual(video.transcript_status, TranscriptStatus.READY)
 
+        # verify sjson data uploaded to s3
+        bucket = conn.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket'])
+        key = Key(bucket)
+        key.key = transcript_create_request_data['name']
+        sjson = json.loads(key.get_contents_as_string())
+        self.assertEqual(sjson, TRANSCRIPT_SJSON_DATA)
+
     @patch('VEDA_OS01.transcripts.LOGGER')
     @responses.activate
     def test_fetch_exception_log(self, mock_logger):
@@ -456,8 +463,8 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         Verify sjson data uploaded to s3
         """
         key = Key(connection.get_bucket(CONFIG_DATA['aws_video_transcripts_bucket']))
-        key.key = '{transcript_name}.sjson'.format(
-            transcript_name=transcripts.construct_transcript_names(CONFIG_DATA)[1]
+        key.key = '{directory}{uuid}.sjson'.format(
+            directory=CONFIG_DATA['aws_video_transcripts_prefix'], uuid=self.uuid_hex
         )
         sjson_transcript = json.loads(key.get_contents_as_string())
         self.assertEqual(sjson_transcript, TRANSCRIPT_SJSON_DATA)
@@ -1558,26 +1565,3 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             self.edx_video_id,
             self.file_id,
         )
-
-
-class TranscriptNameConstructionTests(APITestCase):
-    """
-    Tests for `construct_transcript_names` util function
-    """
-    def setUp(self):
-        """
-        Tests setup.
-        """
-        super(TranscriptNameConstructionTests, self).setUp()
-
-    def test_upload_sjson_to_s3(self):
-        """
-        Verify that `construct_transcript_names` works as expected.
-        """
-        edxval_name, s3_name = transcripts.construct_transcript_names(CONFIG_DATA)
-        self.assertTrue(
-            s3_name.startswith(CONFIG_DATA['instance_prefix'])
-        )
-        self.assertTrue(
-            s3_name.endswith(edxval_name)
-        )

VEDA_OS01/transcripts.py

@@ -314,53 +314,21 @@ def convert_srt_to_sjson(srt_data):
     return subs
 
 
-def construct_transcript_names(config):
-    """
-    Constructs transcript names for 'edxval' and 's3'
-
-    Arguments:
-        config (dict): instance configuration
-
-    Returns:
-        transcript names for 'edxval' and 's3'
-    """
-    transcript_name_without_instance_prefix = build_url(
-        config['aws_video_transcripts_prefix'],
-        uuid.uuid4().hex
-    )
-
-    transcript_name_with_instance_prefix = build_url(
-        config['instance_prefix'],
-        transcript_name_without_instance_prefix
-    )
-
-    return transcript_name_without_instance_prefix, transcript_name_with_instance_prefix
-
-
 def upload_sjson_to_s3(config, sjson_data):
     """
     Upload sjson data to s3.
-
-    Arguments:
-        config (dict): instance configuration
-        sjson_data (list): transcript data to be uploaded to `s3`
-
-    Returns:
-        transcript name for 'edxval'
     """
     s3_conn = boto.connect_s3()
     bucket = s3_conn.get_bucket(config['aws_video_transcripts_bucket'])
     k = Key(bucket)
     k.content_type = 'application/json'
-
-    transcript_name_without_instance_prefix, transcript_name_with_instance_prefix = construct_transcript_names(config)
-
-    k.key = '{}.sjson'.format(transcript_name_with_instance_prefix)
+    k.key = '{directory}{uuid}.sjson'.format(
+        directory=config['aws_video_transcripts_prefix'],
+        uuid=uuid.uuid4().hex
+    )
     k.set_contents_from_string(json.dumps(sjson_data))
     k.set_acl('public-read')
-
-    # transcript path is stored in edxval without `instance_prefix`
-    return '{}.sjson'.format(transcript_name_without_instance_prefix)
+    return k.key
 
 
 class ThreePlayMediaCallbackHandlerView(APIView):

bin/loop.py

@@ -7,8 +7,6 @@ from django.db import reset_queries
 import resource
 import time
 
-from control.control_env import WORK_DIRECTORY
-
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
     sys.path.append(project_path)
@@ -71,8 +69,14 @@ class DaemonCli:
     def ingest_daemon(self):
         x = 0
         while True:
+            node_work_directory = os.path.join(
+                os.path.dirname(os.path.dirname(os.path.dirname(
+                    os.path.abspath(__file__)
+                ))),
+                'VEDA_WORKING'
+            )
             FD = FileDiscovery(
-                node_work_directory=WORK_DIRECTORY
+                node_work_directory=node_work_directory
             )
 
             FD.discover_studio_ingested_videos()

control/control_env.py

@@ -25,18 +25,14 @@ from VEDA_OS01.models import Destination
 from VEDA_OS01.models import Encode
 from VEDA_OS01.models import URL
 from VEDA_OS01.models import VedaUpload
-from VEDA.utils import get_config
 
 """
 Central Config
 """
-CONFIG = get_config()
-
-DEFAULT_WORK_DIRECTORY = os.path.join(
+WORK_DIRECTORY = os.path.join(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
     'VEDA_WORKING'
 )
-WORK_DIRECTORY = CONFIG.get('VEDA_WORKING', DEFAULT_WORK_DIRECTORY)
 
 if not os.path.exists(WORK_DIRECTORY):
     os.mkdir(WORK_DIRECTORY)

control/veda_deliver.py

@@ -25,7 +25,7 @@ from veda_deliver_youtube import DeliverYoutube
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
                               TranscriptStatus)
-from VEDA.utils import build_url, extract_course_org, get_config, delete_directory_contents
+from VEDA.utils import build_url, extract_course_org, get_config
 from veda_utils import ErrorObject, Metadata, Output, VideoProto
 from veda_val import VALAPICall
 from veda_video_validation import Validation
@@ -77,7 +77,8 @@ class VedaDelivery:
 
         else:
             if os.path.exists(WORK_DIRECTORY):
-                delete_directory_contents(WORK_DIRECTORY)
+                shutil.rmtree(WORK_DIRECTORY)
+                os.mkdir(WORK_DIRECTORY)
 
             self._INFORM_INTAKE()
 

control/veda_file_discovery.py

@@ -250,7 +250,7 @@ class FileDiscovery(object):
                 connection = boto.connect_s3()
                 self.bucket = connection.get_bucket(self.auth_dict['edx_s3_ingest_bucket'])
                 for video_s3_key in self.bucket.list(self.auth_dict['edx_s3_ingest_prefix'], '/'):
-                    if video_s3_key.name != self.auth_dict['edx_s3_ingest_prefix']:
+                    if video_s3_key.name != 'prod-edx/unprocessed/':
                         self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
             except S3ResponseError:
                 ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')

frontend/views.py

@@ -196,8 +196,7 @@ def upload_alpha_1(request):
             'policy': policy,
             'signature': signature,
             'abvid_serial': abvid_serial,
-            'access_key': auth_dict['veda_access_key_id'],
-            'upload_bucket': auth_dict['veda_upload_bucket'],
+            'access_key': auth_dict['veda_access_key_id']
         })
     )
     return HttpResponse(template.render(context))

instance_config.yaml

@@ -98,8 +98,8 @@ val_api_url:
 val_token_url:
 val_video_images_url:
 # Credentials
-val_client_id: 'clientkey'
-val_secret_key: 'secretkey'
+val_client_id:
+val_secret_key:
 val_password:
 val_username:
 val_transcript_create_url:
@@ -121,6 +121,3 @@ sg_script_key:
 # Endpoints
 # ---
 threeplay_ftphost:
-
-lms_base_url: ''
-instance_prefix: ''

templates/upload_video.html

@@ -37,7 +37,7 @@ var abvid_serial = "{{abvid_serial}}"
   </div></div>
     <div id="uploadselect">
 
-      <form class="dropzone" id="dmz" action="https://{{ upload_bucket }}.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
+      <form class="dropzone" id="dmz" action="https://veda-uploads.s3.amazonaws.com/" method="put" enctype="multipart/form-data" >
         <input type="hidden" name="key" value="upload/{{ abvid_serial }}">
         <input type="hidden" name="AWSAccessKeyId" value="{{ access_key }}">
         <input type="hidden" name="acl" value="private">

test_config.yaml

@@ -87,4 +87,3 @@ heal_end: 50
 
 global_timeout: 40
 
-instance_prefix: '127.0.0.1'