refactor config setting file read

VEDA/django_secrets.py

@@ -5,25 +5,16 @@ This acts as a django-secret shimmer until we can finish pushing all changes to
 
 """
 import os
-import yaml
+from VEDA.utils import get_config
 
-read_yaml = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-    'instance_config.yaml'
-)
-
-with open(read_yaml, 'r') as stream:
-    try:
-        return_dict = yaml.load(stream)
-    except yaml.YAMLError as exc:
-        return_dict = None
+CONFIG_DATA = get_config()
 
 
-DJANGO_SECRET_KEY = return_dict['django_secret_key'] or 'test_secret_key'
+DJANGO_SECRET_KEY = CONFIG_DATA['django_secret_key'] or 'test_secret_key'
 DJANGO_ADMIN = ('', '')
-DJANGO_DEBUG = return_dict['debug'] if 'debug' in return_dict else False
-DATABASES = return_dict['DATABASES']
-STATIC_ROOT_PATH = return_dict.get(
+DJANGO_DEBUG = CONFIG_DATA['debug'] if 'debug' in CONFIG_DATA else False
+DATABASES = CONFIG_DATA['DATABASES']
+STATIC_ROOT_PATH = CONFIG_DATA.get(
     'STATIC_ROOT_PATH',
     os.path.join(
         os.path.dirname(os.path.dirname(os.path.abspath(__file__))),

VEDA/tests/test_utils.py

+"""
+Tests common utils
+"""
+import os
+import tempfile
+from unittest import TestCase
+
+import yaml
+from ddt import data, ddt, unpack
+from mock import MagicMock, Mock, patch
+
+from VEDA import utils
+
+TEST_CONFIG = {
+    'var1': 123,
+    'var2': 999,
+    'sub': {
+        'sub_var': 'http://example.com'
+    }
+}
+
+
+@ddt
+class UtilTests(TestCase):
+    """
+    Common util tests.
+    """
+    def setUp(self):
+        """
+        Tests setup.
+        """
+        self._orig_environ = dict(os.environ)
+        _, self.file_path = tempfile.mkstemp(
+            suffix='.yml',
+            dir=tempfile.tempdir
+        )
+        with open(self.file_path, 'w') as outfile:
+            yaml.dump(TEST_CONFIG, outfile, default_flow_style=False)
+
+        os.environ['VIDEO_PIPELINE_CFG'] = self.file_path
+
+    def tearDown(self):
+        """
+        Reverse the setup
+        """
+        # Reset Environment back to original state
+        os.environ.clear()
+        os.environ.update(self._orig_environ)
+
+    @data(
+        {
+            'urls': ('http://api.cielo24/', '/add/job'),
+            'params': {},
+            'expected_url': 'http://api.cielo24/add/job'
+        },
+        {
+            'urls': ('http://api.cielo24', '/add/job'),
+            'params': {'a': 1, 'b': 2},
+            'expected_url': 'http://api.cielo24/add/job?a=1&b=2'
+        },
+        {
+            'urls': ('http://api.cielo24/', 'add/job'),
+            'params': {'c': 3, 'd': 4},
+            'expected_url': 'http://api.cielo24/add/job?c=3&d=4'
+        },
+        {
+            'urls': ('http://api.cielo24','add/job'),
+            'params': {'p': 100},
+            'expected_url': 'http://api.cielo24/add/job?p=100'
+        },
+        {
+            'urls': ('http://api.cielo24', 'add/job', 'media'),
+            'params': {'p': 100},
+            'expected_url': 'http://api.cielo24/add/job/media?p=100'
+        }
+    )
+    @unpack
+    def test_build_url(self, urls, params, expected_url):
+        """
+        Tests that utils.build_url works as expected.
+        """
+        url = utils.build_url(
+            *urls,
+            **params
+        )
+        self.assertEqual(
+            url,
+            expected_url
+        )
+
+    @data(
+        {
+            'course_id': 'course-v1:MITx+4.605x+3T2017',
+            'expected_org': 'MITx'
+        },
+        {
+            'course_id': 'WestonHS/PFLC1x/3T2015',
+            'expected_org': 'WestonHS'
+        },
+        {
+            'course_id': '',
+            'expected_org': None
+        },
+
+    )
+    @unpack
+    def test_extract_course_org(self, course_id, expected_org):
+        """
+        Tests that utils.extract_course_org works as expected.
+        """
+        org = utils.extract_course_org(course_id)
+        self.assertEqual(
+            org,
+            expected_org
+        )
+
+    def test_get_config_does_not_exist(self):
+        """
+        Tests that utils.get_config if file does not exist.
+        """
+        del os.environ['VIDEO_PIPELINE_CFG']
+
+        with self.assertRaises(IOError):
+            utils.get_config(yaml_config_file='does_not_exist')
+
+    def test_get_config_with_default(self):
+        """
+        Tests that utils.get_config works as expected when reading default config.
+        """
+        del os.environ['VIDEO_PIPELINE_CFG']
+
+        instance_config = utils.get_config()
+        self.assertNotEqual(instance_config, {})
+
+        # read the default config file to verify that correct config is loaded
+        default_yaml_config_file = os.path.join(
+            utils.DEFAULT_CONFIG_FILE_PATH,
+            utils.DEFAULT_CONFIG_FILE_NAME
+        )
+        with open(default_yaml_config_file, 'r') as config:
+            config_dict = yaml.load(config)
+            self.assertDictEqual(instance_config, config_dict)
+
+    def test_get_config_with_path(self):
+        """
+        Tests that utils.get_config works as expected when reading config from environment path.
+        """
+        instance_config = utils.get_config()
+        self.assertDictEqual(instance_config, TEST_CONFIG)

VEDA/utils.py

+"""
+Common utils.
+"""
+import os
+import yaml
+import urllib
+
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+
+DEFAULT_CONFIG_FILE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DEFAULT_CONFIG_FILE_NAME = 'instance_config.yaml'
+
+
+def extract_course_org(course_id):
+    """
+    Extract video organization from course url.
+    """
+    org = None
+
+    try:
+        org = CourseKey.from_string(course_id).org
+    except InvalidKeyError:
+        pass
+
+    return org
+
+
+def build_url(*urls, **query_params):
+    """
+    Build a url from specified params.
+
+    Arguments:
+        base_url (str): base url
+        relative_url (str): endpoint
+        query_params (dict): query params
+
+    Returns:
+        absolute url
+    """
+    url = '/'.join(item.strip('/') for item in urls)
+    if query_params:
+        url = '{}?{}'.format(url, urllib.urlencode(query_params))
+
+    return url
+
+
+def get_config(yaml_config_file=DEFAULT_CONFIG_FILE_NAME):
+    """
+    Read yaml config file.
+
+    Arguments:
+        yaml_config_file (str): yaml config file name
+
+    Returns:
+        dict: yaml config
+    """
+    config_dict = {}
+
+    try:
+        yaml_config_file = os.environ['VIDEO_PIPELINE_CFG']
+    except KeyError:
+        yaml_config_file = os.path.join(
+            DEFAULT_CONFIG_FILE_PATH,
+            yaml_config_file
+        )
+
+    with open(yaml_config_file, 'r') as config:
+        config_dict = yaml.load(config)
+
+    return config_dict

VEDA_OS01/tests/test_transcripts.py

@@ -17,12 +17,13 @@ from moto import mock_s3_deprecated
 from rest_framework import status
 from rest_framework.test import APITestCase
 
+from VEDA.utils import get_config, build_url
 from VEDA_OS01 import transcripts, utils
 from VEDA_OS01.models import (Course, TranscriptCredentials,
                               TranscriptProcessMetadata, TranscriptProvider,
                               TranscriptStatus, Video)
 
-CONFIG_DATA = utils.get_config('test_config.yaml')
+CONFIG_DATA = get_config('test_config.yaml')
 
 VIDEO_DATA = {
     'studio_id': '12345',
@@ -87,7 +88,7 @@ TRANSCRIPT_SJSON_DATA = {
 
 @ddt
 @patch.dict('VEDA_OS01.transcripts.CONFIG', CONFIG_DATA)
-@patch('VEDA_OS01.utils.get_config', Mock(return_value=CONFIG_DATA))
+@patch('VEDA.utils.get_config', Mock(return_value=CONFIG_DATA))
 class Cielo24TranscriptTests(APITestCase):
     """
     Cielo24 Transcript Tests
@@ -401,7 +402,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         """
         response = self.client.post(
             # `build_url` strips `/`, putting it back and add necessary query params.
-            '/{}'.format(utils.build_url(
+            '/{}'.format(build_url(
                 self.url, edx_video_id=self.video.studio_id,
                 org=self.org, lang_code=self.video_source_language
             )),
@@ -483,7 +484,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         Test the callback in case of missing attributes.
         """
         response = self.client.post(
-            '/{}'.format(utils.build_url(self.url, **request_data['query_params'])),
+            '/{}'.format(build_url(self.url, **request_data['query_params'])),
             content_type='application/x-www-form-urlencoded',
             data=urllib.urlencode(request_data['data']),
         )
@@ -519,7 +520,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         """
         Tests the callback for all the non-success statuses.
         """
-        self.url = '/{}'.format(utils.build_url(
+        self.url = '/{}'.format(build_url(
             self.url, edx_video_id='12345', org='MAx', lang_code=self.video_source_language
         ))
         self.client.post(self.url, content_type='application/x-www-form-urlencoded', data=urllib.urlencode({
@@ -579,7 +580,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         expected_requests = [
             # request - 1
             {
-                'url': utils.build_url(
+                'url': build_url(
                     transcripts.THREE_PLAY_TRANSCRIPT_URL.format(file_id=self.file_id),
                     apikey=self.transcript_prefs.api_key
                 )
@@ -742,7 +743,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
         expected_requests = [
             # request - 1
             {
-                'url': utils.build_url(
+                'url': build_url(
                     transcripts.THREE_PLAY_TRANSCRIPT_URL.format(file_id=self.file_id),
                     apikey=self.transcript_prefs.api_key
                 )
@@ -779,7 +780,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             },
             # request - 4
             {
-                'url': utils.build_url(
+                'url': build_url(
                     transcripts.THREE_PLAY_TRANSLATION_SERVICES_URL,
                     apikey=self.transcript_prefs.api_key
                 )
@@ -1165,7 +1166,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
 
         # Assert that the first request was made for getting translations metadata from 3Play Media.
         expected_video_status_update_request = {
-            'url': utils.build_url(
+            'url': build_url(
                 transcripts.THREE_PLAY_TRANSLATIONS_METADATA_URL.format(file_id=self.file_id),
                 apikey=self.transcript_prefs.api_key
             )
@@ -1180,7 +1181,7 @@ class ThreePlayTranscriptionCallbackTest(APITestCase):
             expected_requests = [
                 # request - 1
                 {
-                    'url': utils.build_url(transcripts.THREE_PLAY_TRANSLATION_DOWNLOAD_URL.format(
+                    'url': build_url(transcripts.THREE_PLAY_TRANSLATION_DOWNLOAD_URL.format(
                         file_id=self.file_id, translation_id=translation_id
                     ), apikey=self.transcript_prefs.api_key)
                 },

VEDA_OS01/tests/test_utils.py

@@ -14,80 +14,6 @@ class UtilTests(TestCase):
     Common util tests.
     """
     @data(
-        {
-            'urls': ('http://api.cielo24/', '/add/job'),
-            'params': {},
-            'expected_url': 'http://api.cielo24/add/job'
-        },
-        {
-            'urls': ('http://api.cielo24', '/add/job'),
-            'params': {'a': 1, 'b': 2},
-            'expected_url': 'http://api.cielo24/add/job?a=1&b=2'
-        },
-        {
-            'urls': ('http://api.cielo24/', 'add/job'),
-            'params': {'c': 3, 'd': 4},
-            'expected_url': 'http://api.cielo24/add/job?c=3&d=4'
-        },
-        {
-            'urls': ('http://api.cielo24','add/job'),
-            'params': {'p': 100},
-            'expected_url': 'http://api.cielo24/add/job?p=100'
-        },
-        {
-            'urls': ('http://api.cielo24', 'add/job', 'media'),
-            'params': {'p': 100},
-            'expected_url': 'http://api.cielo24/add/job/media?p=100'
-        }
-    )
-    @unpack
-    def test_build_url(self, urls, params, expected_url):
-        """
-        Tests that utils.build_url works as expected.
-        """
-        url = utils.build_url(
-            *urls,
-            **params
-        )
-        self.assertEqual(
-            url,
-            expected_url
-        )
-
-    @data(
-        {
-            'course_id': 'course-v1:MITx+4.605x+3T2017',
-            'expected_org': 'MITx'
-        },
-        {
-            'course_id': 'WestonHS/PFLC1x/3T2015',
-            'expected_org': 'WestonHS'
-        },
-        {
-            'course_id': '',
-            'expected_org': None
-        },
-
-    )
-    @unpack
-    def test_extract_course_org(self, course_id, expected_org):
-        """
-        Tests that utils.extract_course_org works as expected.
-        """
-        org = utils.extract_course_org(course_id)
-        self.assertEqual(
-            org,
-            expected_org
-        )
-
-    def test_get_config(self):
-        """
-        Tests that utils.get_config works as expected.
-        """
-        config = utils.get_config()
-        self.assertNotEqual(config, {})
-
-    @data(
         ('IN PROGRESS', True),
         ('FAILED', False)
     )

VEDA_OS01/transcripts.py

@@ -19,6 +19,7 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from control.veda_val import VALAPICall
+from VEDA.utils import get_config, build_url
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProcessMetadata,
                               TranscriptProvider, TranscriptStatus, Video)
@@ -37,10 +38,10 @@ TRANSCRIPT_SJSON = 'sjson'
 CIELO24_TRANSCRIPT_COMPLETED = django.dispatch.Signal(providing_args=[
     'job_id', 'iwp_name', 'lang_code', 'org', 'video_id'
 ])
-CONFIG = utils.get_config()
+CONFIG = get_config()
 
 # Cielo24 API URLs
-CIELO24_GET_CAPTION_URL = utils.build_url(
+CIELO24_GET_CAPTION_URL = build_url(
     CONFIG['cielo24_api_base_url'],
     'job/get_caption'
 )
@@ -50,23 +51,23 @@ THREE_PLAY_TRANSCRIPTION_DONE = django.dispatch.Signal(
     providing_args=['org', 'lang_code', 'edx_video_id', 'file_id', 'status', 'error_description']
 )
 # 3PlayMedia API URLs.
-THREE_PLAY_TRANSCRIPT_URL = utils.build_url(
+THREE_PLAY_TRANSCRIPT_URL = build_url(
     CONFIG['three_play_api_transcript_url'],
     'files/{file_id}/transcript.srt'
 )
-THREE_PLAY_TRANSLATION_SERVICES_URL = utils.build_url(
+THREE_PLAY_TRANSLATION_SERVICES_URL = build_url(
     CONFIG['three_play_api_transcript_url'],
     'translation_services'
 )
-THREE_PLAY_ORDER_TRANSLATION_URL = utils.build_url(
+THREE_PLAY_ORDER_TRANSLATION_URL = build_url(
     CONFIG['three_play_api_base_url'],
     'files/{file_id}/translations/order'
 )
-THREE_PLAY_TRANSLATIONS_METADATA_URL = utils.build_url(
+THREE_PLAY_TRANSLATIONS_METADATA_URL = build_url(
     CONFIG['three_play_api_transcript_url'],
     'files/{file_id}/translations'
 )
-THREE_PLAY_TRANSLATION_DOWNLOAD_URL = utils.build_url(
+THREE_PLAY_TRANSLATION_DOWNLOAD_URL = build_url(
     CONFIG['three_play_api_transcript_url'],
     'files/{file_id}/translations/{translation_id}/captions.srt'
 )
@@ -267,7 +268,7 @@ def fetch_srt_data(url, **request_params):
     """
     # return TRANSCRIPT_SRT_DATA
     response = requests.get(
-        utils.build_url(url, **request_params)
+        build_url(url, **request_params)
     )
 
     if not response.ok:
@@ -376,7 +377,7 @@ def get_translation_services(api_key):
     Returns:
         Available 3Play Media Translation services.
     """
-    response = requests.get(utils.build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key))
+    response = requests.get(build_url(THREE_PLAY_TRANSLATION_SERVICES_URL, apikey=api_key))
     if not response.ok:
         raise TranscriptTranslationError(
             u'[3PlayMedia Callback] Error while fetching the translation services -- {status}, {response}'.format(
@@ -824,7 +825,7 @@ def get_translations_metadata(api_key, file_id, edx_video_id):
             }
         ]
     """
-    translations_metadata_url = utils.build_url(
+    translations_metadata_url = build_url(
         THREE_PLAY_TRANSLATIONS_METADATA_URL.format(
             file_id=file_id,
         ),

VEDA_OS01/utils.py

 """
 Common utils.
 """
-import os
-import urllib
-
-import yaml
-from opaque_keys import InvalidKeyError
-from opaque_keys.edx.keys import CourseKey
 
 from VEDA_OS01.models import TranscriptStatus
 
@@ -26,65 +20,6 @@ VAL_TRANSCRIPT_STATUS_MAP = {
 }
 
 
-def get_config(yaml_config_file='instance_config.yaml'):
-    """
-    Read yaml config file.
-
-    Arguments:
-        yaml_config_file (str): yaml config file name
-
-    Returns:
-        dict: yaml conifg
-    """
-    config_dict = {}
-
-    yaml_config_file = os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        yaml_config_file
-    )
-
-    with open(yaml_config_file, 'r') as config:
-        try:
-            config_dict = yaml.load(config)
-        except yaml.YAMLError:
-            pass
-
-    return config_dict
-
-
-def extract_course_org(course_id):
-    """
-    Extract video organization from course url.
-    """
-    org = None
-
-    try:
-        org = CourseKey.from_string(course_id).org
-    except InvalidKeyError:
-        pass
-
-    return org
-
-
-def build_url(*urls, **query_params):
-    """
-    Build a url from specified params.
-
-    Arguments:
-        base_url (str): base url
-        relative_url (str): endpoint
-        query_params (dict): query params
-
-    Returns:
-        absolute url
-    """
-    url = '/'.join(item.strip('/') for item in urls)
-    if query_params:
-        url = '{}?{}'.format(url, urllib.urlencode(query_params))
-
-    return url
-
-
 def update_video_status(val_api_client, video, status):
     """
     Updates video status both in edx-val and edx-video-pipeline.

bin/deliver

@@ -2,7 +2,7 @@
 import os
 import sys
 import argparse
-import yaml
+from VEDA.utils import get_config
 
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
@@ -24,7 +24,6 @@ class DeliverCli:
         self.ROOT_DIR = os.path.dirname(os.path.dirname(
             os.path.abspath(__file__)
         ))
-        self.auth_yaml = os.path.join(self.ROOT_DIR, 'instance_config.yaml')
         self.celery_daemon = os.path.join(self.ROOT_DIR, 'control', 'celeryapp.py')
 
     def get_args(self):
@@ -52,11 +51,7 @@ class DeliverCli:
         Launch Celery Delivery Worker
 
         """
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-            except yaml.YAMLError as exc:
-                auth_dict = None
+        auth_dict = get_config()
 
         if auth_dict is not None:
             os.system(

control/celeryapp.py

@@ -6,7 +6,7 @@ from __future__ import absolute_import
 
 import os
 from celery import Celery
-import yaml
+from VEDA.utils import get_config
 
 try:
     from control.veda_deliver import VedaDelivery
@@ -14,15 +14,7 @@ except ImportError:
     from veda_deliver import VedaDelivery
 
 
-auth_yaml = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-    'instance_config.yaml'
-)
-with open(auth_yaml, 'r') as stream:
-    try:
-        auth_dict = yaml.load(stream)
-    except yaml.YAMLError as exc:
-        auth_dict = None
+auth_dict = get_config()
 
 CEL_BROKER = 'amqp://{rabbitmq_user}:{rabbitmq_pass}@{rabbitmq_broker}:5672//'.format(
     rabbitmq_user=auth_dict['rabbitmq_user'],

control/tests/test_deliver_3play.py

@@ -19,7 +19,7 @@ from VEDA_OS01.models import (
     Video,
     ThreePlayTurnaround,
 )
-from VEDA_OS01.utils import build_url
+from VEDA.utils import build_url
 
 VIDEO_DATA = {
     'studio_id': '12345',

control/tests/test_deliver_cielo.py

@@ -11,7 +11,7 @@ from control.veda_deliver_cielo import Cielo24Transcript
 from VEDA_OS01.models import (Cielo24Fidelity, Cielo24Turnaround, Course,
                               TranscriptProcessMetadata, TranscriptStatus,
                               Video)
-from VEDA_OS01.utils import build_url
+from VEDA.utils import build_url
 
 VIDEO_DATA = {
     'studio_id': '12345',

control/tests/test_file_ingest.py

@@ -6,7 +6,6 @@ import unittest
 
 from django.test import TestCase
 import requests
-import yaml
 
 from veda_file_ingest import VedaIngest, VideoProto
 

control/tests/test_heal.py

@@ -14,7 +14,8 @@ from mock import PropertyMock, patch
 
 from control.veda_heal import VedaHeal
 from VEDA_OS01.models import URL, Course, Destination, Encode, Video, TranscriptStatus
-from VEDA_OS01.utils import build_url, get_config, ValTranscriptStatus
+from VEDA_OS01.utils import ValTranscriptStatus
+from VEDA.utils import build_url, get_config
 
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.abspath(__file__)

control/tests/test_hotstore.py

@@ -9,7 +9,7 @@ from django.test import TestCase
 from boto.s3.connection import S3Connection
 from mock import PropertyMock, patch
 from moto import mock_s3_deprecated
-from VEDA_OS01 import utils
+from VEDA import utils
 
 from control.veda_file_ingest import VideoProto
 from control.veda_hotstore import Hotstore

control/tests/test_val.py

@@ -11,7 +11,7 @@ import responses
 
 from control.veda_val import VALAPICall
 from veda_file_ingest import VideoProto
-from VEDA_OS01 import utils
+from VEDA import utils
 
 
 requests.packages.urllib3.disable_warnings()

control/veda_deliver.py

@@ -7,7 +7,6 @@ from os.path import expanduser
 import boto
 import boto.s3
 import requests
-import yaml
 from boto.exception import S3ResponseError
 from boto.s3.key import Key
 from django.core.urlresolvers import reverse
@@ -19,7 +18,7 @@ from veda_deliver_youtube import DeliverYoutube
 from VEDA_OS01 import utils
 from VEDA_OS01.models import (TranscriptCredentials, TranscriptProvider,
                               TranscriptStatus)
-from VEDA_OS01.utils import build_url
+from VEDA.utils import build_url, extract_course_org, get_config
 from veda_utils import ErrorObject, Metadata, Output, VideoProto
 from veda_val import VALAPICall
 from veda_video_validation import Validation
@@ -60,14 +59,7 @@ class VedaDelivery:
         self.veda_id = veda_id
         self.encode_profile = encode_profile
 
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            ),
-        )
-        self.auth_dict = self._READ_YAML(self.auth_yaml)
+        self.auth_dict = get_config()
         # Internal Methods
         self.video_query = None
         self.encode_query = None
@@ -78,19 +70,6 @@ class VedaDelivery:
         self.endpoint_url = None
         self.video_proto = None
 
-    def _READ_YAML(self, read_yaml):
-        if read_yaml is None:
-            return None
-        if not os.path.exists(read_yaml):
-            return None
-
-        with open(read_yaml, 'r') as stream:
-            try:
-                return_dict = yaml.load(stream)
-                return return_dict
-            except yaml.YAMLError as exc:
-                return None
-
     def run(self):
         """
         Check the destination, route via available methods,
@@ -536,7 +515,7 @@ class VedaDelivery:
         """
         Cielo24 transcription flow.
         """
-        org = utils.extract_course_org(self.video_proto.platform_course_url[0])
+        org = extract_course_org(self.video_proto.platform_course_url[0])
 
         try:
             api_key = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider).api_key
@@ -586,7 +565,7 @@ class VedaDelivery:
         try:
             # Picks the first course from the list as there may be multiple
             # course runs in that list (i.e. all having the same org).
-            org = utils.extract_course_org(self.video_proto.platform_course_url[0])
+            org = extract_course_org(self.video_proto.platform_course_url[0])
             transcript_secrets = TranscriptCredentials.objects.get(org=org, provider=self.video_query.provider)
 
             # update transcript status for video.

control/veda_deliver_3play.py

@@ -8,7 +8,7 @@ import sys
 
 from requests.packages.urllib3.exceptions import InsecurePlatformWarning
 from VEDA_OS01.models import TranscriptProcessMetadata, TranscriptProvider, TranscriptStatus
-from VEDA_OS01.utils import build_url
+from VEDA.utils import build_url
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 

control/veda_deliver_cielo.py

@@ -10,7 +10,7 @@ from requests.packages.urllib3.exceptions import InsecurePlatformWarning
 
 from VEDA_OS01.models import (TranscriptProcessMetadata, TranscriptProvider,
                               TranscriptStatus)
-from VEDA_OS01.utils import build_url
+from VEDA.utils import build_url
 
 requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
 

control/veda_deliver_xuetang.py

@@ -8,7 +8,7 @@ import requests
 import json
 import time
 from time import strftime
-import yaml
+from VEDA.utils import get_config
 
 """
 Authored by Ed Zarecor / edx DevOps
@@ -18,20 +18,13 @@ included by request
 Some adaptations for VEDA:
     -auth yaml
 
-   **VEDA Note: since this isn't a real CDN, and represents the 
-    'least effort' response to getting video into china, 
+   **VEDA Note: since this isn't a real CDN, and represents the
+    'least effort' response to getting video into china,
     we shan't monitor for success**
 
 
 """
-read_yaml = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-    'instance_config.yaml'
-    )
-
-if os.path.exists(read_yaml):
-    with open(read_yaml, 'r') as stream:
-        auth_dict = yaml.load(stream)
+auth_dict = get_config()
 
 API_SHARED_SECRET = auth_dict['xuetang_api_shared_secret']
 API_ENDPOINT = auth_dict['xuetang_api_url']
@@ -48,7 +41,7 @@ This script provides a functions for accessing the Xuetang CDN API
 It expects that an environment variable name XUETANG_SHARED_SECRET is
 available and refers to a valid secret provided by the Xuetang CDN team.
 
-Running this script will cause a video hosted in cloudfront to be 
+Running this script will cause a video hosted in cloudfront to be
 uploaded to the CDN via the API.
 
 The status of the video will be monitored in a loop, exiting when
@@ -77,10 +70,10 @@ def build_message(verb, uri, query_string, date, payload_hash):
     format is defined in their CDN API specification document.
     """
     return os.linesep.join([verb, uri, query_string, date, payload_hash])
-    
+
 def sign_message(message, secret):
     """
-    Returns a hexdigest of HMAC generated using sha256.  The value is included in 
+    Returns a hexdigest of HMAC generated using sha256.  The value is included in
     the HTTP headers and used for mutual authentication via a shared secret.
     """
     return hmac.new(bytes(secret), bytes(message), digestmod=hashlib.sha256).hexdigest()
@@ -125,7 +118,7 @@ def prepare_check_task_status(edx_url):
 
 def _prepare_api_request(http_verb, api_target, payload):
     """
-    General convenience function for creating prepared HTTP requests that conform the 
+    General convenience function for creating prepared HTTP requests that conform the
     Xuetang API specificiation.
     """
     payload_json = json.dumps(payload)
@@ -188,7 +181,7 @@ if __name__ == '__main__':
 
     #     if res.json()['status'] == 'available':
     #         break
-        
+
     #     time.sleep(5)
 
     # delete file

control/veda_encode.py

@@ -4,11 +4,10 @@ import sys
 import uuid
 
 import django
-import yaml
 
 from control_env import *
 from dependencies.shotgun_api3 import Shotgun
-
+from VEDA.utils import get_config
 
 """
 Get a list of needed encodes from VEDA
@@ -31,30 +30,11 @@ class VedaEncode(object):
         self.overencode = kwargs.get('overencode', False)
         self.veda_id = kwargs.get('veda_id', None)
 
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            ),
-        )
-        self.encode_dict = self._READ_AUTH()['encode_dict']
-        self.sg_server_path = self._READ_AUTH()['sg_server_path']
-        self.sg_script_name = self._READ_AUTH()['sg_script_name']
-        self.sg_script_key = self._READ_AUTH()['sg_script_key']
-
-    def _READ_AUTH(self):
-        if self.auth_yaml is None:
-            return None
-        if not os.path.exists(self.auth_yaml):
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                return None
+        config_data = get_config()
+        self.encode_dict = config_data['encode_dict']
+        self.sg_server_path = config_data['sg_server_path']
+        self.sg_script_name = config_data['sg_script_name']
+        self.sg_script_key = config_data['sg_script_key']
 
     def determine_encodes(self):
         """

control/veda_file_discovery.py

@@ -5,10 +5,9 @@ import os.path
 import boto
 import boto.s3
 from boto.exception import S3ResponseError, S3DataError
-import yaml
 
 from VEDA_OS01.models import TranscriptCredentials
-from VEDA_OS01.utils import extract_course_org
+from VEDA.utils import extract_course_org, get_config
 
 try:
     boto.config.add_section('Boto')
@@ -39,16 +38,7 @@ class FileDiscovery(object):
     def __init__(self, **kwargs):
         self.video_info = {}
 
-        self.auth_dict = {}
-        self.auth_yaml = os.path.join(
-            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-            'instance_config.yaml'
-        )
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                self.auth_dict = yaml.load(stream)
-            except yaml.YAMLError as exc:
-                pass
+        self.auth_dict = get_config()
 
         self.bucket = None
         """

control/veda_file_ingest.py

@@ -11,8 +11,8 @@ from django.db.utils import DatabaseError
 from django.utils.timezone import utc
 from django.db import reset_queries
 import uuid
-import yaml
 import hashlib
+from VEDA.utils import get_config
 
 
 """
@@ -89,14 +89,7 @@ class VedaIngest:
     def __init__(self, course_object, video_proto, **kwargs):
         self.course_object = course_object
         self.video_proto = video_proto
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            ),
-        )
-        self.auth_dict = self._READ_AUTH()
+        self.auth_dict = get_config()
 
         # --- #
         self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
@@ -104,19 +97,6 @@ class VedaIngest:
         self.complete = False
         self.archived = False
 
-    def _READ_AUTH(self):
-        if self.auth_yaml is None:
-            return None
-        if not os.path.exists(self.auth_yaml):
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                return None
-
     def insert(self):
         """
         NOTE:

control/veda_heal.py

@@ -23,6 +23,7 @@ import celeryapp
 from control_env import WORK_DIRECTORY
 from veda_encode import VedaEncode
 from veda_val import VALAPICall
+from VEDA.utils import get_config
 
 time_safetygap = datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=1)
 
@@ -34,33 +35,13 @@ class VedaHeal(object):
     """
     def __init__(self, **kwargs):
         self.current_time = datetime.datetime.utcnow().replace(tzinfo=utc)
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            ),
-        )
-        self.auth_dict = self._READ_AUTH()
+        self.auth_dict = get_config()
         # for individuals
         self.video_query = kwargs.get('video_query', None)
         self.freezing_bug = kwargs.get('freezing_bug', True)
         self.val_status = None
         self.retry_barrier_hours = 24
 
-    def _READ_AUTH(self):
-        if self.auth_yaml is None:
-            return None
-        if not os.path.exists(self.auth_yaml):
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                return None
-
     def discovery(self):
         self.video_query = Video.objects.filter(
             video_trans_start__lt=self.current_time - timedelta(

control/veda_hotstore.py

@@ -11,6 +11,7 @@ from boto.exception import S3ResponseError
 from os.path import expanduser
 
 from veda_utils import ErrorObject
+from VEDA.utils import get_config
 
 try:
     boto.config.add_section('Boto')
@@ -32,28 +33,11 @@ class Hotstore(object):
         # is this a final/encode file?
         self.endpoint = kwargs.get('endpoint', False)
 
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            ),
-        )
         self.auth_dict = self._READ_AUTH()
         self.endpoint_url = None
 
     def _READ_AUTH(self):
-        if self.auth_yaml is None:
-            return None
-        if not os.path.exists(self.auth_yaml):
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                return None
+        return get_config()
 
     def upload(self):
         if self.auth_dict is None:

control/veda_utils.py

 
 import os
 import sys
-import yaml
 import datetime
 import boto.ses
 import hashlib
@@ -14,35 +13,15 @@ Let's do some quick and dirty error handling & logging
 """
 from control.control_env import *
 from control.veda_encode import VedaEncode
-
+from VEDA.utils import get_config
 
 class EmailAlert():
 
     def __init__(self, **kwargs):
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            )
-        )
-        self.auth_dict = self._AUTH()
+        self.auth_dict = get_config()
         self.message = kwargs.get('message', None)
         self.subject = kwargs.get('subject', None)
 
-    def _AUTH(self):
-        if self.auth_yaml is None:
-            return None
-        if not os.path.exists(self.auth_yaml):
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                return None
-
     def email(self):
         email_subject = '[ VEDA ALERTING ]'
         email_subject += ' : ' + self.subject
@@ -132,31 +111,11 @@ class Output(object):
 class Report():
 
     def __init__(self, **kwargs):
-        self.auth_yaml = kwargs.get(
-            'auth_yaml',
-            os.path.join(
-                os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                'instance_config.yaml'
-            )
-        )
-        self.auth_dict = self._AUTH()
+        self.auth_dict = get_config()
         self.status = kwargs.get('status', None)
         self.upload_serial = kwargs.get('upload_serial', None)
         self.youtube_id = kwargs.get('youtube_id', None)
 
-    def _AUTH(self):
-        if self.auth_yaml is None:
-            return None
-        if not os.path.exists(self.auth_yaml):
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                return None
-
     def upload_status(self):
         if self.upload_serial is None:
             return None

control/veda_val.py

@@ -7,6 +7,7 @@ import ast
 import json
 import datetime
 import yaml
+from VEDA.utils import get_config
 
 LOGGER = logging.getLogger(__name__)
 
@@ -59,10 +60,6 @@ class VALAPICall():
         self.headers = None
 
         """Credentials"""
-        self.auth_yaml = os.path.join(
-            os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-            'instance_config.yaml'
-        )
         self.auth_dict = self._AUTH()
 
     def call(self):
@@ -81,21 +78,7 @@ class VALAPICall():
             self.send_val_data()
 
     def _AUTH(self):
-        if not os.path.exists(self.auth_yaml):
-            ErrorObject.print_error(
-                message='No Auth YAML'
-            )
-            return None
-
-        with open(self.auth_yaml, 'r') as stream:
-            try:
-                auth_dict = yaml.load(stream)
-                return auth_dict
-            except yaml.YAMLError as exc:
-                ErrorObject.print_error(
-                    message='YAML READ ERROR'
-                )
-                return None
+        return get_config()
 
     def val_tokengen(self):
         """

frontend/abvid_reporting.py

@@ -3,7 +3,7 @@ import sys
 from email.mime.text import MIMEText
 from datetime import date
 import boto.ses
-import yaml
+from VEDA.utils import get_config
 
 '''
 ABVID REPORTING - email / etc.
@@ -26,16 +26,7 @@ if v1.abvid_serial != None:
 get auth keys from instance yaml
 
 """
-auth_yaml = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-    'instance_config.yaml'
-)
-
-with open(auth_yaml, 'r') as stream:
-    try:
-        auth_dict = yaml.load(stream)
-    except yaml.YAMLError as exc:
-        print 'AUTH ERROR'
+auth_dict = get_config()
 
 
 def report_status(status, abvid_serial, youtube_id):

frontend/course_validate.py

@@ -14,16 +14,6 @@ from frontend_env import *
 """
 Import Django Shit
 """
-auth_yaml = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-    'instance_config.yaml'
-)
-
-with open(auth_yaml, 'r') as stream:
-    try:
-        auth_dict = yaml.load(stream)
-    except yaml.YAMLError as exc:
-        print 'AUTH ERROR'
 
 
 class VEDACat():

frontend/views.py

@@ -3,7 +3,6 @@
 import json
 import datetime
 from datetime import timedelta
-import yaml
 import base64
 import hmac
 import hashlib
@@ -16,11 +15,7 @@ from django.http import HttpResponseRedirect
 from frontend_env import *
 from course_validate import VEDACat
 from abvid_validate import validate_incoming, create_record, send_to_pipeline
-
-auth_yaml = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-    'instance_config.yaml'
-)
+from VEDA.utils import get_config
 
 """
 Here's the links for the main Page
@@ -169,11 +164,7 @@ def upload_alpha_1(request):
         Generate metadata From Fields
         Auth?
     """
-    with open(auth_yaml, 'r') as stream:
-        try:
-            auth_dict = yaml.load(stream)
-        except yaml.YAMLError as exc:
-            print 'AUTH ERROR'
+    auth_dict = get_config()
 
     policy_expiration = datetime.datetime.utcnow() + timedelta(hours=24)
     policy_exp = str(policy_expiration).replace(' ', 'T').split('.')[0] + 'Z'