separate static configuration

VEDA/tests/test_utils.py

@@ -19,6 +19,13 @@ TEST_CONFIG = {
     }
 }
 
+TEST_STATIC_CONFIG = {
+    'abc': 999,
+    'nested': {
+        'nested_url': 'nested.example.com'
+    }
+}
+
 
 @ddt
 class UtilTests(TestCase):
@@ -30,6 +37,8 @@ class UtilTests(TestCase):
         Tests setup.
         """
         self._orig_environ = dict(os.environ)
+
+        # create a temporary default config file
         _, self.file_path = tempfile.mkstemp(
             suffix='.yml',
             dir=tempfile.tempdir
@@ -39,6 +48,14 @@ class UtilTests(TestCase):
 
         os.environ['VIDEO_PIPELINE_CFG'] = self.file_path
 
+        # create a temporary static config file
+        _, self.static_file_path = tempfile.mkstemp(
+            suffix='.yml',
+            dir=tempfile.tempdir
+        )
+        with open(self.static_file_path, 'w') as outfile:
+            yaml.dump(TEST_STATIC_CONFIG, outfile, default_flow_style=False)
+
     def tearDown(self):
         """
         Reverse the setup
@@ -47,6 +64,10 @@ class UtilTests(TestCase):
         os.environ.clear()
         os.environ.update(self._orig_environ)
 
+        # remove temporary files
+        os.remove(self.file_path)
+        os.remove(self.static_file_path)
+
     @data(
         {
             'urls': ('http://api.cielo24/', '/add/job'),
@@ -132,21 +153,30 @@ class UtilTests(TestCase):
         instance_config = utils.get_config()
         self.assertNotEqual(instance_config, {})
 
-        # read the default config file to verify that correct config is loaded
+        # read the default config file
         default_yaml_config_file = os.path.join(
-            utils.DEFAULT_CONFIG_FILE_PATH,
+            utils.CONFIG_ROOT_DIR,
             utils.DEFAULT_CONFIG_FILE_NAME
         )
         with open(default_yaml_config_file, 'r') as config:
             config_dict = yaml.load(config)
-            self.assertDictEqual(instance_config, config_dict)
+
+        # read the default static config file
+        with open(utils.STATIC_CONFIG_FILE_PATH, 'r') as config:
+            static_config_dict = yaml.load(config)
+
+        self.assertDictEqual(
+            instance_config,
+            dict(config_dict, **static_config_dict)
+        )
 
     def test_get_config_with_path(self):
         """
         Tests that utils.get_config works as expected when reading config from environment path.
         """
-        instance_config = utils.get_config()
-        self.assertDictEqual(instance_config, TEST_CONFIG)
+        with patch('VEDA.utils.STATIC_CONFIG_FILE_PATH', self.static_file_path):
+            instance_config = utils.get_config()
+        self.assertDictEqual(instance_config, dict(TEST_CONFIG, **TEST_STATIC_CONFIG))
 
     @data(
         {

VEDA/utils.py

@@ -8,8 +8,9 @@ import yaml
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 
-DEFAULT_CONFIG_FILE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CONFIG_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 DEFAULT_CONFIG_FILE_NAME = 'instance_config.yaml'
+STATIC_CONFIG_FILE_PATH = os.path.join(CONFIG_ROOT_DIR, 'static_config.yaml')
 
 
 def extract_course_org(course_id):
@@ -61,14 +62,18 @@ def get_config(yaml_config_file=DEFAULT_CONFIG_FILE_NAME):
         yaml_config_file = os.environ['VIDEO_PIPELINE_CFG']
     except KeyError:
         yaml_config_file = os.path.join(
-            DEFAULT_CONFIG_FILE_PATH,
+            CONFIG_ROOT_DIR,
             yaml_config_file
         )
 
     with open(yaml_config_file, 'r') as config:
         config_dict = yaml.load(config)
 
-    return config_dict
+    # read static config file
+    with open(STATIC_CONFIG_FILE_PATH, 'r') as config:
+        static_config_dict = yaml.load(config)
+
+    return dict(config_dict, **static_config_dict)
 
 
 def scrub_query_params(url, params_to_scrub):

instance_config.yaml

@@ -24,11 +24,6 @@ SECRET_KEY: ""
 # Fernet keys
 FERNET_KEYS: []
 
-# Django DEBUG global
-# (set to false in prod)
-debug: false
-
-
 # JWT AUTH settings
 JWT_AUTH:
     JWT_SECRET_KEY:
@@ -57,7 +52,6 @@ veda_s3_upload_bucket:
 veda_s3_hotstore_bucket:
 veda_deliverable_bucket:
 # Settings
-multi_upload_barrier: 2000000000
 veda_base_url:
 s3_base_url: https://s3.amazonaws.com
 
@@ -111,15 +105,6 @@ val_username:
 val_transcript_create_url:
 val_video_transcript_status_url:
 
-# ---
-# Celery Info
-# ---
-celery_app_name: veda_production
-# can do multiple queues like so: foo,bar,baz
-celery_worker_queue: encode_worker
-celery_deliver_queue: deliver_worker
-celery_threads: 1
-
 
 rabbitmq_broker:
 rabbitmq_pass:
@@ -136,66 +121,3 @@ sg_script_key:
 # Endpoints
 # ---
 threeplay_ftphost:
-
-## Encoding Config
-ffmpeg_compiled: "ffmpeg"
-ffprobe_compiled: "ffprobe"
-target_aspect_ratio: 1.7777778
-
-# ----------
-##---
-# This is a list of encodes and their respective course
-# boolean matches
-encode_dict:
-    review_proc:
-        - review
-    mobile_override:
-        - override
-    s3_proc:
-        - mobile_high
-        - mobile_low
-        - audio_mp3
-        - desktop_webm
-        - desktop_mp4
-        - hls
-
-    yt_proc:
-        - youtube
-
-##---
-# This is a list of encode profiles and their val profile matches
-# boolean matches
-val_profile_dict:
-    mobile_low:
-        - mobile_low
-
-    desktop_mp4:
-        - desktop_mp4
-
-    override:
-        - desktop_mp4
-        - mobile_low
-        - mobile_high
-
-    mobile_high:
-        - mobile_high
-
-    audio_mp3:
-        - audio_mp3
-
-    desktop_webm:
-        - desktop_webm
-
-    youtube:
-        - youtube
-    review:
-    hls:
-        - hls
-
-#--
-# Heal settings
-heal_start: 1
-heal_end: 144
-global_timeout: 40
-
-...

static_config.yaml

+---
+# This configuration should only have static settings.
+
+# Celery Info
+celery_app_name: veda_production
+# can do multiple queues like so: foo,bar,baz
+celery_worker_queue: encode_worker
+celery_deliver_queue: deliver_worker
+celery_threads: 1
+
+# S3 upload settings
+multi_upload_barrier: 2000000000
+
+# Encoding Config
+ffmpeg_compiled: "ffmpeg"
+ffprobe_compiled: "ffprobe"
+target_aspect_ratio: 1.7777778
+
+# This is a list of encodes and their respective course
+# boolean matches
+encode_dict:
+    review_proc:
+        - review
+    mobile_override:
+        - override
+    s3_proc:
+        - mobile_high
+        - mobile_low
+        - audio_mp3
+        - desktop_webm
+        - desktop_mp4
+        - hls
+
+    yt_proc:
+        - youtube
+
+# This is a list of encode profiles and their val profile matches
+# boolean matches
+val_profile_dict:
+    mobile_low:
+        - mobile_low
+
+    desktop_mp4:
+        - desktop_mp4
+
+    override:
+        - desktop_mp4
+        - mobile_low
+        - mobile_high
+
+    mobile_high:
+        - mobile_high
+
+    audio_mp3:
+        - audio_mp3
+
+    desktop_webm:
+        - desktop_webm
+
+    youtube:
+        - youtube
+    review:
+    hls:
+        - hls
+
+# Heal settings
+heal_start: 1
+heal_end: 144
+
+global_timeout: 40