moving the convert_tokens function to a common file

cms/envs/yaml_config.py

@@ -18,6 +18,7 @@ import yaml
 
 from .common import *
 from logsettings import get_logger_config
+from util.config_parse import convert_tokens
 import os
 
 from path import path
@@ -35,20 +36,6 @@ def construct_yaml_str(self, node):
 Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
 SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
 
-
-def convert_tokens(tokens):
-    """
-    This function is called on the token
-    dictionary, at the top level it converts
-    all strings containing 'None' to a literal
-    None due to a bug in Ansible which creates
-    the yaml files
-    """
-
-    for k, v in tokens.iteritems():
-        if v == 'None':
-            tokens[k] = None
-
 # SERVICE_VARIANT specifies name of the variant used, which decides what YAML
 # configuration files are read during startup.
 SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
@@ -134,7 +121,7 @@ CELERY_QUEUES = {
 with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file:
     ENV_TOKENS = yaml.load(env_file)
 
-convert_tokens(ENV_TOKENS)
+ENV_TOKENS = convert_tokens(ENV_TOKENS)
 
 ##########################################
 # Merge settings from common.py
@@ -225,7 +212,7 @@ MICROSITE_ROOT_DIR = path(MICROSITE_ROOT_DIR)
 with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file:
     AUTH_TOKENS = yaml.load(auth_file)
 
-convert_tokens(AUTH_TOKENS)
+AUTH_TOKENS = convert_tokens(AUTH_TOKENS)
 
 vars().update(AUTH_TOKENS)
 

common/djangoapps/util/config_parse.py

+"""
+Helper functions for configuration parsing
+"""
+import collections
+
+
+def convert_tokens(tokens):
+    """
+    This function is called on the token
+    dictionary that is imported from a yaml file.
+    It returns a new dictionary where
+    all strings containing 'None' are converted
+    to a literal None due to a bug in Ansible
+    """
+
+    if tokens == 'None':
+        return None
+    elif isinstance(tokens, basestring) or (not isinstance(tokens, collections.Iterable)):
+        return tokens
+    elif isinstance(tokens, dict):
+        return {
+            convert_tokens(k): convert_tokens(v)
+            for k, v in tokens.items()
+        }
+    else:
+        return [convert_tokens(v) for v in tokens]

lms/envs/yaml_config.py

@@ -14,6 +14,7 @@ import yaml
 
 from .common import *
 from logsettings import get_logger_config
+from util.config_parse import convert_tokens
 import os
 
 from path import path
@@ -29,20 +30,6 @@ def construct_yaml_str(self, node):
 Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
 SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)
 
-
-def convert_tokens(tokens):
-    """
-    This function is called on the token
-    dictionary, at the top level it converts
-    all strings containing 'None' to a literal
-    None due to a bug in Ansible which creates
-    the yaml files
-    """
-
-    for k, v in tokens.iteritems():
-        if v == 'None':
-            tokens[k] = None
-
 # SERVICE_VARIANT specifies name of the variant used, which decides what YAML
 # configuration files are read during startup.
 SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
@@ -150,7 +137,7 @@ with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file:
     ENV_TOKENS = yaml.load(env_file)
 
 # Works around an Ansible bug
-convert_tokens(ENV_TOKENS)
+ENV_TOKENS = convert_tokens(ENV_TOKENS)
 
 ##########################################
 # Merge settings from common.py
@@ -251,7 +238,7 @@ with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file:
     AUTH_TOKENS = yaml.load(auth_file)
 
 # Works around an Ansible bug
-convert_tokens(AUTH_TOKENS)
+AUTH_TOKENS = convert_tokens(AUTH_TOKENS)
 
 vars().update(AUTH_TOKENS)