Merge pull request #15445 from edx/jmbowman/PLAT-1499

PLAT-1499 Use appropriate default settings in Docker devstack

cms/envs/devstack_optimized.py

@@ -18,9 +18,14 @@ as they are for non-optimized devstack. Instead, update_assets must be
 invoked each time that changes have been made.
 """
 
+import os
+
 ########################## Devstack settings ###################################
 
-from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
+if 'BOK_CHOY_HOSTNAME' in os.environ:
+    from .devstack_docker import *  # pylint: disable=wildcard-import, unused-wildcard-import
+else:
+    from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
 
 TEST_ROOT = REPO_ROOT / "test_root"
 

cms/envs/devstack_with_worker.py

@@ -11,10 +11,15 @@ In two separate processes on devstack:
     ./manage.py cms celery worker --settings=devstack_with_worker
 """
 
+import os
+
 # We intentionally define lots of variables that aren't used, and
 # want to import all variables from base settings files
 # pylint: disable=wildcard-import, unused-wildcard-import
-from cms.envs.devstack import *
+if 'BOK_CHOY_HOSTNAME' in os.environ:
+    from cms.envs.devstack_docker import *
+else:
+    from cms.envs.devstack import *
 
 # Require a separate celery worker
 CELERY_ALWAYS_EAGER = False

common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py

@@ -7,4 +7,5 @@ overrides don't exist
 import os
 
 MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
-MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
+MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST',
+                            'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost')

common/lib/xmodule/xmodule/tests/test_video.py

@@ -12,6 +12,7 @@ You can then use the CourseFactory and XModuleItemFactory as defined
 in common/lib/xmodule/xmodule/modulestore/tests/factories.py to create
 the course, section, subsection, unit, etc.
 """
+import os
 import unittest
 import datetime
 from uuid import uuid4
@@ -821,7 +822,7 @@ class VideoDescriptorIndexingTestCase(unittest.TestCase):
         settings.CONTENTSTORE = {
             'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
             'DOC_STORE_CONFIG': {
-                'host': 'localhost',
+                'host': 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost',
                 'db': 'test_xcontent_%s' % uuid4().hex,
             },
             # allow for additional options that can be keyed on a name, e.g. 'trashcan'

lms/envs/devstack_optimized.py

@@ -18,9 +18,14 @@ as they are for non-optimized devstack. Instead, update_assets must be
 invoked each time that changes have been made.
 """
 
+import os
+
 ########################## Devstack settings ###################################
 
-from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
+if 'BOK_CHOY_HOSTNAME' in os.environ:
+    from .devstack_docker import *  # pylint: disable=wildcard-import, unused-wildcard-import
+else:
+    from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
 
 TEST_ROOT = REPO_ROOT / "test_root"
 

lms/envs/devstack_with_worker.py

@@ -11,10 +11,15 @@ In two separate processes on devstack:
     ./manage.py lms celery worker --settings=devstack_with_worker
 """
 
+import os
+
 # We intentionally define lots of variables that aren't used, and
 # want to import all variables from base settings files
 # pylint: disable=wildcard-import, unused-wildcard-import
-from lms.envs.devstack import *
+if 'BOK_CHOY_HOSTNAME' in os.environ:
+    from lms.envs.devstack_docker import *
+else:
+    from lms.envs.devstack import *
 
 # Require a separate celery worker
 CELERY_ALWAYS_EAGER = False

pavelib/assets.py

@@ -799,7 +799,7 @@ def watch_assets(options):
 
     # We only want Webpack to re-run on changes to its own entry points, not all JS files, so we use its own watcher
     # instead of subclassing from Watchdog like the other watchers do
-    execute_webpack_watch(settings='devstack')
+    execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
 
     if not getattr(options, 'background', False):
         # when running as a separate process, the main thread needs to loop
@@ -828,7 +828,7 @@ def update_assets(args):
         help="lms or studio",
     )
     parser.add_argument(
-        '--settings', type=str, default="devstack",
+        '--settings', type=str, default=Env.DEVSTACK_SETTINGS,
         help="Django settings module",
     )
     parser.add_argument(
@@ -861,7 +861,7 @@ def update_assets(args):
     process_xmodule_assets()
     process_npm_assets()
     compile_coffeescript()
-    execute_webpack(prod=(args.settings != "devstack"), settings=args.settings)
+    execute_webpack(prod=(args.settings != Env.DEVSTACK_SETTINGS), settings=args.settings)
 
     # Compile sass for themes and system
     execute_compile_sass(args)

pavelib/i18n.py

@@ -10,6 +10,7 @@ from path import Path as path
 from paver.easy import cmdopts, needs, sh, task
 
 from .utils.cmd import django_cmd
+from .utils.envs import Env
 from .utils.timer import timed
 
 try:
@@ -17,7 +18,7 @@ try:
 except ImportError:
     colorize = lambda color, text: text
 
-DEFAULT_SETTINGS = 'devstack'
+DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS
 
 
 @task

pavelib/paver_tests/test_assets.py

@@ -10,6 +10,7 @@ from watchdog.observers.polling import PollingObserver
 
 from pavelib.assets import COLLECTSTATIC_LOG_DIR_ARG, collect_assets
 
+from ..utils.envs import Env
 from .utils import PaverTestCase
 
 ROOT_PATH = path(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
@@ -252,12 +253,12 @@ class TestCollectAssets(PaverTestCase):
         if specified_log_loc is None:
             collect_assets(
                 systems,
-                "devstack"
+                Env.DEVSTACK_SETTINGS
             )
         else:
             collect_assets(
                 systems,
-                "devstack",
+                Env.DEVSTACK_SETTINGS,
                 **specified_log_dict
             )
         self.assertEqual(self.task_messages, expected_messages)
@@ -271,7 +272,7 @@ class TestCollectAssets(PaverTestCase):
         systems = ["lms"]
         kwargs = {COLLECTSTATIC_LOG_DIR_ARG: None}
         expected_messages = self._set_expected_messages(log_location=expected_log_loc, systems=systems)
-        collect_assets(systems, "devstack", **kwargs)
+        collect_assets(systems, Env.DEVSTACK_SETTINGS, **kwargs)
         self.assertEqual(self.task_messages, expected_messages)
 
     def _set_expected_messages(self, log_location, systems):
@@ -284,8 +285,9 @@ class TestCollectAssets(PaverTestCase):
         expected_messages = []
         for sys in systems:
             expected_messages.append(
-                'python manage.py {system} --settings=devstack collectstatic --noinput {log_loc}'.format(
+                'python manage.py {system} --settings={settings} collectstatic --noinput {log_loc}'.format(
                     system=sys,
+                    settings=Env.DEVSTACK_SETTINGS,
                     log_loc=log_location
                 )
             )

pavelib/paver_tests/test_i18n.py

@@ -11,6 +11,7 @@ from paver.easy import call_task, task
 
 import pavelib.i18n
 from pavelib.paver_tests.utils import PaverTestCase
+from pavelib.utils.envs import Env
 
 
 TX_CONFIG_SIMPLE = """\
@@ -155,14 +156,14 @@ class TestI18nDummy(PaverTestCase):
         """
         self.reset_task_messages()
         os.environ['NO_PREREQ_INSTALL'] = "true"
-        call_task('pavelib.i18n.i18n_dummy', options={"settings": 'test'})
+        call_task('pavelib.i18n.i18n_dummy', options={"settings": Env.TEST_SETTINGS})
         self.assertEquals(
             self.task_messages,
             [
                 u'i18n_tool extract',
                 u'i18n_tool dummy',
                 u'i18n_tool generate',
-                u'python manage.py lms --settings=test compilejsi18n',
-                u'python manage.py cms --settings=test compilejsi18n',
+                u'python manage.py lms --settings={} compilejsi18n'.format(Env.TEST_SETTINGS),
+                u'python manage.py cms --settings={} compilejsi18n'.format(Env.TEST_SETTINGS),
             ]
         )

pavelib/paver_tests/test_servers.py

@@ -102,7 +102,7 @@ class TestPaverServerTasks(PaverTestCase):
         """
         options = server_options.copy()
         is_optimized = options.get("optimized", False)
-        expected_settings = "devstack_optimized" if is_optimized else options.get("settings", "devstack")
+        expected_settings = "devstack_optimized" if is_optimized else options.get("settings", Env.DEVSTACK_SETTINGS)
 
         # First test with LMS
         options["system"] = "lms"
@@ -162,7 +162,7 @@ class TestPaverServerTasks(PaverTestCase):
         """
         Test the "update_db" task.
         """
-        settings = options.get("settings", "devstack")
+        settings = options.get("settings", Env.DEVSTACK_SETTINGS)
         call_task("pavelib.servers.update_db", options=options)
         # pylint: disable=line-too-long
         db_command = "NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. "
@@ -185,7 +185,7 @@ class TestPaverServerTasks(PaverTestCase):
         """
         Test the "check_settings" task.
         """
-        settings = options.get("settings", "devstack")
+        settings = options.get("settings", Env.DEVSTACK_SETTINGS)
         call_task("pavelib.servers.check_settings", args=[system, settings])
         self.assertEquals(
             self.task_messages,
@@ -231,19 +231,19 @@ class TestPaverServerTasks(PaverTestCase):
         else:
             call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options)
         expected_messages = options.get("expected_messages", [])
-        expected_settings = settings if settings else "devstack"
+        expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
         expected_asset_settings = asset_settings if asset_settings else expected_settings
         if is_optimized:
             expected_settings = "devstack_optimized"
             expected_asset_settings = "test_static_optimized"
-        expected_collect_static = not is_fast and expected_settings != "devstack"
+        expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
         if not is_fast:
             expected_messages.append(u"xmodule_assets common/static/xmodule")
             expected_messages.append(u"install npm_assets")
             expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
             expected_messages.extend([c.format(settings=expected_asset_settings) for c in EXPECTED_PRINT_SETTINGS_COMMAND])
             expected_messages.append(EXPECTED_WEBPACK_COMMAND.format(
-                node_env="production" if expected_asset_settings != "devstack" else "development",
+                node_env="production" if expected_asset_settings != Env.DEVSTACK_SETTINGS else "development",
                 static_root_lms=None,
                 static_root_cms=None
             ))
@@ -273,12 +273,12 @@ class TestPaverServerTasks(PaverTestCase):
         is_fast = options.get("fast", False)
         self.reset_task_messages()
         call_task("pavelib.servers.run_all_servers", options=options)
-        expected_settings = settings if settings else "devstack"
+        expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
         expected_asset_settings = asset_settings if asset_settings else expected_settings
         if is_optimized:
             expected_settings = "devstack_optimized"
             expected_asset_settings = "test_static_optimized"
-        expected_collect_static = not is_fast and expected_settings != "devstack"
+        expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
         expected_messages = []
         if not is_fast:
             expected_messages.append(u"xmodule_assets common/static/xmodule")
@@ -286,7 +286,7 @@ class TestPaverServerTasks(PaverTestCase):
             expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
             expected_messages.extend([c.format(settings=expected_asset_settings) for c in EXPECTED_PRINT_SETTINGS_COMMAND])
             expected_messages.append(EXPECTED_WEBPACK_COMMAND.format(
-                node_env="production" if expected_asset_settings != "devstack" else "development",
+                node_env="production" if expected_asset_settings != Env.DEVSTACK_SETTINGS else "development",
                 static_root_lms=None,
                 static_root_cms=None
             ))

pavelib/servers.py

@@ -10,11 +10,12 @@ from paver.easy import call_task, cmdopts, consume_args, needs, sh, task
 
 from .assets import collect_assets
 from .utils.cmd import django_cmd
+from .utils.envs import Env
 from .utils.process import run_multi_processes, run_process
 from .utils.timer import timed
 
 DEFAULT_PORT = {"lms": 8000, "studio": 8001}
-DEFAULT_SETTINGS = 'devstack'
+DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS
 OPTIMIZED_SETTINGS = "devstack_optimized"
 OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized"
 

pavelib/utils/envs.py

@@ -69,6 +69,8 @@ class Env(object):
     SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0')
     USING_DOCKER = SERVER_HOST != '0.0.0.0'
     SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy'
+    DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack'
+    TEST_SETTINGS = 'test_docker' if USING_DOCKER else 'test'
 
     BOK_CHOY_SERVERS = {
         'lms': {

pavelib/utils/test/suites/nose_suite.py

@@ -119,7 +119,7 @@ class SystemTestSuite(NoseTestSuite):
 
         self.processes = kwargs.get('processes', None)
         self.randomize = kwargs.get('randomize', None)
-        self.settings = kwargs.get('settings', 'test')
+        self.settings = kwargs.get('settings', Env.TEST_SETTINGS)
 
         if self.processes is None:
             # Don't use multiprocessing by default