Merge pull request #4380 from edx/nimisha/mongo-bwc-config

auto-migrate old mongo to draft modulestore

cms/envs/aws.py

@@ -15,6 +15,7 @@ import os
 
 from path import path
 from dealer.git import git
+from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
 
 # SERVICE_VARIANT specifies name of the variant used, which decides what JSON
 # configuration files are read during startup.
@@ -229,7 +230,7 @@ if AWS_SECRET_ACCESS_KEY == "":
     AWS_SECRET_ACCESS_KEY = None
 
 DATABASES = AUTH_TOKENS['DATABASES']
-MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS['MODULESTORE'])
+MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE))
 CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
 DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
 # Datadog for events!

cms/envs/common.py

@@ -30,11 +30,11 @@ import sys
 import lms.envs.common
 # Although this module itself may not use these imported variables, other dependent modules may.
 from lms.envs.common import (
-    USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE
+    USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE,
+    update_module_store_settings
 )
 from path import path
 from warnings import simplefilter
-from lms.envs.modulestore_settings import *
 
 from lms.lib.xblock.mixin import LmsBlockMixin
 from dealer.git import git

cms/envs/devstack.py

@@ -72,8 +72,12 @@ DEBUG_TOOLBAR_CONFIG = {
 DEBUG_TOOLBAR_MONGO_STACKTRACES = False
 
 ###############################################################################
-# Lastly, see if the developer has any local overrides.
+# See if the developer has any local overrides.
 try:
     from .private import *  # pylint: disable=F0401
 except ImportError:
     pass
+
+#####################################################################
+# Lastly, run any migrations, if needed.
+MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE)

lms/envs/modulestore_settings.py → common/lib/xmodule/xmodule/modulestore/modulestore_settings.py

@@ -2,6 +2,12 @@
 This file contains helper functions for configuring module_store_setting settings and support for backward compatibility with older formats.
 """
 
+import warnings
+
+
+# Python 2.7 by default suppresses DeprecationWarnings. Make sure we show these, always.
+warnings.simplefilter('once', DeprecationWarning)
+
 
 def convert_module_store_setting_if_needed(module_store_setting):
     """
@@ -14,17 +20,26 @@ def convert_module_store_setting_if_needed(module_store_setting):
         """
         new_store_list = []
         for store_name, store_settings in old_stores.iteritems():
+
             store_settings['NAME'] = store_name
             if store_name == 'default':
                 new_store_list.insert(0, store_settings)
             else:
                 new_store_list.append(store_settings)
+
+            # migrate request for the old 'direct' Mongo store to the Draft store
+            if store_settings['ENGINE'] == 'xmodule.modulestore.mongo.MongoModuleStore':
+                warnings.warn("MongoModuleStore is deprecated! Please use DraftModuleStore.", DeprecationWarning)
+                store_settings['ENGINE'] = 'xmodule.modulestore.mongo.draft.DraftModuleStore'
+
         return new_store_list
 
     if module_store_setting is None:
         return None
 
     if module_store_setting['default']['ENGINE'] != 'xmodule.modulestore.mixed.MixedModuleStore':
+        warnings.warn("Direct access to a modulestore is deprecated. Please use MixedModuleStore.", DeprecationWarning)
+
         # convert to using mixed module_store
         new_module_store_setting = {
             "default": {
@@ -44,6 +59,11 @@ def convert_module_store_setting_if_needed(module_store_setting):
         module_store_setting = new_module_store_setting
 
     elif isinstance(module_store_setting['default']['OPTIONS']['stores'], dict):
+        warnings.warn(
+            "Using a dict for the Stores option in the MixedModuleStore is deprecated.  Please use a list instead.",
+            DeprecationWarning
+        )
+
         # convert old-style (unordered) dict to (an ordered) list
         module_store_setting['default']['OPTIONS']['stores'] = convert_old_stores_into_list(
             module_store_setting['default']['OPTIONS']['stores']

common/lib/xmodule/xmodule/modulestore/tests/test_modulestore_settings.py

+"""
+Tests for testing the modulestore settings migration code.
+"""
+import copy
+from unittest import TestCase
+from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
+
+
+class ModuleStoreSettingsMigration(TestCase):
+    """
+    Tests for the migration code for the module store settings
+    """
+
+    OLD_CONFIG = {
+        "default": {
+            "ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
+            "OPTIONS": {
+                "data_dir": "directory",
+                "default_class": "xmodule.hidden_module.HiddenDescriptor",
+            },
+            "DOC_STORE_CONFIG": {},
+        }
+    }
+
+    OLD_CONFIG_WITH_DIRECT_MONGO = {
+        "default": {
+            "ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
+            "OPTIONS": {
+                "collection": "modulestore",
+                "db": "edxapp",
+                "default_class": "xmodule.hidden_module.HiddenDescriptor",
+                "fs_root": "/edx/var/edxapp/data",
+                "host": "localhost",
+                "password": "password",
+                "port": 27017,
+                "render_template": "edxmako.shortcuts.render_to_string",
+                "user": "edxapp"
+            },
+            "DOC_STORE_CONFIG": {},
+        }
+    }
+
+    OLD_MIXED_CONFIG_WITH_DICT = {
+        "default": {
+            "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
+            "OPTIONS": {
+                "mappings": {},
+                "reference_type": "Location",
+                "stores": {
+                    "an_old_mongo_store": {
+                        "DOC_STORE_CONFIG": {},
+                        "ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
+                        "OPTIONS": {
+                            "collection": "modulestore",
+                            "db": "test",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor",
+                        }
+                    },
+                    "default": {
+                        "ENGINE": "the_default_store",
+                        "OPTIONS": {
+                            "option1": "value1",
+                            "option2": "value2"
+                        },
+                        "DOC_STORE_CONFIG": {}
+                    },
+                    "xml": {
+                        "ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
+                        "OPTIONS": {
+                            "data_dir": "directory",
+                            "default_class": "xmodule.hidden_module.HiddenDescriptor"
+                        },
+                        "DOC_STORE_CONFIG": {}
+                    }
+                }
+            }
+        }
+    }
+
+    def _get_mixed_stores(self, mixed_setting):
+        """
+        Helper for accessing stores in a configuration setting for the Mixed modulestore
+        """
+        return mixed_setting["default"]["OPTIONS"]["stores"]
+
+    def assertStoreValuesEqual(self, store_setting1, store_setting2):
+        """
+        Tests whether the fields in the given store_settings are equal
+        """
+        store_fields = ["OPTIONS", "DOC_STORE_CONFIG"]
+        for field in store_fields:
+            self.assertEqual(store_setting1[field], store_setting2[field])
+
+    def assertMigrated(self, old_setting):
+        """
+        Migrates the given setting and checks whether it correctly converted
+        to an ordered list of stores within Mixed.
+        """
+        # pass a copy of the old setting since the migration modifies the given setting
+        new_mixed_setting = convert_module_store_setting_if_needed(copy.deepcopy(old_setting))
+
+        # check whether the configuration is encapsulated within Mixed.
+        self.assertEqual(new_mixed_setting["default"]["ENGINE"], "xmodule.modulestore.mixed.MixedModuleStore")
+
+        # check whether the stores are in an ordered list
+        new_stores = self._get_mixed_stores(new_mixed_setting)
+        self.assertIsInstance(new_stores, list)
+
+        return new_mixed_setting, new_stores[0]
+
+    def test_convert_into_mixed(self):
+        old_setting = self.OLD_CONFIG
+        _, new_default_store_setting = self.assertMigrated(old_setting)
+        self.assertStoreValuesEqual(new_default_store_setting, old_setting["default"])
+        self.assertEqual(new_default_store_setting["ENGINE"], old_setting["default"]["ENGINE"])
+
+    def test_convert_from_old_mongo_to_draft_store(self):
+        old_setting = self.OLD_CONFIG_WITH_DIRECT_MONGO
+        _, new_default_store_setting = self.assertMigrated(old_setting)
+        self.assertStoreValuesEqual(new_default_store_setting, old_setting["default"])
+        self.assertEqual(new_default_store_setting["ENGINE"], "xmodule.modulestore.mongo.draft.DraftModuleStore")
+
+    def test_convert_from_dict_to_list(self):
+        old_mixed_setting = self.OLD_MIXED_CONFIG_WITH_DICT
+        new_mixed_setting, new_default_store_setting = self.assertMigrated(old_mixed_setting)
+        self.assertEqual(new_default_store_setting["ENGINE"], "the_default_store")
+
+        # compare each store configured in mixed
+        old_stores = self._get_mixed_stores(self.OLD_MIXED_CONFIG_WITH_DICT)
+        new_stores = self._get_mixed_stores(new_mixed_setting)
+        self.assertEqual(len(new_stores), len(old_stores))
+        for new_store_setting in self._get_mixed_stores(new_mixed_setting):
+            self.assertStoreValuesEqual(new_store_setting, old_stores[new_store_setting['NAME']])

lms/envs/aws.py

@@ -18,6 +18,7 @@ from logsettings import get_logger_config
 import os
 
 from path import path
+from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
 
 # SERVICE_VARIANT specifies name of the variant used, which decides what JSON
 # configuration files are read during startup.

lms/envs/common.py

@@ -33,7 +33,7 @@ from path import path
 from warnings import simplefilter
 
 from .discussionsettings import *
-from .modulestore_settings import *
+from xmodule.modulestore.modulestore_settings import update_module_store_settings
 
 from lms.lib.xblock.mixin import LmsBlockMixin
 

lms/envs/devstack.py

@@ -82,8 +82,12 @@ FEATURES['ENABLE_PAYMENT_FAKE'] = True
 CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = '/shoppingcart/payment_fake/'
 
 #####################################################################
-# Lastly, see if the developer has any local overrides.
+# See if the developer has any local overrides.
 try:
     from .private import *      # pylint: disable=F0401
 except ImportError:
     pass
+
+#####################################################################
+# Lastly, run any migrations, if needed.
+MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE)