initial commit for a mixed module store which can interoperate with both XML and…

initial commit for a mixed module store which can interoperate with both XML and Mongo module stores

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -258,7 +258,7 @@ class ModuleStore(object):
     An abstract interface for a database backend that stores XModuleDescriptor
     instances
     """
-    def has_item(self, location):
+    def has_item(self, course_id, location):
         """
         Returns True if location exists in this ModuleStore.
         """

common/lib/xmodule/xmodule/modulestore/django.py

@@ -25,24 +25,31 @@ def load_function(path):
     return getattr(import_module(module_path), name)
 
 
-def modulestore(name='default'):
-    if name not in _MODULESTORES:
-        class_ = load_function(settings.MODULESTORE[name]['ENGINE'])
+def create_modulestore_instance(engine, options):
+    """
+    This will return a new instance of a modulestore given an engine and options
+    """
+    class_ = load_function(engine)
 
-        options = {}
+    _options = {}
+    _options.update(options)
 
-        options.update(settings.MODULESTORE[name]['OPTIONS'])
-        for key in FUNCTION_KEYS:
-            if key in options:
-                options[key] = load_function(options[key])
+    for key in FUNCTION_KEYS:
+        if key in _options:
+            _options[key] = load_function(_options[key])
 
-        _MODULESTORES[name] = class_(
-            **options
-        )
+    return class_(
+        **_options
+    )
 
-    return _MODULESTORES[name]
 
-# if 'DJANGO_SETTINGS_MODULE' in environ:
-#     # Initialize the modulestores immediately
-#     for store_name in settings.MODULESTORE:
-#         modulestore(store_name)
+def modulestore(name='default'):
+    """
+    This returns an instance of a modulestore of given name. This will wither return an existing
+    modulestore or create a new one
+    """
+    if name not in _MODULESTORES:
+        _MODULESTORES[name] = create_modulestore_instance(settings.MODULESTORE[name]['ENGINE'],
+                                                          settings.MODULESTORE[name]['OPTIONS'])
+
+    return _MODULESTORES[name]

common/lib/xmodule/xmodule/modulestore/mixed.py

+"""
+MixedModuleStore allows for aggregation between multiple modulestores.
+
+In this way, courses can be served up both - say - XMLModuleStore or MongoModuleStore
+
+IMPORTANT: This modulestore is experimental AND INCOMPLETE. Therefore this should only be used cautiously
+"""
+
+from . import ModuleStoreBase
+from django import create_modulestore_instance
+
+
+class MixedModuleStore(ModuleStoreBase):
+    """
+    ModuleStore that can be backed by either XML or Mongo
+    """
+    def __init__(self, mappings, stores):
+        """
+        Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
+        collection of other modulestore configuration informations
+        """
+        super(MixedModuleStore, self).__init__()
+
+        self.modulestores = {}
+        self.mappings = mappings
+        for key in stores:
+            self.modulestores[key] = create_modulestore_instance(stores[key]['ENGINE'],
+                                                                 stores[key]['OPTIONS'])
+
+    def _get_modulestore_for_courseid(self, course_id):
+        """
+        For a given course_id, look in the mapping table and see if it has been pinned
+        to a particular modulestore
+        """
+        return self.mappings.get(course_id, self.mappings['default'])
+
+    def has_item(self, course_id, location):
+        return self._get_modulestore_for_courseid(course_id).has_item(course_id, location)
+
+    def get_item(self, location, depth=0):
+        """
+        This method is explicitly not implemented as we need a course_id to disambiguate
+        We should be able to fix this when the data-model rearchitecting is done
+        """
+        raise NotImplementedError
+
+    def get_instance(self, course_id, location, depth=0):
+        return self._get_modulestore_for_courseid(course_id).get_instance(course_id, location, depth)
+
+    def get_items(self, location, course_id=None, depth=0):
+        """
+        Returns a list of XModuleDescriptor instances for the items
+        that match location. Any element of location that is None is treated
+        as a wildcard that matches any value
+
+        location: Something that can be passed to Location
+
+        depth: An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of calls to
+            get_children() to cache. None indicates to cache all descendents
+        """
+        if not course_id:
+            raise Exception("Must pass in a course_id when calling get_items() with MixedModuleStore")
+
+        return self._get_modulestore_for_courseid(course_id).get_items(location, course_id, depth)
+
+    def update_item(self, location, data, allow_not_found=False):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def update_children(self, location, children):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def update_metadata(self, location, metadata):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def delete_item(self, location):
+        """
+        MixedModuleStore is for read-only (aka LMS)
+        """
+        raise NotImplementedError
+
+    def get_courses(self):
+        '''
+        Returns a list containing the top level XModuleDescriptors of the courses
+        in this modulestore.
+        '''
+        courses = []
+        for key in self.modulestores:
+            courses.append(self.modulestores[key].get_courses)
+        return courses
+
+    def get_course(self, course_id):
+        return self._get_modulestore_for_courseid(course_id).get_course(course_id)
+
+    def get_parent_locations(self, location, course_id):
+        return self._get_modulestore_for_courseid(course_id).get_parent_locations(location, course_id)

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -547,7 +547,7 @@ class MongoModuleStore(ModuleStoreBase):
             raise ItemNotFoundError(location)
         return item
 
-    def has_item(self, location):
+    def has_item(self, course_id, location):
         """
         Returns True if location exists in this ModuleStore.
         """

common/lib/xmodule/xmodule/modulestore/search.py

@@ -81,7 +81,7 @@ def path_to_location(modulestore, course_id, location):
         # If we're here, there is no path
         return None
 
-    if not modulestore.has_item(location):
+    if not modulestore.has_item(course_id, location):
         raise ItemNotFoundError
 
     path = find_path_to_course()

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -275,7 +275,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
         result = self._load_items(course_entry, [root], 0, lazy=True)
         return result[0]
 
-    def has_item(self, block_location):
+    def get_course_for_item(self, location):
+        '''
+        Provided for backward compatibility. Is equivalent to calling get_course
+        :param location:
+        '''
+        return self.get_course(location)
+
+    def has_item(self, course_id, block_location):
         """
         Returns True if location exists in its course. Returns false if
         the course or the block w/in the course do not exist for the given version.

common/lib/xmodule/xmodule/modulestore/store_utilities.py

@@ -152,7 +152,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
 
     # check to see if the dest_location exists as an empty course
     # we need an empty course because the app layers manage the permissions and users
-    if not modulestore.has_item(dest_location):
+    if not modulestore.has_item(dest_location.course_id, dest_location):
         raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
 
     # verify that the dest_location really is an empty course, which means only one with an optional 'overview'
@@ -171,7 +171,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
         raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
 
     # check to see if the source course is actually there
-    if not modulestore.has_item(source_location):
+    if not modulestore.has_item(source_location.course_id, source_location):
         raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
 
     # Get all modules under this namespace which is (tag, org, course) tuple
@@ -250,7 +250,7 @@ def delete_course(modulestore, contentstore, source_location, commit=False):
     """
 
     # check to see if the source course is actually there
-    if not modulestore.has_item(source_location):
+    if not modulestore.has_item(source_location.course_id, source_location):
         raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
 
     # first delete all of the thumbnails

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -257,18 +257,19 @@ class SplitModuleItemTests(SplitModuleTest):
         '''
         has_item(BlockUsageLocator)
         '''
+        course_id = 'GreekHero'
         # positive tests of various forms
         locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345')
-        self.assertTrue(modulestore().has_item(locator),
+        self.assertTrue(modulestore().has_item(course_id, locator),
                         "couldn't find in %s" % self.GUID_D1)
 
         locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', branch='draft')
         self.assertTrue(
-            modulestore().has_item(locator),
+            modulestore().has_item(course_id, locator),
             "couldn't find in 12345"
         )
         self.assertTrue(
-            modulestore().has_item(BlockUsageLocator(
+            modulestore().has_item(course_id, BlockUsageLocator(
                 course_id=locator.course_id,
                 branch='draft',
                 usage_id=locator.usage_id
@@ -276,7 +277,7 @@ class SplitModuleItemTests(SplitModuleTest):
             "couldn't find in draft 12345"
         )
         self.assertFalse(
-            modulestore().has_item(BlockUsageLocator(
+            modulestore().has_item(course_id, BlockUsageLocator(
                 course_id=locator.course_id,
                 branch='published',
                 usage_id=locator.usage_id)),
@@ -284,40 +285,41 @@ class SplitModuleItemTests(SplitModuleTest):
         )
         locator.branch = 'draft'
         self.assertTrue(
-            modulestore().has_item(locator),
+            modulestore().has_item(course_id, locator),
             "not found in draft 12345"
         )
 
         # not a course obj
         locator = BlockUsageLocator(course_id='GreekHero', usage_id='chapter1', branch='draft')
         self.assertTrue(
-            modulestore().has_item(locator),
+            modulestore().has_item(course_id, locator),
             "couldn't find chapter1"
         )
 
         # in published course
-        locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", branch='draft')
-        self.assertTrue(modulestore().has_item(BlockUsageLocator(course_id=locator.course_id,
-                                                                 usage_id=locator.usage_id,
-                                                                 branch='published')),
+        locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", revision='draft')
+        self.assertTrue(modulestore().has_item(course_id, BlockUsageLocator(course_id=locator.course_id,
+                                                                            usage_id=locator.usage_id,
+                                                                            revision='published')),
                         "couldn't find in 23456")
         locator.branch = 'published'
-        self.assertTrue(modulestore().has_item(locator), "couldn't find in 23456")
+        self.assertTrue(modulestore().has_item(course_id, locator), "couldn't find in 23456")
 
     def test_negative_has_item(self):
         # negative tests--not found
         # no such course or block
+        course_id = 'GreekHero'
         locator = BlockUsageLocator(course_id="doesnotexist", usage_id="head23456", branch='draft')
-        self.assertFalse(modulestore().has_item(locator))
+        self.assertFalse(modulestore().has_item(course_id, locator))
         locator = BlockUsageLocator(course_id="wonderful", usage_id="doesnotexist", branch='draft')
-        self.assertFalse(modulestore().has_item(locator))
+        self.assertFalse(modulestore().has_item(course_id, locator))
 
         # negative tests--insufficient specification
         self.assertRaises(InsufficientSpecificationError, BlockUsageLocator)
         self.assertRaises(InsufficientSpecificationError,
-                          modulestore().has_item, BlockUsageLocator(version_guid=self.GUID_D1))
+                          modulestore().has_item, None, BlockUsageLocator(version_guid=self.GUID_D1))
         self.assertRaises(InsufficientSpecificationError,
-                          modulestore().has_item, BlockUsageLocator(course_id='GreekHero'))
+                          modulestore().has_item, None, BlockUsageLocator(course_id='GreekHero'))
 
     def test_get_item(self):
         '''
@@ -737,13 +739,13 @@ class TestItemCrud(SplitModuleTest):
         deleted = BlockUsageLocator(course_id=reusable_location.course_id,
                                     branch=reusable_location.branch,
                                     usage_id=locn_to_del.usage_id)
-        self.assertFalse(modulestore().has_item(deleted))
-        self.assertRaises(VersionConflictError, modulestore().has_item, locn_to_del)
+        self.assertFalse(modulestore().has_item(reusable_location.course_id, deleted))
+        self.assertRaises(VersionConflictError, modulestore().has_item, reusable_location.course_id, locn_to_del)
         locator = BlockUsageLocator(
             version_guid=locn_to_del.version_guid,
             usage_id=locn_to_del.usage_id
         )
-        self.assertTrue(modulestore().has_item(locator))
+        self.assertTrue(modulestore().has_item(reusable_location.course_id, locator))
         self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid)
 
         # delete a subtree
@@ -754,7 +756,7 @@ class TestItemCrud(SplitModuleTest):
         def check_subtree(node):
             if node:
                 node_loc = node.location
-                self.assertFalse(modulestore().has_item(
+                self.assertFalse(modulestore().has_item(reusable_location.course_id,
                     BlockUsageLocator(
                         course_id=node_loc.course_id,
                         branch=node_loc.branch,
@@ -762,7 +764,7 @@ class TestItemCrud(SplitModuleTest):
                 locator = BlockUsageLocator(
                     version_guid=node.location.version_guid,
                     usage_id=node.location.usage_id)
-                self.assertTrue(modulestore().has_item(locator))
+                self.assertTrue(modulestore().has_item(reusable_location.course_id, locator))
                 if node.has_children:
                     for sub in node.get_children():
                         check_subtree(sub)
@@ -873,7 +875,7 @@ class TestCourseCreation(SplitModuleTest):
         original_course = modulestore().get_course(original_locator)
         self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
         self.assertFalse(
-            modulestore().has_item(BlockUsageLocator(
+            modulestore().has_item(new_draft_locator.course_id, BlockUsageLocator(
                 original_locator,
                 usage_id=new_item.location.usage_id
             ))

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -505,12 +505,12 @@ class XMLModuleStore(ModuleStoreBase):
         except KeyError:
             raise ItemNotFoundError(location)
 
-    def has_item(self, location):
+    def has_item(self, course_id, location):
         """
         Returns True if location exists in this ModuleStore.
         """
         location = Location(location)
-        return any(location in course_modules for course_modules in self.modules.values())
+        return location in self.modules[course_id]
 
     def get_item(self, location, depth=0):
         """

lms/envs/cms/mixed_dev.py

+"""
+This configuration is to run the MixedModuleStore on a localdev environment
+"""
+
+from .dev import *
+
+MODULESTORE = {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore',
+        'OPTIONS': {
+            'mappings': {
+                '6.002/a/a': 'xml',
+                '6.002/b/b': 'xml'
+            },
+            'stores': {
+                'xml': {
+                    'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
+                    'OPTIONS': {
+                        'data_dir': DATA_DIR,
+                        'default_class': 'xmodule.hidden_module.HiddenDescriptor',
+                    }
+                },
+                'default': {
+                    'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+                    'OPTIONS': {
+                        'default_class': 'xmodule.raw_module.RawDescriptor',
+                        'host': 'localhost',
+                        'db': 'xmodule',
+                        'collection': 'modulestore',
+                        'fs_root': DATA_DIR,
+                        'render_template': 'mitxmako.shortcuts.render_to_string',
+                    }
+                }
+            },
+        }
+    }
+}