Merge pull request #2631 from edx/dbarch/xml_ms_filter

Have xml modulestore only load course in course_ids if provided

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -44,6 +44,13 @@ class MixedModuleStore(ModuleStoreWriteBase):
             raise Exception('Missing a default modulestore in the MixedModuleStore __init__ method.')
 
         for key, store in stores.items():
+            is_xml = 'XMLModuleStore' in store['ENGINE']
+            if is_xml:
+                store['OPTIONS']['course_ids'] = [
+                    course_id
+                    for course_id, store_key in self.mappings.iteritems()
+                    if store_key == key
+                ]
             self.modulestores[key] = create_modulestore_instance(
                 store['ENGINE'],
                 # XMLModuleStore's don't have doc store configs

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

 # pylint: disable=E0611
 from nose.tools import assert_equals, assert_raises, assert_false, \
-    assert_true, assert_not_equals
+    assert_true, assert_not_equals, assert_in, assert_not_in
 # pylint: enable=E0611
 import pymongo
 from uuid import uuid4
@@ -218,6 +218,18 @@ class TestMixedModuleStore(object):
         assert_true(XML_COURSEID1 in course_ids)
         assert_true(XML_COURSEID2 in course_ids)
 
+    def test_xml_get_courses(self):
+        """
+        Test that the xml modulestore only loaded the courses from the maps.
+        """
+        courses = self.store.modulestores['xml'].get_courses()
+        assert_equals(len(courses), 2)
+        course_ids = [course.location.course_id for course in courses]
+        assert_in(XML_COURSEID1, course_ids)
+        assert_in(XML_COURSEID2, course_ids)
+        # this course is in the directory from which we loaded courses but not in the map
+        assert_not_in("edX/toy/TT_2012_Fall", course_ids)
+
     def test_get_course(self):
         module = self.store.get_course(IMPORT_COURSEID)
         assert_equals(module.location.course, self.import_course)

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -20,7 +20,6 @@ from xmodule.course_module import CourseDescriptor
 from xmodule.mako_module import MakoDescriptorSystem
 from xmodule.x_module import XMLParsingSystem, policy_key
 
-from xmodule.html_module import HtmlDescriptor
 from xblock.fields import ScopeIds
 from xblock.field_data import DictFieldData
 from xblock.runtime import DictKeyValueStore, IdReader, IdGenerator
@@ -348,7 +347,8 @@ class XMLModuleStore(ModuleStoreReadBase):
     """
     An XML backed ModuleStore
     """
-    def __init__(self, data_dir, default_class=None, course_dirs=None, load_error_modules=True, **kwargs):
+    def __init__(self, data_dir, default_class=None, course_dirs=None, course_ids=None,
+                 load_error_modules=True, **kwargs):
         """
         Initialize an XMLModuleStore from data_dir
 
@@ -357,8 +357,8 @@ class XMLModuleStore(ModuleStoreReadBase):
         default_class: dot-separated string defining the default descriptor
             class to use if none is specified in entry_points
 
-        course_dirs: If specified, the list of course_dirs to load. Otherwise,
-            load all course dirs
+        course_dirs or course_ids: If specified, the list of course_dirs or course_ids to load. Otherwise,
+            load all courses. Note, providing both
         """
         super(XMLModuleStore, self).__init__(**kwargs)
 
@@ -391,11 +391,12 @@ class XMLModuleStore(ModuleStoreReadBase):
             course_dirs = sorted([d for d in os.listdir(self.data_dir) if
                                   os.path.exists(self.data_dir / d / "course.xml")])
         for course_dir in course_dirs:
-            self.try_load_course(course_dir)
+            self.try_load_course(course_dir, course_ids)
 
-    def try_load_course(self, course_dir):
+    def try_load_course(self, course_dir, course_ids=None):
         '''
-        Load a course, keeping track of errors as we go along.
+        Load a course, keeping track of errors as we go along. If course_ids is not None,
+        then reject the course unless it's id is in course_ids.
         '''
         # Special-case code here, since we don't have a location for the
         # course before it loads.
@@ -404,21 +405,24 @@ class XMLModuleStore(ModuleStoreReadBase):
         errorlog = make_error_tracker()
         course_descriptor = None
         try:
-            course_descriptor = self.load_course(course_dir, errorlog.tracker)
+            course_descriptor = self.load_course(course_dir, course_ids, errorlog.tracker)
         except Exception as e:
             msg = "ERROR: Failed to load course '{0}': {1}".format(
                 course_dir.encode("utf-8"), unicode(e)
             )
             log.exception(msg)
             errorlog.tracker(msg)
+            self.errored_courses[course_dir] = errorlog
 
-        if course_descriptor is not None and not isinstance(course_descriptor, ErrorDescriptor):
+        if course_descriptor is None:
+            pass
+        elif isinstance(course_descriptor, ErrorDescriptor):
+            # Didn't load course.  Instead, save the errors elsewhere.
+            self.errored_courses[course_dir] = errorlog
+        else:
             self.courses[course_dir] = course_descriptor
             self._location_errors[course_descriptor.scope_ids.usage_id] = errorlog
             self.parent_trackers[course_descriptor.id].make_known(course_descriptor.scope_ids.usage_id)
-        else:
-            # Didn't load course.  Instead, save the errors elsewhere.
-            self.errored_courses[course_dir] = errorlog
 
     def __unicode__(self):
         '''
@@ -446,7 +450,7 @@ class XMLModuleStore(ModuleStoreReadBase):
             log.warning(msg + " " + str(err))
         return {}
 
-    def load_course(self, course_dir, tracker):
+    def load_course(self, course_dir, course_ids, tracker):
         """
         Load a course into this module store
         course_path: Course directory name
@@ -509,6 +513,8 @@ class XMLModuleStore(ModuleStoreReadBase):
                                      "(or 'name') set.  Set url_name.")
 
             course_id = CourseDescriptor.make_id(org, course, url_name)
+            if course_ids is not None and course_id not in course_ids:
+                return None
 
             def get_policy(usage_id):
                 """