Merge pull request #5212 from cpennington/split/import-export-performance

Improve performance of split-mongo import/export performance

cms/djangoapps/contentstore/tests/test_import.py

@@ -56,7 +56,7 @@ class ContentStoreImportTest(ModuleStoreTestCase):
             target_course_id=target_course_id,
             create_new_course_if_not_present=create_new_course_if_not_present,
         )
-        course_id = SlashSeparatedCourseKey('edX', 'test_import_course', '2012_Fall')
+        course_id = module_store.make_course_key('edX', 'test_import_course', '2012_Fall')
         course = module_store.get_course(course_id)
         self.assertIsNotNone(course)
 

common/lib/xmodule/.coveragerc

@@ -5,6 +5,9 @@ source = common/lib/xmodule
 
 [report]
 ignore_errors = True
+exclude_lines =
+    pragma: no cover
+    raise NotImplementedError
 
 [html]
 title = XModule Python Test Coverage Report

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -167,7 +167,7 @@ class MongoContentStore(ContentStore):
                     policy.setdefault(asset['asset_key'].name, {})[attr] = value
 
         with open(assets_policy_file, 'w') as f:
-            json.dump(policy, f)
+            json.dump(policy, f, sort_keys=True, indent=4)
 
     def get_all_content_thumbnails_for_course(self, course_key):
         return self._get_all_content_for_course(course_key, get_thumbnails=True)[0]

common/lib/xmodule/xmodule/fields.py

@@ -2,7 +2,7 @@ import time
 import logging
 import re
 
-from xblock.fields import Field
+from xblock.fields import JSONField
 import datetime
 import dateutil.parser
 
@@ -11,7 +11,7 @@ from pytz import UTC
 log = logging.getLogger(__name__)
 
 
-class Date(Field):
+class Date(JSONField):
     '''
     Date fields know how to parse and produce json (iso) compatible formats. Converts to tz aware datetimes.
     '''
@@ -85,7 +85,7 @@ class Date(Field):
 TIMEDELTA_REGEX = re.compile(r'^((?P<days>\d+?) day(?:s?))?(\s)?((?P<hours>\d+?) hour(?:s?))?(\s)?((?P<minutes>\d+?) minute(?:s)?)?(\s)?((?P<seconds>\d+?) second(?:s)?)?$')
 
 
-class Timedelta(Field):
+class Timedelta(JSONField):
     # Timedeltas are immutable, see http://docs.python.org/2/library/datetime.html#available-types
     MUTABLE = False
 
@@ -101,6 +101,10 @@ class Timedelta(Field):
         """
         if time_str is None:
             return None
+
+        if isinstance(time_str, datetime.timedelta):
+            return time_str
+
         parts = TIMEDELTA_REGEX.match(time_str)
         if not parts:
             return
@@ -112,6 +116,9 @@ class Timedelta(Field):
         return datetime.timedelta(**time_params)
 
     def to_json(self, value):
+        if value is None:
+            return None
+
         values = []
         for attr in ('days', 'hours', 'minutes', 'seconds'):
             cur_value = getattr(value, attr, 0)
@@ -129,7 +136,7 @@ class Timedelta(Field):
         return self.from_json(value)
 
 
-class RelativeTime(Field):
+class RelativeTime(JSONField):
     """
     Field for start_time and end_time video module properties.
 
@@ -182,6 +189,9 @@ class RelativeTime(Field):
         if not value:
             return datetime.timedelta(seconds=0)
 
+        if isinstance(value, datetime.timedelta):
+            return value
+
         # We've seen serialized versions of float in this field
         if isinstance(value, float):
             return datetime.timedelta(seconds=value)

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -634,7 +634,7 @@ class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead):
         '''
         Set up the error-tracking logic.
         '''
-        super(ModuleStoreReadBase, self).__init__()
+        super(ModuleStoreReadBase, self).__init__(**kwargs)
         self._course_errors = defaultdict(make_error_tracker)  # location -> ErrorLog
         # TODO move the inheritance_cache_subsystem to classes which use it
         self.metadata_inheritance_cache_subsystem = metadata_inheritance_cache_subsystem

common/lib/xmodule/xmodule/modulestore/django.py

@@ -18,6 +18,8 @@ import threading
 
 from xmodule.util.django import get_current_request_hostname
 import xmodule.modulestore  # pylint: disable=unused-import
+from xmodule.modulestore.mixed import MixedModuleStore
+from xmodule.modulestore.draft_and_published import BranchSettingMixin
 from xmodule.contentstore.django import contentstore
 import xblock.reference.plugins
 
@@ -66,6 +68,12 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options
     except InvalidCacheBackendError:
         metadata_inheritance_cache = get_cache('default')
 
+    if issubclass(class_, MixedModuleStore):
+        _options['create_modulestore_instance'] = create_modulestore_instance
+
+    if issubclass(class_, BranchSettingMixin):
+        _options['branch_setting_func'] = _get_modulestore_branch_setting
+
     return class_(
         contentstore=content_store,
         metadata_inheritance_cache_subsystem=metadata_inheritance_cache,
@@ -75,8 +83,6 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options
         doc_store_config=doc_store_config,
         i18n_service=i18n_service or ModuleI18nService(),
         fs_service=fs_service or xblock.reference.plugins.FSService(),
-        branch_setting_func=_get_modulestore_branch_setting,
-        create_modulestore_instance=create_modulestore_instance,
         **_options
     )
 

common/lib/xmodule/xmodule/modulestore/draft_and_published.py

@@ -25,11 +25,11 @@ class BranchSettingMixin(object):
         :param branch_setting_func: a function that returns the default branch setting for this object.
             If not specified, ModuleStoreEnum.Branch.published_only is used as the default setting.
         """
-        super(BranchSettingMixin, self).__init__(*args, **kwargs)
         self.default_branch_setting_func = kwargs.pop(
             'branch_setting_func',
             lambda: ModuleStoreEnum.Branch.published_only
         )
+        super(BranchSettingMixin, self).__init__(*args, **kwargs)
 
         # cache the branch setting on a local thread to support a multi-threaded environment
         self.thread_cache = threading.local()
@@ -69,9 +69,6 @@ class ModuleStoreDraftAndPublished(BranchSettingMixin):
     """
     __metaclass__ = ABCMeta
 
-    def __init__(self, *args, **kwargs):
-        super(ModuleStoreDraftAndPublished, self).__init__(*args, **kwargs)
-
     @abstractmethod
     def delete_item(self, location, user_id, revision=None, **kwargs):
         raise NotImplementedError

common/lib/xmodule/xmodule/modulestore/exceptions.py

@@ -74,7 +74,9 @@ class DuplicateCourseError(Exception):
         """
         existing_entry will have the who, when, and other properties of the existing entry
         """
-        super(DuplicateCourseError, self).__init__()
+        super(DuplicateCourseError, self).__init__(
+            u'Cannot create course {}, which duplicates {}'.format(course_id, existing_entry)
+        )
         self.course_id = course_id
         self.existing_entry = existing_entry
 
@@ -84,9 +86,6 @@ class InvalidBranchSetting(Exception):
     Raised when the process' branch setting did not match the required setting for the attempted operation on a store.
     """
     def __init__(self, expected_setting, actual_setting):
-        super(InvalidBranchSetting, self).__init__()
+        super(InvalidBranchSetting, self).__init__(u"Invalid branch: expected {} but got {}".format(expected_setting, actual_setting))
         self.expected_setting = expected_setting
         self.actual_setting = actual_setting
-
-    def __unicode__(self, *args, **kwargs):
-        return u"Invalid branch: expected {} but got {}".format(self.expected_setting, self.actual_setting)

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -214,11 +214,19 @@ class InheritingFieldData(KvsFieldData):
         """
         The default for an inheritable name is found on a parent.
         """
-        if name in self.inheritable_names and block.parent is not None:
-            parent = block.get_parent()
-            if parent:
-                return getattr(parent, name)
-        super(InheritingFieldData, self).default(block, name)
+        if name in self.inheritable_names:
+            # Walk up the content tree to find the first ancestor
+            # that this field is set on. Use the field from the current
+            # block so that if it has a different default than the root
+            # node of the tree, the block's default will be used.
+            field = block.fields[name]
+            ancestor = block.get_parent()
+            while ancestor is not None:
+                if field.is_set_on(ancestor):
+                    return field.read_json(ancestor)
+                else:
+                    ancestor = ancestor.get_parent()
+        return super(InheritingFieldData, self).default(block, name)
 
 
 def inheriting_field_data(kvs):

common/lib/xmodule/xmodule/modulestore/split_mongo/__init__.py

@@ -20,3 +20,5 @@ class BlockKey(namedtuple('BlockKey', 'type id')):
     def from_usage_key(cls, usage_key):
         return cls(usage_key.block_type, usage_key.block_id)
 
+
+CourseEnvelope = namedtuple('CourseEnvelope', 'course_key structure')

common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py

 import sys
 import logging
 from contracts import contract, new_contract
+from lazy import lazy
 from xblock.runtime import KvsFieldData
 from xblock.fields import ScopeIds
 from opaque_keys.edx.locator import BlockUsageLocator, LocalId, CourseLocator, DefinitionLocator
@@ -12,12 +13,14 @@ from .split_mongo_kvs import SplitMongoKVS
 from fs.osfs import OSFS
 from .definition_lazy_loader import DefinitionLazyLoader
 from xmodule.modulestore.edit_info import EditInfoRuntimeMixin
-from xmodule.modulestore.split_mongo import BlockKey
+from xmodule.modulestore.inheritance import inheriting_field_data, InheritanceMixin
+from xmodule.modulestore.split_mongo import BlockKey, CourseEnvelope
 
 log = logging.getLogger(__name__)
 
 new_contract('BlockUsageLocator', BlockUsageLocator)
 new_contract('BlockKey', BlockKey)
+new_contract('CourseEnvelope', CourseEnvelope)
 
 
 class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
@@ -27,6 +30,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
 
     Computes the settings (nee 'metadata') inheritance upon creation.
     """
+    @contract(course_entry=CourseEnvelope)
     def __init__(self, modulestore, course_entry, default_class, module_data, lazy, **kwargs):
         """
         Computes the settings inheritance and sets up the cache.
@@ -34,8 +38,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
         modulestore: the module store that can be used to retrieve additional
         modules
 
-        course_entry: the originally fetched enveloped course_structure w/ branch and course id info
-        plus a dictionary of cached inherited_settings indexed by (block_type, block_id) tuple.
+        course_entry: the originally fetched enveloped course_structure w/ branch and course id info.
         Callers to _load_item provide an override but that function ignores the provided structure and
         only looks at the branch and course id
 
@@ -43,10 +46,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
             underlying modulestore
         """
         # needed by capa_problem (as runtime.filestore via this.resources_fs)
-        if 'course' in course_entry:
-            root = modulestore.fs_root / course_entry['org'] / course_entry['course'] / course_entry['run']
+        if course_entry.course_key.course:
+            root = modulestore.fs_root / course_entry.course_key.org / course_entry.course_key.course / course_entry.course_key.run
         else:
-            root = modulestore.fs_root / course_entry['structure']['_id']
+            root = modulestore.fs_root / course_entry.structure['_id']
         root.makedirs_p()  # create directory if it doesn't exist
 
         super(CachingDescriptorSystem, self).__init__(
@@ -59,16 +62,19 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
         self.course_entry = course_entry
         self.lazy = lazy
         self.module_data = module_data
-        # Compute inheritance
-        modulestore.inherit_settings(
-            course_entry['structure'].get('blocks', {}),
-            course_entry['structure'].get('root'),
-            course_entry.setdefault('inherited_settings', {}),
-        )
         self.default_class = default_class
         self.local_modules = {}
 
-    @contract(usage_key="BlockUsageLocator | BlockKey")
+    @lazy
+    @contract(returns="dict(BlockKey: BlockKey)")
+    def _parent_map(self):
+        parent_map = {}
+        for block_key, block in self.course_entry.structure['blocks'].iteritems():
+            for child in block['fields'].get('children', []):
+                parent_map[child] = block_key
+        return parent_map
+
+    @contract(usage_key="BlockUsageLocator | BlockKey", course_entry_override="CourseEnvelope | None")
     def _load_item(self, usage_key, course_entry_override=None, **kwargs):
         # usage_key is either a UsageKey or just the block_key. if a usage_key,
         if isinstance(usage_key, BlockUsageLocator):
@@ -88,20 +94,17 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
             block_key = usage_key
 
             course_info = course_entry_override or self.course_entry
-            course_key = CourseLocator(
-                version_guid=course_info['structure']['_id'],
-                org=course_info.get('org'),
-                course=course_info.get('course'),
-                run=course_info.get('run'),
-                branch=course_info.get('branch'),
-            )
+            course_key = course_info.course_key
+
+        if course_entry_override:
+            structure_id = course_entry_override.structure.get('_id')
+        else:
+            structure_id = self.course_entry.structure.get('_id')
 
         json_data = self.get_module_data(block_key, course_key)
 
         class_ = self.load_block_type(json_data.get('block_type'))
-        # pass None for inherited_settings to signal that it should get the settings from cache
-        new_item = self.xblock_from_json(class_, course_key, block_key, json_data, None, course_entry_override, **kwargs)
-        return new_item
+        return self.xblock_from_json(class_, course_key, block_key, json_data, course_entry_override, **kwargs)
 
     @contract(block_key=BlockKey, course_key=CourseLocator)
     def get_module_data(self, block_key, course_key):
@@ -134,36 +137,29 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
     # is the intended one when not given a course_entry_override; thus, the caching of the last branch/course id.
     @contract(block_key="BlockKey | None")
     def xblock_from_json(
-        self, class_, course_key, block_key, json_data, inherited_settings, course_entry_override=None, **kwargs
+        self, class_, course_key, block_key, json_data, course_entry_override=None, **kwargs
     ):
         if course_entry_override is None:
             course_entry_override = self.course_entry
         else:
             # most recent retrieval is most likely the right one for next caller (see comment above fn)
-            self.course_entry['branch'] = course_entry_override['branch']
-            self.course_entry['org'] = course_entry_override['org']
-            self.course_entry['course'] = course_entry_override['course']
-            self.course_entry['run'] = course_entry_override['run']
+            self.course_entry = CourseEnvelope(course_entry_override.course_key, self.course_entry.structure)
 
         definition_id = json_data.get('definition')
 
         # If no usage id is provided, generate an in-memory id
         if block_key is None:
             block_key = BlockKey(json_data['block_type'], LocalId())
-        else:
-            if inherited_settings is None:
-                # see if there's a value in course_entry
-                if block_key in self.course_entry['inherited_settings']:
-                    inherited_settings = self.course_entry['inherited_settings'][block_key]
-            elif block_key not in self.course_entry['inherited_settings']:
-                self.course_entry['inherited_settings'][block_key] = inherited_settings
 
         if definition_id is not None and not json_data.get('definition_loaded', False):
             definition_loader = DefinitionLazyLoader(
-                self.modulestore, block_key.type, definition_id,
+                self.modulestore,
+                course_key,
+                block_key.type,
+                definition_id,
                 lambda fields: self.modulestore.convert_references_to_keys(
                     course_key, self.load_block_type(block_key.type),
-                    fields, self.course_entry['structure']['blocks'],
+                    fields, self.course_entry.structure['blocks'],
                 )
             )
         else:
@@ -180,14 +176,23 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
         )
 
         converted_fields = self.modulestore.convert_references_to_keys(
-            block_locator.course_key, class_, json_data.get('fields', {}), self.course_entry['structure']['blocks'],
+            block_locator.course_key, class_, json_data.get('fields', {}), self.course_entry.structure['blocks'],
         )
+        if block_key in self._parent_map:
+            parent_key = self._parent_map[block_key]
+            parent = course_key.make_usage_key(parent_key.type, parent_key.id)
+        else:
+            parent = None
         kvs = SplitMongoKVS(
             definition_loader,
             converted_fields,
-            inherited_settings,
-            **kwargs
+            parent=parent,
+            field_decorator=kwargs.get('field_decorator')
         )
+
+        if InheritanceMixin in self.modulestore.xblock_mixins:
+            field_data = inheriting_field_data(kvs)
+        else:
             field_data = KvsFieldData(kvs)
 
         try:
@@ -201,8 +206,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin):
             return ErrorDescriptor.from_json(
                 json_data,
                 self,
-                BlockUsageLocator(
-                    CourseLocator(version_guid=course_entry_override['structure']['_id']),
+                course_entry_override.course_key.make_usage_key(
                     block_type='error',
                     block_id=block_key.id
                 ),

common/lib/xmodule/xmodule/modulestore/split_mongo/definition_lazy_loader.py

@@ -8,13 +8,14 @@ class DefinitionLazyLoader(object):
     object doesn't force access during init but waits until client wants the
     definition. Only works if the modulestore is a split mongo store.
     """
-    def __init__(self, modulestore, block_type, definition_id, field_converter):
+    def __init__(self, modulestore, course_key, block_type, definition_id, field_converter):
         """
         Simple placeholder for yet-to-be-fetched data
         :param modulestore: the pymongo db connection with the definitions
         :param definition_locator: the id of the record in the above to fetch
         """
         self.modulestore = modulestore
+        self.course_key = course_key
         self.definition_locator = DefinitionLocator(block_type, definition_id)
         self.field_converter = field_converter
 
@@ -23,4 +24,4 @@ class DefinitionLazyLoader(object):
         Fetch the definition. Note, the caller should replace this lazy
         loader pointer with the result so as not to fetch more than once
         """
-        return self.modulestore.db_connection.get_definition(self.definition_locator.definition_id)
+        return self.modulestore.get_definition(self.course_key, self.definition_locator.definition_id)

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -3,7 +3,14 @@ Segregation of pymongo functions from the data modeling mechanisms for split mod
 """
 import re
 import pymongo
+import time
+
+# Import this just to export it
+from pymongo.errors import DuplicateKeyError  # pylint: disable=unused-import
+
 from contracts import check
+from functools import wraps
+from pymongo.errors import AutoReconnect
 from xmodule.exceptions import HeartbeatFailure
 from xmodule.modulestore.split_mongo import BlockKey
 from datetime import tzinfo
@@ -62,6 +69,32 @@ def structure_to_mongo(structure):
     return new_structure
 
 
+def autoretry_read(wait=0.1, retries=5):
+    """
+    Automatically retry a read-only method in the case of a pymongo
+    AutoReconnect exception.
+
+    See http://emptysqua.re/blog/save-the-monkey-reliably-writing-to-mongodb/
+    for a discussion of this technique.
+    """
+    def decorate(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            for attempt in xrange(retries):
+                try:
+                    return fn(*args, **kwargs)
+                    break
+                except AutoReconnect:
+                    # Reraise if we failed on our last attempt
+                    if attempt == retries - 1:
+                        raise
+
+                    if wait:
+                        time.sleep(wait)
+        return wrapper
+    return decorate
+
+
 class MongoConnection(object):
     """
     Segregation of pymongo functions from the data modeling mechanisms for split modulestore.
@@ -106,12 +139,14 @@ class MongoConnection(object):
         else:
             raise HeartbeatFailure("Can't connect to {}".format(self.database.name))
 
+    @autoretry_read()
     def get_structure(self, key):
         """
         Get the structure from the persistence mechanism whose id is the given key
         """
         return structure_from_mongo(self.structures.find_one({'_id': key}))
 
+    @autoretry_read()
     def find_structures_by_id(self, ids):
         """
         Return all structures that specified in ``ids``.
@@ -121,6 +156,7 @@ class MongoConnection(object):
         """
         return [structure_from_mongo(structure) for structure in self.structures.find({'_id': {'$in': ids}})]
 
+    @autoretry_read()
     def find_structures_derived_from(self, ids):
         """
         Return all structures that were immediately derived from a structure listed in ``ids``.
@@ -130,6 +166,7 @@ class MongoConnection(object):
         """
         return [structure_from_mongo(structure) for structure in self.structures.find({'previous_version': {'$in': ids}})]
 
+    @autoretry_read()
     def find_ancestor_structures(self, original_version, block_key):
         """
         Find all structures that originated from ``original_version`` that contain ``block_key``.
@@ -149,12 +186,13 @@ class MongoConnection(object):
             }
         })]
 
-    def upsert_structure(self, structure):
+    def insert_structure(self, structure):
         """
-        Update the db record for structure, creating that record if it doesn't already exist
+        Insert a new structure into the database.
         """
-        self.structures.update({'_id': structure['_id']}, structure_to_mongo(structure), upsert=True)
+        self.structures.insert(structure_to_mongo(structure))
 
+    @autoretry_read()
     def get_course_index(self, key, ignore_case=False):
         """
         Get the course_index from the persistence mechanism whose id is the given key
@@ -171,6 +209,7 @@ class MongoConnection(object):
             }
         return self.course_index.find_one(query)
 
+    @autoretry_read()
     def find_matching_course_indexes(self, branch=None, search_targets=None):
         """
         Find the course_index matching particular conditions.
@@ -229,18 +268,19 @@ class MongoConnection(object):
             'run': course_index['run'],
         })
 
+    @autoretry_read()
     def get_definition(self, key):
         """
         Get the definition from the persistence mechanism whose id is the given key
         """
         return self.definitions.find_one({'_id': key})
 
-    def find_matching_definitions(self, query):
+    @autoretry_read()
+    def get_definitions(self, definitions):
         """
-        Find the definitions matching the query. Right now the query must be a legal mongo query
-        :param query: a mongo-style query of {key: [value|{$in ..}|..], ..}
+        Retrieve all definitions listed in `definitions`.
         """
-        return self.definitions.find(query)
+        return self.definitions.find({'$in': {'_id': definitions}})
 
     def insert_definition(self, definition):
         """

common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py

@@ -242,7 +242,7 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
         :return: True if the draft and published versions differ
         """
         def get_course(branch_name):
-            return self._lookup_course(xblock.location.course_key.for_branch(branch_name))['structure']
+            return self._lookup_course(xblock.location.course_key.for_branch(branch_name)).structure
 
         def get_block(course_structure, block_key):
             return self._get_block_from_structure(course_structure, block_key)
@@ -318,7 +318,7 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
             # get head version of Published branch
             published_course_structure = self._lookup_course(
                 location.course_key.for_branch(ModuleStoreEnum.BranchName.published)
-            )['structure']
+            ).structure
             published_block = self._get_block_from_structure(
                 published_course_structure,
                 BlockKey.from_usage_key(location)
@@ -327,7 +327,7 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
                 raise InvalidVersionError(location)
 
             # create a new versioned draft structure
-            draft_course_structure = self._lookup_course(draft_course_key)['structure']
+            draft_course_structure = self._lookup_course(draft_course_key).structure
             new_structure = self.version_structure(draft_course_key, draft_course_structure, user_id)
 
             # remove the block and its descendants from the new structure
@@ -394,7 +394,7 @@ class DraftVersioningModuleStore(ModuleStoreDraftAndPublished, SplitMongoModuleS
         pass
 
     def _get_head(self, xblock, branch):
-        course_structure = self._lookup_course(xblock.location.course_key.for_branch(branch))['structure']
+        course_structure = self._lookup_course(xblock.location.course_key.for_branch(branch)).structure
         return self._get_block_from_structure(course_structure, BlockKey.from_usage_key(xblock.location))
 
     def _get_version(self, block):

common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py

 import copy
+from contracts import contract, new_contract
 from xblock.fields import Scope
 from collections import namedtuple
 from xblock.exceptions import InvalidScopeError
 from .definition_lazy_loader import DefinitionLazyLoader
 from xmodule.modulestore.inheritance import InheritanceKeyValueStore
+from opaque_keys.edx.locator import BlockUsageLocator
 
 # id is a BlockUsageLocator, def_id is the definition's guid
 SplitMongoKVSid = namedtuple('SplitMongoKVSid', 'id, def_id')
+new_contract('BlockUsageLocator', BlockUsageLocator)
 
 
 class SplitMongoKVS(InheritanceKeyValueStore):
@@ -15,22 +18,25 @@ class SplitMongoKVS(InheritanceKeyValueStore):
     known to the MongoModuleStore (data, children, and metadata)
     """
 
-    def __init__(self, definition, initial_values, inherited_settings, **kwargs):
+    @contract(parent="BlockUsageLocator | None")
+    def __init__(self, definition, initial_values, parent, field_decorator=None):
         """
 
         :param definition: either a lazyloader or definition id for the definition
         :param initial_values: a dictionary of the locally set values
-        :param inherited_settings: the json value of each inheritable field from above this.
-            Note, local fields may override and disagree w/ this b/c this says what the value
-            should be if the field is undefined.
         """
         # deepcopy so that manipulations of fields does not pollute the source
-        super(SplitMongoKVS, self).__init__(copy.deepcopy(initial_values), inherited_settings)
+        super(SplitMongoKVS, self).__init__(copy.deepcopy(initial_values))
         self._definition = definition  # either a DefinitionLazyLoader or the db id of the definition.
         # if the db id, then the definition is presumed to be loaded into _fields
 
         # a decorator function for field values (to be called when a field is accessed)
-        self.field_decorator = kwargs.get('field_decorator', lambda x: x)
+        if field_decorator is None:
+            self.field_decorator = lambda x: x
+        else:
+            self.field_decorator = field_decorator
+
+        self.parent = parent
 
 
     def get(self, key):
@@ -38,8 +44,7 @@ class SplitMongoKVS(InheritanceKeyValueStore):
         if key.field_name not in self._fields:
             # parent undefined in editing runtime (I think)
             if key.scope == Scope.parent:
-                # see STUD-624. Right now copies MongoKeyValueStore.get's behavior of returning None
-                return None
+                return self.parent
             if key.scope == Scope.children:
                 # didn't find children in _fields; so, see if there's a default
                 raise KeyError()

common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py

@@ -28,6 +28,8 @@ from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.xml_exporter import export_to_xml
 from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleStore
 from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
+from xmodule.modulestore.inheritance import InheritanceMixin
+from xmodule.x_module import XModuleMixin
 
 
 COMMON_DOCSTORE_CONFIG = {
@@ -36,6 +38,9 @@ COMMON_DOCSTORE_CONFIG = {
 }
 
 
+XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin)
+
+
 class MemoryCache(object):
     """
     This fits the metadata_inheritance_cache_subsystem interface used by
@@ -95,6 +100,7 @@ class MongoModulestoreBuilder(object):
             render_template=repr,
             branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
             metadata_inheritance_cache_subsystem=MemoryCache(),
+            xblock_mixins=XBLOCK_MIXINS,
         )
         modulestore.ensure_indexes()
 
@@ -139,6 +145,7 @@ class VersioningModulestoreBuilder(object):
             doc_store_config,
             fs_root,
             render_template=repr,
+            xblock_mixins=XBLOCK_MIXINS,
         )
         modulestore.ensure_indexes()
 
@@ -189,7 +196,13 @@ class MixedModulestoreBuilder(object):
             # Generate a fake list of stores to give the already generated stores appropriate names
             stores = [{'NAME': name, 'ENGINE': 'This space deliberately left blank'} for name in names]
 
-            modulestore = MixedModuleStore(contentstore, self.mappings, stores, create_modulestore_instance=create_modulestore_instance)
+            modulestore = MixedModuleStore(
+                contentstore,
+                self.mappings,
+                stores,
+                create_modulestore_instance=create_modulestore_instance,
+                xblock_mixins=XBLOCK_MIXINS,
+            )
 
             yield modulestore
 
@@ -269,8 +282,8 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest):
                 with dest_content_builder.build() as dest_content:
                     # Construct the modulestore for storing the second import (using the second contentstore)
                     with dest_builder.build(dest_content) as dest_store:
-                        source_course_key = source_store.make_course_key('source', 'course', 'key')
-                        dest_course_key = dest_store.make_course_key('dest', 'course', 'key')
+                        source_course_key = source_store.make_course_key('a', 'course', 'course')
+                        dest_course_key = dest_store.make_course_key('a', 'course', 'course')
 
                         import_from_xml(
                             source_store,
@@ -287,20 +300,30 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest):
                             source_content,
                             source_course_key,
                             self.export_dir,
-                            'exported_course',
+                            'exported_source_course',
                         )
 
                         import_from_xml(
                             dest_store,
                             'test_user',
                             self.export_dir,
+                            course_dirs=['exported_source_course'],
                             static_content_store=dest_content,
                             target_course_id=dest_course_key,
                             create_new_course_if_not_present=True,
                         )
 
+                        export_to_xml(
+                            dest_store,
+                            dest_content,
+                            dest_course_key,
+                            self.export_dir,
+                            'exported_dest_course',
+                        )
+
                         self.exclude_field(None, 'wiki_slug')
                         self.exclude_field(None, 'xml_attributes')
+                        self.exclude_field(None, 'parent')
                         self.ignore_asset_key('_id')
                         self.ignore_asset_key('uploadDate')
                         self.ignore_asset_key('content_son')

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -18,6 +18,7 @@ from uuid import uuid4
 # TODO remove this import and the configuration -- xmodule should not depend on django!
 from django.conf import settings
 from xmodule.modulestore.edit_info import EditInfoMixin
+from xmodule.modulestore.inheritance import InheritanceMixin
 
 if not settings.configured:
     settings.configure()
@@ -26,17 +27,17 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
 from xmodule.exceptions import InvalidVersionError
 from xmodule.modulestore import ModuleStoreEnum
-from xmodule.modulestore.draft_and_published import UnsupportedRevisionError
+from xmodule.modulestore.draft_and_published import UnsupportedRevisionError, ModuleStoreDraftAndPublished
 from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateCourseError, ReferentialIntegrityError, NoPathToItem
 from xmodule.modulestore.mixed import MixedModuleStore
 from xmodule.modulestore.search import path_to_location
 from xmodule.modulestore.tests.factories import check_mongo_calls
 from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
-from xmodule.tests import DATA_DIR
+from xmodule.tests import DATA_DIR, CourseComparisonTest
 
 
 @ddt.ddt
-class TestMixedModuleStore(unittest.TestCase):
+class TestMixedModuleStore(CourseComparisonTest):
     """
     Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
     Location-based dbs)
@@ -58,7 +59,7 @@ class TestMixedModuleStore(unittest.TestCase):
         'default_class': DEFAULT_CLASS,
         'fs_root': DATA_DIR,
         'render_template': RENDER_TEMPLATE,
-        'xblock_mixins': (EditInfoMixin,)
+        'xblock_mixins': (EditInfoMixin, InheritanceMixin),
     }
     DOC_STORE_CONFIG = {
         'host': HOST,
@@ -244,7 +245,8 @@ class TestMixedModuleStore(unittest.TestCase):
             for course_id, course_key in self.course_locations.iteritems()  # pylint: disable=maybe-no-member
         }
 
-        self.fake_location = self.course_locations[self.MONGO_COURSEID].course_key.make_usage_key('vertical', 'fake')
+        mongo_course_key = self.course_locations[self.MONGO_COURSEID].course_key
+        self.fake_location = self.store.make_course_key(mongo_course_key.org, mongo_course_key.course, mongo_course_key.run).make_usage_key('vertical', 'fake')
 
         self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
             category='chapter', name='Overview'
@@ -1046,7 +1048,7 @@ class TestMixedModuleStore(unittest.TestCase):
         self.store.revert_to_published(self.vertical_x1a, self.user_id)
         reverted_parent = self.store.get_item(self.vertical_x1a)
         self.assertEqual(vertical_children_num, len(published_parent.children))
-        self.assertEqual(reverted_parent, published_parent)
+        self.assertBlocksEqualByFields(reverted_parent, published_parent)
         self.assertFalse(self._has_changes(self.vertical_x1a))
 
     @ddt.data('draft', 'split')
@@ -1081,7 +1083,8 @@ class TestMixedModuleStore(unittest.TestCase):
         orig_vertical = self.store.get_item(self.vertical_x1a)
         self.store.revert_to_published(self.vertical_x1a, self.user_id)
         reverted_vertical = self.store.get_item(self.vertical_x1a)
-        self.assertEqual(orig_vertical, reverted_vertical)
+
+        self.assertBlocksEqualByFields(orig_vertical, reverted_vertical)
 
     @ddt.data('draft', 'split')
     def test_revert_to_published_no_published(self, default_ms):
@@ -1787,9 +1790,11 @@ def create_modulestore_instance(engine, contentstore, doc_store_config, options,
     """
     class_ = load_function(engine)
 
+    if issubclass(class_, ModuleStoreDraftAndPublished):
+        options['branch_setting_func'] = lambda: ModuleStoreEnum.Branch.draft_preferred
+
     return class_(
         doc_store_config=doc_store_config,
         contentstore=contentstore,
-        branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
         **options
     )

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -10,6 +10,7 @@ from contracts import contract
 from importlib import import_module
 from path import path
 
+from xblock.fields import Reference, ReferenceList, ReferenceValueDict
 from xmodule.course_module import CourseDescriptor
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.exceptions import (
@@ -1592,7 +1593,7 @@ class TestInheritance(SplitModuleTest):
 
         # unset on parent, retrieve child, verify unset
         chapter = modulestore().get_item(chapter.location.version_agnostic())
-        chapter.fields['visible_to_staff_only'].delete_from(chapter)
+        del chapter.visible_to_staff_only
         modulestore().update_item(chapter, self.user_id)
 
         problem = modulestore().get_item(problem.location.version_agnostic())
@@ -1756,12 +1757,26 @@ class TestPublish(SplitModuleTest):
             for field in source.fields.values():
                 if field.name == 'children':
                     self._compare_children(field.read_from(source), field.read_from(pub_copy), unexpected_blocks)
+                elif isinstance(field, (Reference, ReferenceList, ReferenceValueDict)):
+                    self.assertReferenceEqual(field.read_from(source), field.read_from(pub_copy))
                 else:
                     self.assertEqual(field.read_from(source), field.read_from(pub_copy))
         for unexp in unexpected_blocks:
             with self.assertRaises(ItemNotFoundError):
                 modulestore().get_item(dest_course_loc.make_usage_key(unexp.type, unexp.id))
 
+    def assertReferenceEqual(self, expected, actual):
+        if isinstance(expected, BlockUsageLocator):
+            expected = BlockKey.from_usage_key(expected)
+            actual = BlockKey.from_usage_key(actual)
+        elif isinstance(expected, list):
+            expected = [BlockKey.from_usage_key(key) for key in expected]
+            actual = [BlockKey.from_usage_key(key) for key in actual]
+        elif isinstance(expected, dict):
+            expected = {key: BlockKey.from_usage_key(val) for (key, val) in expected}
+            actual = {key: BlockKey.from_usage_key(val) for (key, val) in actual}
+        self.assertEqual(expected, actual)
+
     @contract(
         source_children="list(BlockUsageLocator)",
         dest_children="list(BlockUsageLocator)",

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -370,7 +370,7 @@ class XMLModuleStore(ModuleStoreReadBase):
     """
     def __init__(
         self, data_dir, default_class=None, course_dirs=None, course_ids=None,
-        load_error_modules=True, i18n_service=None, pyfs_service=None, **kwargs
+        load_error_modules=True, i18n_service=None, fs_service=None, **kwargs
     ):
         """
         Initialize an XMLModuleStore from data_dir
@@ -409,7 +409,7 @@ class XMLModuleStore(ModuleStoreReadBase):
         self.field_data = inheriting_field_data(kvs=DictKeyValueStore())
 
         self.i18n_service = i18n_service
-        self.pyfs_service = pyfs_service
+        self.fs_service = fs_service
 
         # If we are specifically asked for missing courses, that should
         # be an error.  If we are asked for "all" courses, find the ones
@@ -555,8 +555,8 @@ class XMLModuleStore(ModuleStoreReadBase):
             if self.i18n_service:
                 services['i18n'] = self.i18n_service
 
-            if self.pyfs_service:
-                services['fs'] = self.pyfs_service
+            if self.fs_service:
+                services['fs'] = self.fs_service
 
             system = ImportSystem(
                 xmlstore=self,

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

@@ -37,6 +37,8 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
     `course_dir`: The name of the directory inside `root_dir` to write the course content to
     """
 
+    with modulestore.bulk_operations(course_key):
+
         course = modulestore.get_course(course_key, depth=None)  # None means infinite
         fsm = OSFS(root_dir)
         export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
@@ -83,26 +85,26 @@ def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
                         course_image_file.write(course_image.data)
 
         # export the static tabs
-    export_extra_content(export_fs, modulestore, xml_centric_course_key, 'static_tab', 'tabs', '.html')
+        export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'static_tab', 'tabs', '.html')
 
         # export the custom tags
-    export_extra_content(export_fs, modulestore, xml_centric_course_key, 'custom_tag_template', 'custom_tags')
+        export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'custom_tag_template', 'custom_tags')
 
         # export the course updates
-    export_extra_content(export_fs, modulestore, xml_centric_course_key, 'course_info', 'info', '.html')
+        export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'course_info', 'info', '.html')
 
         # export the 'about' data (e.g. overview, etc.)
-    export_extra_content(export_fs, modulestore, xml_centric_course_key, 'about', 'about', '.html')
+        export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'about', 'about', '.html')
 
         # export the grading policy
         course_run_policy_dir = policies_dir.makeopendir(course.location.name)
         with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
-        grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder))
+            grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
 
         # export all of the course metadata in policy.json
         with course_run_policy_dir.open('policy.json', 'w') as course_policy:
             policy = {'course/' + course.location.name: own_metadata(course)}
-        course_policy.write(dumps(policy, cls=EdxJSONEncoder))
+            course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
 
         #### DRAFTS ####
         # xml backed courses don't support drafts!
@@ -178,16 +180,16 @@ def _export_field_content(xblock_item, item_dir):
                 # filename format: {dirname}.{field_name}.json
                 with item_dir.open('{0}.{1}.{2}'.format(xblock_item.location.name, field_name, 'json'),
                                    'w') as field_content_file:
-                    field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder))
+                    field_content_file.write(dumps(module_data.get(field_name, {}), cls=EdxJSONEncoder, sort_keys=True, indent=4))
 
 
-def export_extra_content(export_fs, modulestore, course_key, category_type, dirname, file_suffix=''):
-    items = modulestore.get_items(course_key, qualifiers={'category': category_type})
+def export_extra_content(export_fs, modulestore, source_course_key, dest_course_key, category_type, dirname, file_suffix=''):
+    items = modulestore.get_items(source_course_key, qualifiers={'category': category_type})
 
     if len(items) > 0:
         item_dir = export_fs.makeopendir(dirname)
         for item in items:
-            adapt_references(item, course_key, export_fs)
+            adapt_references(item, dest_course_key, export_fs)
             with item_dir.open(item.location.name + file_suffix, 'w') as item_file:
                 item_file.write(item.data.encode('utf8'))
 

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -42,6 +42,7 @@ from xmodule.modulestore.django import ASSET_IGNORE_REGEX
 from xmodule.modulestore.exceptions import DuplicateCourseError
 from xmodule.modulestore.mongo.base import MongoRevisionKey
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.exceptions import ItemNotFoundError
 
 log = logging.getLogger(__name__)
 
@@ -588,6 +589,7 @@ def _import_course_draft(
 
                             # IMPORTANT: Be sure to update the sequential in the NEW namespace
                             seq_location = seq_location.map_into_course(target_course_id)
+
                             sequential = store.get_item(seq_location, depth=0)
 
                             non_draft_location = module.location.map_into_course(target_course_id)

common/lib/xmodule/xmodule/partitions/partitions.py

@@ -42,6 +42,9 @@ class Group(namedtuple("Group", "id name")):
 
         Raises TypeError if the value doesn't have the right keys.
         """
+        if isinstance(value, Group):
+            return value
+
         for key in ('id', 'name', 'version'):
             if key not in value:
                 raise TypeError("Group dict {0} missing value key '{1}'".format(
@@ -96,6 +99,9 @@ class UserPartition(namedtuple("UserPartition", "id name description groups")):
 
         Raises TypeError if the value doesn't have the right keys.
         """
+        if isinstance(value, UserPartition):
+            return value
+
         for key in ('id', 'name', 'description', 'version', 'groups'):
             if key not in value:
                 raise TypeError("UserPartition dict {0} missing value key '{1}'"

common/lib/xmodule/xmodule/tests/__init__.py

@@ -13,7 +13,9 @@ import pprint
 import unittest
 
 from contextlib import contextmanager
+from lazy import lazy
 from mock import Mock
+from operator import attrgetter
 from path import path
 
 from xblock.field_data import DictFieldData
@@ -193,7 +195,7 @@ class BulkAssertionManager(object):
             self._equal_actual.append((description, actual))
 
     def run_assertions(self):
-        self._test_case.assertEqual(self._equal_expected, self._equal_actual)
+        super(BulkAssertionTest, self._test_case).assertEqual(self._equal_expected, self._equal_actual)
 
 
 class BulkAssertionTest(unittest.TestCase):
@@ -224,8 +226,29 @@ class BulkAssertionTest(unittest.TestCase):
             self._manager.assertEqual(expected, actual, message)
         else:
             super(BulkAssertionTest, self).assertEqual(expected, actual, message)
+    assertEquals = assertEqual
 
 
+class LazyFormat(object):
+    """
+    An stringy object that delays formatting until it's put into a string context.
+    """
+    __slots__ = ('template', 'args', 'kwargs', '_message')
+
+    def __init__(self, template, *args, **kwargs):
+        self.template = template
+        self.args = args
+        self.kwargs = kwargs
+        self._message = None
+
+    def __unicode__(self):
+        if self._message is None:
+            self._message = self.template.format(*self.args, **self.kwargs)
+        return self._message
+
+    def __repr__(self):
+        return unicode(self)
+
 class CourseComparisonTest(BulkAssertionTest):
     """
     Mixin that has methods for comparing courses for equality.
@@ -255,6 +278,65 @@ class CourseComparisonTest(BulkAssertionTest):
         """
         self.ignored_asset_keys.add(key_name)
 
+    def assertReferenceRelativelyEqual(self, reference_field, expected_block, actual_block):
+        """
+        Assert that the supplied reference field is identical on the expected_block and actual_block,
+        assoming that the references are only relative (that is, comparing only on block_type and block_id,
+        not course_key).
+        """
+        def extract_key(usage_key):
+            if usage_key is None:
+                return None
+            else:
+                return (usage_key.block_type, usage_key.block_id)
+        expected = reference_field.read_from(expected_block)
+        actual = reference_field.read_from(actual_block)
+        if isinstance(reference_field, Reference):
+            expected = extract_key(expected)
+            actual = extract_key(actual)
+        elif isinstance(reference_field, ReferenceList):
+            expected = [extract_key(key) for key in expected]
+            actual = [extract_key(key) for key in actual]
+        elif isinstance(reference_field, ReferenceValueDict):
+            expected = {key: extract_key(val) for (key, val) in expected.iteritems()}
+            actual = {key: extract_key(val) for (key, val) in actual.iteritems()}
+        self.assertEqual(
+            expected,
+            actual,
+            LazyFormat(
+                "Field {} doesn't match between usages {} and {}: {!r} != {!r}",
+                reference_field.name,
+                expected_block.scope_ids.usage_id,
+                actual_block.scope_ids.usage_id,
+                expected,
+                actual
+            )
+        )
+
+    def assertBlocksEqualByFields(self, expected_block, actual_block):
+        self.assertEqual(expected_block.fields, actual_block.fields)
+        for field in expected_block.fields.values():
+            self.assertFieldEqual(field, expected_block, actual_block)
+
+    def assertFieldEqual(self, field, expected_block, actual_block):
+        if isinstance(field, (Reference, ReferenceList, ReferenceValueDict)):
+            self.assertReferenceRelativelyEqual(field, expected_block, actual_block)
+        else:
+            expected = field.read_from(expected_block)
+            actual = field.read_from(actual_block)
+            self.assertEqual(
+                expected,
+                actual,
+                LazyFormat(
+                    "Field {} doesn't match between usages {} and {}: {!r} != {!r}",
+                    field.name,
+                    expected_block.scope_ids.usage_id,
+                    actual_block.scope_ids.usage_id,
+                    expected,
+                    actual
+                )
+            )
+
     def assertCoursesEqual(self, expected_store, expected_course_key, actual_store, actual_course_key):
         """
         Assert that the courses identified by ``expected_course_key`` in ``expected_store`` and
@@ -312,11 +394,7 @@ class CourseComparisonTest(BulkAssertionTest):
                     actual_item = actual_item_map.get(map_key(actual_item_location))
 
                 # Formatting the message slows down tests of large courses significantly, so only do it if it would be used
-                if actual_item is None:
-                    msg = u'cannot find {} in {}'.format(map_key(actual_item_location), actual_item_map)
-                else:
-                    msg = None
-                self.assertIsNotNone(actual_item, msg)
+                self.assertIsNotNone(actual_item, LazyFormat(u'cannot find {} in {}', map_key(actual_item_location), actual_item_map))
 
                 # compare fields
                 self.assertEqual(expected_item.fields, actual_item.fields)
@@ -332,20 +410,7 @@ class CourseComparisonTest(BulkAssertionTest):
                     if field_name == 'children':
                         continue
 
-                    exp_value = map_references(field.read_from(expected_item), field, actual_course_key)
-                    actual_value = field.read_from(actual_item)
-                    # Formatting the message slows down tests of large courses significantly, so only do it if it would be used
-                    if exp_value != actual_value:
-                        msg = "Field {!r} doesn't match between usages {} and {}: {!r} != {!r}".format(
-                            field_name,
-                            expected_item.scope_ids.usage_id,
-                            actual_item.scope_ids.usage_id,
-                            exp_value,
-                            actual_value,
-                        )
-                    else:
-                        msg = None
-                    self.assertEqual(exp_value, actual_value, msg)
+                    self.assertFieldEqual(field, expected_item, actual_item)
 
                 # compare children
                 self.assertEqual(expected_item.has_children, actual_item.has_children)

common/lib/xmodule/xmodule/tests/test_bulk_assertions.py

+import ddt
+from xmodule.tests import BulkAssertionTest
+
+@ddt.ddt
+class TestBulkAssertionTestCase(BulkAssertionTest):
+
+    @ddt.data(
+        ('assertTrue', True),
+        ('assertFalse', False),
+        ('assertIs', 1, 1),
+        ('assertIsNot', 1, 2),
+        ('assertIsNone', None),
+        ('assertIsNotNone', 1),
+        ('assertIn', 1, (1, 2, 3)),
+        ('assertNotIn', 5, (1, 2, 3)),
+        ('assertIsInstance', 1, int),
+        ('assertNotIsInstance', '1', int),
+        ('assertRaises', KeyError, {}.__getitem__, '1'),
+    )
+    @ddt.unpack
+    def test_passing_asserts_passthrough(self, assertion, *args):
+        getattr(self, assertion)(*args)
+
+
+    @ddt.data(
+        ('assertTrue', False),
+        ('assertFalse', True),
+        ('assertIs', 1, 2),
+        ('assertIsNot', 1, 1),
+        ('assertIsNone', 1),
+        ('assertIsNotNone', None),
+        ('assertIn', 5, (1, 2, 3)),
+        ('assertNotIn', 1, (1, 2, 3)),
+        ('assertIsInstance', '1', int),
+        ('assertNotIsInstance', 1, int),
+        ('assertRaises', ValueError, lambda: None),
+    )
+    @ddt.unpack
+    def test_failing_asserts_passthrough(self, assertion, *args):
+        # Use super(BulkAssertionTest) to make sure we get un-adulturated assertions
+        with super(BulkAssertionTest, self).assertRaises(AssertionError):
+            getattr(self, assertion)(*args)
+
+    def test_no_bulk_assert_equals(self):
+        # Use super(BulkAssertionTest) to make sure we get un-adulturated assertions
+        with super(BulkAssertionTest, self).assertRaises(AssertionError):
+            self.assertEquals(1, 2)
+
+    @ddt.data(
+        'assertEqual', 'assertEquals'
+    )
+    def test_bulk_assert_equals(self, asserterFn):
+        asserter = getattr(self, asserterFn)
+        contextmanager = self.bulk_assertions()
+
+        contextmanager.__enter__()
+        super(BulkAssertionTest, self).assertIsNotNone(self._manager)
+        asserter(1, 2)
+        asserter(3, 4)
+
+        # Use super(BulkAssertionTest) to make sure we get un-adulturated assertions
+        with super(BulkAssertionTest, self).assertRaises(AssertionError):
+            contextmanager.__exit__(None, None, None)
+
+    @ddt.data(
+        'assertEqual', 'assertEquals'
+    )
+    def test_bulk_assert_closed(self, asserterFn):
+        asserter = getattr(self, asserterFn)
+
+        with self.bulk_assertions():
+            asserter(1, 1)
+            asserter(2, 2)
+
+        # Use super(BulkAssertionTest) to make sure we get un-adulturated assertions
+        with super(BulkAssertionTest, self).assertRaises(AssertionError):
+            asserter(1, 2)

common/test/acceptance/pages/studio/course_page.py

@@ -2,6 +2,8 @@
 Base class for pages specific to a course in Studio.
 """
 
+import os
+from opaque_keys.edx.locator import CourseLocator
 from bok_choy.page_object import PageObject
 from . import BASE_URL
 
@@ -34,5 +36,12 @@ class CoursePage(PageObject):
         """
         Construct a URL to the page within the course.
         """
-        course_key = "{course_org}/{course_num}/{course_run}".format(**self.course_info)
-        return "/".join([BASE_URL, self.url_path, course_key])
+        # TODO - is there a better way to make this agnostic to the underlying default module store?
+        default_store = os.environ.get('DEFAULT_STORE', 'draft')
+        course_key = CourseLocator(
+            self.course_info['course_org'],
+            self.course_info['course_num'],
+            self.course_info['course_run'],
+            deprecated=(default_store == 'draft')
+        )
+        return "/".join([BASE_URL, self.url_path, unicode(course_key)])

lms/djangoapps/courseware/module_render.py

@@ -115,6 +115,7 @@ def toc_for_course(user, request, course, active_chapter, active_section, field_
     field_data_cache must include data from the course module and 2 levels of its descendents
     '''
 
+    with modulestore().bulk_operations(course.id):
         course_module = get_module_for_descriptor(user, request, course, field_data_cache, course.id)
         if course_module is None:
             return None

lms/djangoapps/courseware/tests/test_module_render.py

@@ -326,19 +326,29 @@ class TestTOC(ModuleStoreTestCase):
         self.request = factory.get(chapter_url)
         self.request.user = UserFactory()
         self.modulestore = self.store._get_modulestore_for_courseid(self.course_key)
+        with self.modulestore.bulk_operations(self.course_key):
             with check_mongo_calls(num_finds, num_sends):
                 self.toy_course = self.store.get_course(self.toy_loc, depth=2)
                 self.field_data_cache = FieldDataCache.cache_for_descriptor_descendents(
                     self.toy_loc, self.request.user, self.toy_course, depth=2
                 )
 
-    # TODO: LMS-11220: Document why split find count is 9
-    # TODO: LMS-11220: Document why mongo find count is 4
-    @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0), (ModuleStoreEnum.Type.split, 9, 0))
+    # Mongo makes 3 queries to load the course to depth 2:
+    #     - 1 for the course
+    #     - 1 for its children
+    #     - 1 for its grandchildren
+    # Split makes 6 queries to load the course to depth 2:
+    #     - load the structure
+    #     - load 5 definitions
+    # Split makes 2 queries to render the toc:
+    #     - it loads the active version at the start of the bulk operation
+    #     - it loads the course definition for inheritance, because it's outside
+    #     the bulk-operation marker that loaded the course descriptor
+    @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0, 0), (ModuleStoreEnum.Type.split, 6, 0, 2))
     @ddt.unpack
-    def test_toc_toy_from_chapter(self, default_ms, num_finds, num_sends):
+    def test_toc_toy_from_chapter(self, default_ms, setup_finds, setup_sends, toc_finds):
         with self.store.default_store(default_ms):
-            self.setup_modulestore(default_ms, num_finds, num_sends)
+            self.setup_modulestore(default_ms, setup_finds, setup_sends)
             expected = ([{'active': True, 'sections':
                           [{'url_name': 'Toy_Videos', 'display_name': u'Toy Videos', 'graded': True,
                             'format': u'Lecture Sequence', 'due': None, 'active': False},
@@ -354,20 +364,29 @@ class TestTOC(ModuleStoreTestCase):
                             'format': '', 'due': None, 'active': False}],
                           'url_name': 'secret:magic', 'display_name': 'secret:magic'}])
 
-            with check_mongo_calls(0, 0):
+            with check_mongo_calls(toc_finds, 0):
                 actual = render.toc_for_course(
                     self.request.user, self.request, self.toy_course, self.chapter, None, self.field_data_cache
                 )
         for toc_section in expected:
             self.assertIn(toc_section, actual)
 
-    # TODO: LMS-11220: Document why split find count is 9
-    # TODO: LMS-11220: Document why mongo find count is 4
-    @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0), (ModuleStoreEnum.Type.split, 9, 0))
+    # Mongo makes 3 queries to load the course to depth 2:
+    #     - 1 for the course
+    #     - 1 for its children
+    #     - 1 for its grandchildren
+    # Split makes 6 queries to load the course to depth 2:
+    #     - load the structure
+    #     - load 5 definitions
+    # Split makes 2 queries to render the toc:
+    #     - it loads the active version at the start of the bulk operation
+    #     - it loads the course definition for inheritance, because it's outside
+    #     the bulk-operation marker that loaded the course descriptor
+    @ddt.data((ModuleStoreEnum.Type.mongo, 3, 0, 0), (ModuleStoreEnum.Type.split, 6, 0, 2))
     @ddt.unpack
-    def test_toc_toy_from_section(self, default_ms, num_finds, num_sends):
+    def test_toc_toy_from_section(self, default_ms, setup_finds, setup_sends, toc_finds):
         with self.store.default_store(default_ms):
-            self.setup_modulestore(default_ms, num_finds, num_sends)
+            self.setup_modulestore(default_ms, setup_finds, setup_sends)
             section = 'Welcome'
             expected = ([{'active': True, 'sections':
                           [{'url_name': 'Toy_Videos', 'display_name': u'Toy Videos', 'graded': True,
@@ -384,6 +403,7 @@ class TestTOC(ModuleStoreTestCase):
                             'format': '', 'due': None, 'active': False}],
                           'url_name': 'secret:magic', 'display_name': 'secret:magic'}])
 
+            with check_mongo_calls(toc_finds, 0):
                 actual = render.toc_for_course(self.request.user, self.request, self.toy_course, self.chapter, section, self.field_data_cache)
             for toc_section in expected:
                 self.assertIn(toc_section, actual)

requirements/edx/base.txt

@@ -131,7 +131,7 @@ rednose==0.3
 selenium==2.42.1
 splinter==0.5.4
 testtools==0.9.34
-PyContracts==1.6.4
+PyContracts==1.6.5
 
 # Used for Segment.io analytics
 analytics-python==0.4.4

requirements/edx/github.txt

@@ -30,7 +30,7 @@
 -e git+https://github.com/edx-solutions/django-splash.git@7579d052afcf474ece1239153cffe1c89935bc4f#egg=django-splash
 -e git+https://github.com/edx/acid-block.git@459aff7b63db8f2c5decd1755706c1a64fb4ebb1#egg=acid-xblock
 -e git+https://github.com/edx/edx-ora2.git@release-2014-09-18T16.00#egg=edx-ora2
--e git+https://github.com/edx/opaque-keys.git@d45d0bd8d64c69531be69178b9505b5d38806ce0#egg=opaque-keys
+-e git+https://github.com/edx/opaque-keys.git@295d93170b2f6e57e3a2b9ba0a52087a4e8712c5#egg=opaque-keys
 -e git+https://github.com/edx/ease.git@97de68448e5495385ba043d3091f570a699d5b5f#egg=ease
 -e git+https://github.com/edx/i18n-tools.git@56f048af9b6868613c14aeae760548834c495011#egg=i18n-tools
 -e git+https://github.com/edx/edx-oauth2-provider.git@0.2.2#egg=oauth2-provider