xblock fields persist w/o breaking by scope

Letting xblocks handle scope rather than separating fields into
different attrs. Although, split still shunts content fields to a
different collection than setting and children fields.

The big difference is that content fields will always be a dict and not
sometimes just a string and there's no special casing of 'data' attr.

The other mind change is no more 'metadata' dict.

cms/djangoapps/contentstore/tests/test_crud.py

@@ -75,7 +75,7 @@ class TemplateTests(unittest.TestCase):
             display_name='fun test course', user_id='testbot')
 
         test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
-            'metadata': {'display_name': 'chapter n'}},
+            'fields': {'display_name': 'chapter n'}},
             test_course.system, parent_xblock=test_course)
         self.assertIsInstance(test_chapter, SequenceDescriptor)
         self.assertEqual(test_chapter.display_name, 'chapter n')
@@ -84,7 +84,7 @@ class TemplateTests(unittest.TestCase):
         # test w/ a definition (e.g., a problem)
         test_def_content = '<problem>boo</problem>'
         test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
-            'definition': {'data': test_def_content}},
+            'fields': {'data': test_def_content}},
             test_course.system, parent_xblock=test_chapter)
         self.assertIsInstance(test_problem, CapaDescriptor)
         self.assertEqual(test_problem.data, test_def_content)
@@ -99,11 +99,12 @@ class TemplateTests(unittest.TestCase):
         test_course = persistent_factories.PersistentCourseFactory.create(org='testx', prettyid='tempcourse',
             display_name='fun test course', user_id='testbot')
         test_chapter = XModuleDescriptor.load_from_json({'category': 'chapter',
-            'metadata': {'display_name': 'chapter n'}},
+            'fields': {'display_name': 'chapter n'}},
             test_course.system, parent_xblock=test_course)
         test_def_content = '<problem>boo</problem>'
-        test_problem = XModuleDescriptor.load_from_json({'category': 'problem',
-            'definition': {'data': test_def_content}},
+        # create child
+        _ = XModuleDescriptor.load_from_json({'category': 'problem',
+            'fields': {'data': test_def_content}},
             test_course.system, parent_xblock=test_chapter)
         # better to pass in persisted parent over the subdag so
         # subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
@@ -152,15 +153,24 @@ class TemplateTests(unittest.TestCase):
             parent_location=test_course.location, user_id='testbot')
         sub = persistent_factories.ItemFactory.create(display_name='subsection 1',
             parent_location=chapter.location, user_id='testbot', category='vertical')
-        first_problem = persistent_factories.ItemFactory.create(display_name='problem 1',
-            parent_location=sub.location, user_id='testbot', category='problem', data="<problem></problem>")
+        first_problem = persistent_factories.ItemFactory.create(
+            display_name='problem 1', parent_location=sub.location, user_id='testbot', category='problem',
+            fields={'data':"<problem></problem>"}
+        )
         first_problem.max_attempts = 3
+        first_problem.save()  # decache the above into the kvs
         updated_problem = modulestore('split').update_item(first_problem, 'testbot')
-        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot')
+        self.assertIsNotNone(updated_problem.previous_version)
+        self.assertEqual(updated_problem.previous_version, first_problem.update_version)
+        self.assertNotEqual(updated_problem.update_version, first_problem.update_version)
+        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot', delete_children=True)
 
-        second_problem = persistent_factories.ItemFactory.create(display_name='problem 2',
+        second_problem = persistent_factories.ItemFactory.create(
+            display_name='problem 2',
             parent_location=BlockUsageLocator(updated_loc, usage_id=sub.location.usage_id),
-            user_id='testbot', category='problem', data="<problem></problem>")
+            user_id='testbot', category='problem',
+            fields={'data':"<problem></problem>"}
+        )
 
         # course root only updated 2x
         version_history = modulestore('split').get_block_generations(test_course.location)

common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py

@@ -11,18 +11,17 @@ from .split_mongo_kvs import SplitMongoKVS, SplitMongoKVSid
 
 log = logging.getLogger(__name__)
 
-# TODO should this be here or w/ x_module or ???
 class CachingDescriptorSystem(MakoDescriptorSystem):
     """
     A system that has a cache of a course version's json that it will use to load modules
     from, with a backup of calling to the underlying modulestore for more data.
 
-    Computes the metadata inheritance upon creation.
+    Computes the settings (nee 'metadata') inheritance upon creation.
     """
     def __init__(self, modulestore, course_entry, module_data, lazy,
         default_class, error_tracker, render_template):
         """
-        Computes the metadata inheritance and sets up the cache.
+        Computes the settings inheritance and sets up the cache.
 
         modulestore: the module store that can be used to retrieve additional
         modules
@@ -50,9 +49,10 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
         self.default_class = default_class
         # TODO see if self.course_id is needed: is already in course_entry but could be > 1 value
         # Compute inheritance
-        modulestore.inherit_metadata(course_entry.get('blocks', {}),
-            course_entry.get('blocks', {})
-            .get(course_entry.get('root')))
+        modulestore.inherit_settings(
+            course_entry.get('blocks', {}),
+            course_entry.get('blocks', {}).get(course_entry.get('root'))
+        )
 
     def _load_item(self, usage_id, course_entry_override=None):
         # TODO ensure all callers of system.load_item pass just the id
@@ -73,9 +73,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
     def xblock_from_json(self, class_, usage_id, json_data, course_entry_override=None):
         if course_entry_override is None:
             course_entry_override = self.course_entry
-        # most likely a lazy loader but not the id directly
+        # most likely a lazy loader or the id directly
         definition = json_data.get('definition', {})
-        metadata = json_data.get('metadata', {})
 
         block_locator = BlockUsageLocator(
             version_guid=course_entry_override['_id'],
@@ -86,9 +85,8 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
 
         kvs = SplitMongoKVS(
             definition,
-            json_data.get('children', []),
-            metadata,
-            json_data.get('_inherited_metadata'),
+            json_data.get('fields', {}),
+            json_data.get('_inherited_settings'),
             block_locator,
             json_data.get('category'))
         model_data = DbModel(kvs, class_, None,
@@ -111,10 +109,11 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
                 error_msg=exc_info_to_str(sys.exc_info())
             )
 
-        module.edited_by = json_data.get('edited_by')
-        module.edited_on = json_data.get('edited_on')
-        module.previous_version = json_data.get('previous_version')
-        module.update_version = json_data.get('update_version')
+        edit_info = json_data.get('edit_info', {})
+        module.edited_by = edit_info.get('edited_by')
+        module.edited_on = edit_info.get('edited_on')
+        module.previous_version = edit_info.get('previous_version')
+        module.update_version = edit_info.get('update_version')
         module.definition_locator = self.modulestore.definition_locator(definition)
         # decache any pending field settings
         module.save()

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -16,6 +16,9 @@ from .. import ModuleStoreBase
 from ..exceptions import ItemNotFoundError
 from .definition_lazy_loader import DefinitionLazyLoader
 from .caching_descriptor_system import CachingDescriptorSystem
+from xblock.core import Scope
+from pytz import UTC
+import collections
 
 log = logging.getLogger(__name__)
 #==============================================================================
@@ -102,10 +105,12 @@ class SplitMongoModuleStore(ModuleStoreBase):
         '''
         new_module_data = {}
         for usage_id in base_usage_ids:
-            new_module_data = self.descendants(system.course_entry['blocks'],
+            new_module_data = self.descendants(
+                system.course_entry['blocks'],
                 usage_id,
                 depth,
-                                               new_module_data)
+                new_module_data
+            )
 
         # remove any which were already in module_data (not sure if there's a better way)
         for newkey in new_module_data.iterkeys():
@@ -114,8 +119,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         if lazy:
             for block in new_module_data.itervalues():
-                block['definition'] = DefinitionLazyLoader(self,
-                                                           block['definition'])
+                block['definition'] = DefinitionLazyLoader(self, block['definition'])
         else:
             # Load all descendants by id
             descendent_definitions = self.definitions.find({
@@ -127,7 +131,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
             for block in new_module_data.itervalues():
                 if block['definition'] in definitions:
-                    block['definition'] = definitions[block['definition']]
+                    block['fields'].update(definitions[block['definition']].get('fields'))
 
         system.module_data.update(new_module_data)
         return system.module_data
@@ -317,7 +321,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         definitions.
 
         Common qualifiers are category, definition (provide definition id),
-        metadata: {display_name ..}, children (return
+        display_name, anyfieldname, children (return
         block if its children includes the one given value). If you want
         substring matching use {$regex: /acme.*corp/i} type syntax.
 
@@ -371,7 +375,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         course = self._lookup_course(locator)
         items = []
         for parent_id, value in course['blocks'].iteritems():
-            for child_id in value['children']:
+            for child_id in value['fields'].get('children', []):
                 if locator.usage_id == child_id:
                     items.append(BlockUsageLocator(url=locator.as_course_locator(), usage_id=parent_id))
         return items
@@ -427,11 +431,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         definition = self.definitions.find_one({'_id': definition_locator.definition_id})
         if definition is None:
             return None
-        return {'original_version': definition['original_version'],
-            'previous_version': definition['previous_version'],
-            'edited_by': definition['edited_by'],
-            'edited_on': definition['edited_on']
-        }
+        return definition['edit_info']
 
     def get_course_successors(self, course_locator, version_history_depth=1):
         '''
@@ -471,29 +471,29 @@ class SplitMongoModuleStore(ModuleStoreBase):
         Find the history of this block. Return as a VersionTree of each place the block changed (except
         deletion).
 
-        The block's history tracks its explicit changes; so, changes in descendants won't be reflected
-        as new iterations.
+        The block's history tracks its explicit changes but not the changes in its children.
+
         '''
         block_locator = block_locator.version_agnostic()
         course_struct = self._lookup_course(block_locator)
         usage_id = block_locator.usage_id
-        update_version_field = 'blocks.{}.update_version'.format(usage_id)
+        update_version_field = 'blocks.{}.edit_info.update_version'.format(usage_id)
         all_versions_with_block = self.structures.find({'original_version': course_struct['original_version'],
             update_version_field: {'$exists': True}})
         # find (all) root versions and build map previous: [successors]
         possible_roots = []
         result = {}
         for version in all_versions_with_block:
-            if version['_id'] == version['blocks'][usage_id]['update_version']:
-                if version['blocks'][usage_id].get('previous_version') is None:
-                    possible_roots.append(version['blocks'][usage_id]['update_version'])
+            if version['_id'] == version['blocks'][usage_id]['edit_info']['update_version']:
+                if version['blocks'][usage_id]['edit_info'].get('previous_version') is None:
+                    possible_roots.append(version['blocks'][usage_id]['edit_info']['update_version'])
                 else:
-                    result.setdefault(version['blocks'][usage_id]['previous_version'], set()).add(
-                        version['blocks'][usage_id]['update_version'])
+                    result.setdefault(version['blocks'][usage_id]['edit_info']['previous_version'], set()).add(
+                        version['blocks'][usage_id]['edit_info']['update_version'])
         # more than one possible_root means usage was added and deleted > 1x.
         if len(possible_roots) > 1:
             # find the history segment including block_locator's version
-            element_to_find = course_struct['blocks'][usage_id]['update_version']
+            element_to_find = course_struct['blocks'][usage_id]['edit_info']['update_version']
             if element_to_find in possible_roots:
                 possible_roots = [element_to_find]
             for possibility in possible_roots:
@@ -513,7 +513,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         Find the version_history_depth next versions of this definition. Return as a VersionTree
         '''
         # TODO implement
-        pass
+        raise NotImplementedError()
 
     def create_definition_from_data(self, new_def_data, category, user_id):
         """
@@ -522,16 +522,21 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         :param user_id: request.user object
         """
-        document = {"category" : category,
-            "data": new_def_data,
+        new_def_data = self._filter_special_fields(new_def_data)
+        document = {
+            "category" : category,
+            "fields": new_def_data,
+            "edit_info": {
                 "edited_by": user_id,
-            "edited_on": datetime.datetime.utcnow(),
+                "edited_on": datetime.datetime.now(UTC),
                 "previous_version": None,
-            "original_version": None}
+                "original_version": None
+                }
+            }
         new_id = self.definitions.insert(document)
         definition_locator = DescriptionLocator(new_id)
-        document['original_version'] = new_id
-        self.definitions.update({'_id': new_id}, {'$set': {"original_version": new_id}})
+        document['edit_info']['original_version'] = new_id
+        self.definitions.update({'_id': new_id}, {'$set': {"edit_info.original_version": new_id}})
         return definition_locator
 
     def update_definition_from_data(self, definition_locator, new_def_data, user_id):
@@ -541,16 +546,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         :param user_id: request.user
         """
+        new_def_data = self._filter_special_fields(new_def_data)
         def needs_saved():
-            if isinstance(new_def_data, dict):
             for key, value in new_def_data.iteritems():
-                    if key not in old_definition['data'] or value != old_definition['data'][key]:
+                if key not in old_definition['fields'] or value != old_definition['fields'][key]:
                     return True
-                for key, value in old_definition['data'].iteritems():
+            for key, value in old_definition.get('fields', {}).iteritems():
                 if key not in new_def_data:
                     return True
-            else:
-                return new_def_data != old_definition['data']
 
         # if this looks in cache rather than fresh fetches, then it will probably not detect
         # actual change b/c the descriptor and cache probably point to the same objects
@@ -560,10 +563,10 @@ class SplitMongoModuleStore(ModuleStoreBase):
         del old_definition['_id']
 
         if needs_saved():
-            old_definition['data'] = new_def_data
-            old_definition['edited_by'] = user_id
-            old_definition['edited_on'] = datetime.datetime.utcnow()
-            old_definition['previous_version'] = definition_locator.definition_id
+            old_definition['fields'] = new_def_data
+            old_definition['edit_info']['edited_by'] = user_id
+            old_definition['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            old_definition['edit_info']['previous_version'] = definition_locator.definition_id
             new_id = self.definitions.insert(old_definition)
             return DescriptionLocator(new_id), True
         else:
@@ -605,11 +608,11 @@ class SplitMongoModuleStore(ModuleStoreBase):
         else:
             return id_root
 
-    # TODO I would love to write this to take a real descriptor and persist it BUT descriptors, kvs, and dbmodel
-    #      all assume locators are set and unique! Having this take the model contents piecemeal breaks the separation
-    #      of model from persistence layer
-    def create_item(self, course_or_parent_locator, category, user_id, definition_locator=None, new_def_data=None,
-        metadata=None, force=False):
+    # TODO Should I rewrite this to take a new xblock instance rather than to construct it? That is, require the
+    # caller to use XModuleDescriptor.load_from_json thus reducing similar code and making the object creation and
+    # validation behavior a responsibility of the model layer rather than the persistence layer.
+    def create_item(self, course_or_parent_locator, category, user_id, definition_locator=None, fields=None,
+        force=False):
         """
         Add a descriptor to persistence as the last child of the optional parent_location or just as an element
         of the course (if no parent provided). Return the resulting post saved version with populated locators.
@@ -624,9 +627,10 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         The incoming definition_locator should either be None to indicate this is a brand new definition or
         a pointer to the existing definition to which this block should point or from which this was derived.
-        If new_def_data is None, then definition_locator must have a value meaning that this block points
-        to the existing definition. If new_def_data is not None and definition_location is not None, then
-        new_def_data is assumed to be a new payload for definition_location.
+        If fields does not contain any Scope.content, then definition_locator must have a value meaning that this
+        block points
+        to the existing definition. If fields contains Scope.content and definition_locator is not None, then
+        the Scope.content fields are assumed to be a new payload for definition_locator.
 
         Creates a new version of the course structure, creates and inserts the new block, makes the block point
         to the definition which may be new or a new version of an existing or an existing.
@@ -645,6 +649,8 @@ class SplitMongoModuleStore(ModuleStoreBase):
         index_entry = self._get_index_if_valid(course_or_parent_locator, force)
         structure = self._lookup_course(course_or_parent_locator)
 
+        partitioned_fields = self._partition_fields_by_scope(category, fields)
+        new_def_data = partitioned_fields.get(Scope.content, {})
         # persist the definition if persisted != passed
         if (definition_locator is None or definition_locator.definition_id is None):
             definition_locator = self.create_definition_from_data(new_def_data, category, user_id)
@@ -655,23 +661,27 @@ class SplitMongoModuleStore(ModuleStoreBase):
         new_structure = self._version_structure(structure, user_id)
         # generate an id
         new_usage_id = self._generate_usage_id(new_structure['blocks'], category)
-        update_version_keys = ['blocks.{}.update_version'.format(new_usage_id)]
+        update_version_keys = ['blocks.{}.edit_info.update_version'.format(new_usage_id)]
         if isinstance(course_or_parent_locator, BlockUsageLocator) and course_or_parent_locator.usage_id is not None:
             parent = new_structure['blocks'][course_or_parent_locator.usage_id]
-            parent['children'].append(new_usage_id)
-            parent['edited_on'] = datetime.datetime.utcnow()
-            parent['edited_by'] = user_id
-            parent['previous_version'] = parent['update_version']
-            update_version_keys.append('blocks.{}.update_version'.format(course_or_parent_locator.usage_id))
+            parent['fields'].setdefault('children', []).append(new_usage_id)
+            parent['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            parent['edit_info']['edited_by'] = user_id
+            parent['edit_info']['previous_version'] = parent['edit_info']['update_version']
+            update_version_keys.append('blocks.{}.edit_info.update_version'.format(course_or_parent_locator.usage_id))
+        block_fields = partitioned_fields.get(Scope.settings, {})
+        if Scope.children in partitioned_fields:
+            block_fields.update(partitioned_fields[Scope.children])
         new_structure['blocks'][new_usage_id] = {
-            "children": [],
             "category": category,
             "definition": definition_locator.definition_id,
-            "metadata": metadata if metadata else {},
-            'edited_on': datetime.datetime.utcnow(),
+            "fields": block_fields,
+            'edit_info': {
+                'edited_on': datetime.datetime.now(UTC),
                 'edited_by': user_id,
                 'previous_version': None
             }
+        }
         new_id = self.structures.insert(new_structure)
         update_version_payload = {key: new_id for key in update_version_keys}
         self.structures.update({'_id': new_id},
@@ -689,8 +699,9 @@ class SplitMongoModuleStore(ModuleStoreBase):
                                                usage_id=new_usage_id,
                                                version_guid=new_id))
 
-    def create_course(self, org, prettyid, user_id, id_root=None, metadata=None, course_data=None,
-        master_version='draft', versions_dict=None, root_category='course'):
+    def create_course(
+        self, org, prettyid, user_id, id_root=None, fields=None,
+        master_branch='draft', versions_dict=None, root_category='course'):
         """
         Create a new entry in the active courses index which points to an existing or new structure. Returns
         the course root of the resulting entry (the location has the course id)
@@ -698,93 +709,106 @@ class SplitMongoModuleStore(ModuleStoreBase):
         id_root: allows the caller to specify the course_id. It's a root in that, if it's already taken,
         this method will append things to the root to make it unique. (defaults to org)
 
-        metadata: if provided, will set the metadata of the root course object in the new draft course. If both
-        metadata and a starting version are provided, it will generate a successor version to the given version,
-        and update the metadata with any provided values (via update not setting).
+        fields: if scope.settings fields provided, will set the fields of the root course object in the
+        new course. If both
+        settings fields and a starting version are provided (via versions_dict), it will generate a successor version
+        to the given version,
+        and update the settings fields with any provided values (via update not setting).
 
-        course_data: if provided, will update the data of the new course xblock definition to this. Like metadata,
+        fields (content): if scope.content fields provided, will update the fields of the new course
+        xblock definition to this. Like settings fields,
         if provided, this will cause a new version of any given version as well as a new version of the
         definition (which will point to the existing one if given a version). If not provided and given
-        a draft_version, it will reuse the same definition as the draft course (obvious since it's reusing the draft
-        course). If not provided and no draft is given, it will be empty and get the field defaults (hopefully) when
+        a version_dict, it will reuse the same definition as that version's course
+        (obvious since it's reusing the
+        course). If not provided and no version_dict is given, it will be empty and get the field defaults
+        when
         loaded.
 
-        master_version: the tag (key) for the version name in the dict which is the 'draft' version. Not the actual
+        master_branch: the tag (key) for the version name in the dict which is the 'draft' version. Not the actual
         version guid, but what to call it.
 
         versions_dict: the starting version ids where the keys are the tags such as 'draft' and 'published'
         and the values are structure guids. If provided, the new course will reuse this version (unless you also
-        provide any overrides such as metadata, see above). if not provided, will create a mostly empty course
+        provide any fields overrides, see above). if not provided, will create a mostly empty course
         structure with just a category course root xblock.
         """
-        if metadata is None:
-            metadata = {}
+        partitioned_fields = self._partition_fields_by_scope('course', fields)
+        block_fields = partitioned_fields.setdefault(Scope.settings, {})
+        if Scope.children in partitioned_fields:
+            block_fields.update(partitioned_fields[Scope.children])
+        definition_fields = self._filter_special_fields(partitioned_fields.get(Scope.content, {}))
+
         # build from inside out: definition, structure, index entry
         # if building a wholly new structure
-        if versions_dict is None or master_version not in versions_dict:
+        if versions_dict is None or master_branch not in versions_dict:
             # create new definition and structure
-            if course_data is None:
-                course_data = {}
             definition_entry = {
                 'category': root_category,
-                'data': course_data,
+                'fields': definition_fields,
+                'edit_info': {
                     'edited_by': user_id,
-                'edited_on': datetime.datetime.utcnow(),
+                    'edited_on': datetime.datetime.now(UTC),
                     'previous_version': None,
                 }
+            }
             definition_id = self.definitions.insert(definition_entry)
-            definition_entry['original_version'] = definition_id
-            self.definitions.update({'_id': definition_id}, {'$set': {"original_version": definition_id}})
+            definition_entry['edit_info']['original_version'] = definition_id
+            self.definitions.update({'_id': definition_id}, {'$set': {"edit_info.original_version": definition_id}})
 
             draft_structure = {
                 'root': 'course',
                 'previous_version': None,
                 'edited_by': user_id,
-                'edited_on': datetime.datetime.utcnow(),
+                'edited_on': datetime.datetime.now(UTC),
                 'blocks': {
                     'course': {
-                        'children':[],
                         'category': 'course',
                         'definition': definition_id,
-                        'metadata': metadata,
-                        'edited_on': datetime.datetime.utcnow(),
+                        'fields': block_fields,
+                        'edit_info': {
+                            'edited_on': datetime.datetime.now(UTC),
                             'edited_by': user_id,
-                        'previous_version': None}}}
+                            'previous_version': None
+                        }
+                    }
+                }
+            }
             new_id = self.structures.insert(draft_structure)
             draft_structure['original_version'] = new_id
             self.structures.update({'_id': new_id},
                 {'$set': {"original_version": new_id,
-                    'blocks.course.update_version': new_id}})
+                    'blocks.course.edit_info.update_version': new_id}})
             if versions_dict is None:
-                versions_dict = {master_version: new_id}
+                versions_dict = {master_branch: new_id}
             else:
-                versions_dict[master_version] = new_id
+                versions_dict[master_branch] = new_id
 
         else:
             # just get the draft_version structure
-            draft_version = CourseLocator(version_guid=versions_dict[master_version])
+            draft_version = CourseLocator(version_guid=versions_dict[master_branch])
             draft_structure = self._lookup_course(draft_version)
-            if course_data is not None or metadata:
+            if definition_fields or block_fields:
                 draft_structure = self._version_structure(draft_structure, user_id)
                 root_block = draft_structure['blocks'][draft_structure['root']]
-                if metadata is not None:
-                    root_block['metadata'].update(metadata)
-                if course_data is not None:
+                if block_fields is not None:
+                    root_block['fields'].update(block_fields)
+                if definition_fields is not None:
                     definition = self.definitions.find_one({'_id': root_block['definition']})
-                    definition['data'].update(course_data)
-                    definition['previous_version'] = definition['_id']
-                    definition['edited_by'] = user_id
-                    definition['edited_on'] = datetime.datetime.utcnow()
+                    definition['fields'].update(definition_fields)
+                    definition['edit_info']['previous_version'] = definition['_id']
+                    definition['edit_info']['edited_by'] = user_id
+                    definition['edit_info']['edited_on'] = datetime.datetime.now(UTC)
                     del definition['_id']
                     root_block['definition'] = self.definitions.insert(definition)
-                    root_block['edited_on'] = datetime.datetime.utcnow()
-                    root_block['edited_by'] = user_id
-                    root_block['previous_version'] = root_block.get('update_version')
+                    root_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+                    root_block['edit_info']['edited_by'] = user_id
+                    root_block['edit_info']['previous_version'] = root_block['edit_info'].get('update_version')
                 # insert updates the '_id' in draft_structure
                 new_id = self.structures.insert(draft_structure)
-                versions_dict[master_version] = new_id
+                versions_dict[master_branch] = new_id
                 self.structures.update({'_id': new_id},
-                    {'$set': {'blocks.{}.update_version'.format(draft_structure['root']): new_id}})
+                    {'$set': {'blocks.{}.edit_info.update_version'.format(draft_structure['root']): new_id}})
         # create the index entry
         if id_root is None:
             id_root = org
@@ -795,14 +819,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
             'org': org,
             'prettyid': prettyid,
             'edited_by': user_id,
-            'edited_on': datetime.datetime.utcnow(),
+            'edited_on': datetime.datetime.now(UTC),
             'versions': versions_dict}
         new_id = self.course_index.insert(index_entry)
-        return self.get_course(CourseLocator(course_id=new_id, branch=master_version))
+        return self.get_course(CourseLocator(course_id=new_id, branch=master_branch))
 
     def update_item(self, descriptor, user_id, force=False):
         """
-        Save the descriptor's definition, metadata, & children references (i.e., it doesn't descend the tree).
+        Save the descriptor's fields. it doesn't descend the course dag to save the children.
         Return the new descriptor (updated location).
 
         raises ItemNotFoundError if the location does not exist.
@@ -819,31 +843,38 @@ class SplitMongoModuleStore(ModuleStoreBase):
         index_entry = self._get_index_if_valid(descriptor.location, force)
 
         descriptor.definition_locator, is_updated = self.update_definition_from_data(
-            descriptor.definition_locator, descriptor.xblock_kvs.get_data(), user_id)
+            descriptor.definition_locator, descriptor.get_explicitly_set_fields_by_scope(Scope.content), user_id)
         # check children
         original_entry = original_structure['blocks'][descriptor.location.usage_id]
         if (not is_updated and descriptor.has_children
-            and not self._xblock_lists_equal(original_entry['children'], descriptor.children)):
+            and not self._xblock_lists_equal(original_entry['fields']['children'], descriptor.children)):
             is_updated = True
         # check metadata
         if not is_updated:
-            is_updated = self._compare_metadata(descriptor.xblock_kvs.get_own_metadata(), original_entry['metadata'])
+            is_updated = self._compare_settings(
+                descriptor.get_explicitly_set_fields_by_scope(Scope.settings),
+                original_entry['fields']
+            )
 
         # if updated, rev the structure
         if is_updated:
             new_structure = self._version_structure(original_structure, user_id)
             block_data = new_structure['blocks'][descriptor.location.usage_id]
-            if descriptor.has_children:
-                block_data["children"] = [self._usage_id(child) for child in descriptor.children]
 
             block_data["definition"] = descriptor.definition_locator.definition_id
-            block_data["metadata"] = descriptor.xblock_kvs.get_own_metadata()
-            block_data['edited_on'] = datetime.datetime.utcnow()
-            block_data['edited_by'] = user_id
-            block_data['previous_version'] = block_data['update_version']
+            block_data["fields"] = descriptor.get_explicitly_set_fields_by_scope(Scope.settings)
+            if descriptor.has_children:
+                block_data['fields']["children"] = [self._usage_id(child) for child in descriptor.children]
+
+            block_data['edit_info'] = {
+                'edited_on': datetime.datetime.now(UTC),
+                'edited_by': user_id,
+                'previous_version': block_data['edit_info']['update_version'],
+            }
             new_id = self.structures.insert(new_structure)
-            self.structures.update({'_id': new_id},
-                {'$set': {'blocks.{}.update_version'.format(descriptor.location.usage_id): new_id}})
+            self.structures.update(
+                {'_id': new_id},
+                {'$set': {'blocks.{}.edit_info.update_version'.format(descriptor.location.usage_id): new_id}})
 
             # update the index entry if appropriate
             if index_entry is not None:
@@ -869,8 +900,8 @@ class SplitMongoModuleStore(ModuleStoreBase):
         returns the post-persisted version of the incoming xblock. Note that its children will be ids not
         objects.
 
-        :param xblock:
-        :param user_id:
+        :param xblock: the head of the dag
+        :param user_id: who's doing the change
         """
         # find course_index entry if applicable and structures entry
         index_entry = self._get_index_if_valid(xblock.location, force)
@@ -883,7 +914,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
             new_id = self.structures.insert(new_structure)
             update_command = {}
             for usage_id in changed_blocks:
-                update_command['blocks.{}.update_version'.format(usage_id)] = new_id
+                update_command['blocks.{}.edit_info.update_version'.format(usage_id)] = new_id
             self.structures.update({'_id': new_id}, {'$set': update_command})
 
             # update the index entry if appropriate
@@ -897,14 +928,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
     def _persist_subdag(self, xblock, user_id, structure_blocks):
         # persist the definition if persisted != passed
-        new_def_data = xblock.xblock_kvs.get_data()
+        new_def_data = self._filter_special_fields(xblock.get_explicitly_set_fields_by_scope(Scope.content))
         if (xblock.definition_locator is None or xblock.definition_locator.definition_id is None):
-            xblock.definition_locator = self.create_definition_from_data(new_def_data,
-                xblock.category, user_id)
+            xblock.definition_locator = self.create_definition_from_data(
+                new_def_data, xblock.category, user_id)
             is_updated = True
-        elif new_def_data is not None:
-            xblock.definition_locator, is_updated = self.update_definition_from_data(xblock.definition_locator,
-                new_def_data, user_id)
+        elif new_def_data:
+            xblock.definition_locator, is_updated = self.update_definition_from_data(
+                xblock.definition_locator, new_def_data, user_id)
 
         if xblock.location.usage_id is None:
             # generate an id
@@ -916,7 +947,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
             is_new = False
             usage_id = xblock.location.usage_id
             if (not is_updated and xblock.has_children
-                and not self._xblock_lists_equal(structure_blocks[usage_id]['children'], xblock.children)):
+                and not self._xblock_lists_equal(structure_blocks[usage_id]['fields']['children'], xblock.children)):
                 is_updated = True
 
         children = []
@@ -930,41 +961,52 @@ class SplitMongoModuleStore(ModuleStoreBase):
                     children.append(child)
 
         is_updated = is_updated or updated_blocks
-        metadata = xblock.xblock_kvs.get_own_metadata()
+        block_fields = xblock.get_explicitly_set_fields_by_scope(Scope.settings)
         if not is_new and not is_updated:
-            is_updated = self._compare_metadata(metadata, structure_blocks[usage_id]['metadata'])
+            is_updated = self._compare_settings(block_fields, structure_blocks[usage_id]['fields'])
+        if children:
+            block_fields['children'] = children
 
         if is_updated:
+            previous_version = None if is_new else structure_blocks[usage_id]['edit_info'].get('update_version')
             structure_blocks[usage_id] = {
-                "children": children,
                 "category": xblock.category,
                 "definition": xblock.definition_locator.definition_id,
-                "metadata": metadata if metadata else {},
-                'previous_version': structure_blocks.get(usage_id, {}).get('update_version'),
+                "fields": block_fields,
+                'edit_info': {
+                    'previous_version': previous_version,
                     'edited_by': user_id,
-                'edited_on': datetime.datetime.utcnow()
+                    'edited_on': datetime.datetime.now(UTC)
+                }
             }
             updated_blocks.append(usage_id)
 
         return updated_blocks
 
-    def _compare_metadata(self, metadata, original_metadata):
-        original_keys = original_metadata.keys()
-        if len(metadata) != len(original_keys):
+    def _compare_settings(self, settings, original_fields):
+        """
+        Return True if the settings are not == to the original fields
+        :param settings:
+        :param original_fields:
+        """
+        original_keys = original_fields.keys()
+        if 'children' in original_keys:
+            original_keys.remove('children')
+        if len(settings) != len(original_keys):
             return True
         else:
-            new_keys = metadata.keys()
+            new_keys = settings.keys()
             for key in original_keys:
-                if key not in new_keys or original_metadata[key] != metadata[key]:
+                if key not in new_keys or original_fields[key] != settings[key]:
                     return True
 
-    # TODO change all callers to update_item
-    def update_children(self, course_id, location, children):
-        raise NotImplementedError()
+    def update_children(self, location, children):
+        '''Deprecated, use update_item.'''
+        raise NotImplementedError('use update_item')
 
-    # TODO change all callers to update_item
-    def update_metadata(self, course_id, location, metadata):
-        raise NotImplementedError()
+    def update_metadata(self, location, metadata):
+        '''Deprecated, use update_item.'''
+        raise NotImplementedError('use update_item')
 
     def update_course_index(self, course_locator, new_values_dict, update_versions=False):
         """
@@ -992,9 +1034,9 @@ class SplitMongoModuleStore(ModuleStoreBase):
         self.course_index.update({'_id': course_locator.course_id},
             {'$set': new_values_dict})
 
-    def delete_item(self, usage_locator, user_id, force=False):
+    def delete_item(self, usage_locator, user_id, delete_children=False, force=False):
         """
-        Delete the tree rooted at block and any references w/in the course to the block
+        Delete the block or tree rooted at block (if delete_children) and any references w/in the course to the block
         from a new version of the course structure.
 
         returns CourseLocator for new version
@@ -1018,16 +1060,17 @@ class SplitMongoModuleStore(ModuleStoreBase):
         update_version_keys = []
         for parent in parents:
             parent_block = new_blocks[parent.usage_id]
-            parent_block['children'].remove(usage_locator.usage_id)
-            parent_block['edited_on'] = datetime.datetime.utcnow()
-            parent_block['edited_by'] = user_id
-            parent_block['previous_version'] = parent_block['update_version']
-            update_version_keys.append('blocks.{}.update_version'.format(parent.usage_id))
+            parent_block['fields']['children'].remove(usage_locator.usage_id)
+            parent_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            parent_block['edit_info']['edited_by'] = user_id
+            parent_block['edit_info']['previous_version'] = parent_block['edit_info']['update_version']
+            update_version_keys.append('blocks.{}.edit_info.update_version'.format(parent.usage_id))
         # remove subtree
         def remove_subtree(usage_id):
-            for child in new_blocks[usage_id]['children']:
+            for child in new_blocks[usage_id]['fields'].get('children', []):
                 remove_subtree(child)
             del new_blocks[usage_id]
+        if delete_children:
             remove_subtree(usage_locator.usage_id)
 
         # update index if appropriate and structures
@@ -1062,32 +1105,38 @@ class SplitMongoModuleStore(ModuleStoreBase):
         # this is the only real delete in the system. should it do something else?
         self.course_index.remove(index['_id'])
 
-    def inherit_metadata(self, block_map, block, inheriting_metadata=None):
+    def get_errored_courses(self):
+        """
+        This function doesn't make sense for the mongo modulestore, as structures
+        are loaded on demand, rather than up front
+        """
+        return {}
+
+    def inherit_settings(self, block_map, block, inheriting_settings=None):
         """
-        Updates block with any value
-        that exist in inheriting_metadata and don't appear in block['metadata'],
-        and then inherits block['metadata'] to all of the children in
-        block['children']. Filters by inheritance.INHERITABLE_METADATA
+        Updates block with any inheritable setting set by an ancestor and recurses to children.
         """
         if block is None:
             return
 
-        if inheriting_metadata is None:
-            inheriting_metadata = {}
+        if inheriting_settings is None:
+            inheriting_settings = {}
 
         # the currently passed down values take precedence over any previously cached ones
         # NOTE: this should show the values which all fields would have if inherited: i.e.,
         # not set to the locally defined value but to value set by nearest ancestor who sets it
-        block.setdefault('_inherited_metadata', {}).update(inheriting_metadata)
+        # ALSO NOTE: no xblock should ever define a _inherited_settings field as it will collide w/ this logic.
+        block.setdefault('_inherited_settings', {}).update(inheriting_settings)
 
         # update the inheriting w/ what should pass to children
-        inheriting_metadata = block['_inherited_metadata'].copy()
+        inheriting_settings = block['_inherited_settings'].copy()
+        block_fields = block['fields']
         for field in inheritance.INHERITABLE_METADATA:
-            if field in block['metadata']:
-                inheriting_metadata[field] = block['metadata'][field]
+            if field in block_fields:
+                inheriting_settings[field] = block_fields[field]
 
-        for child in block.get('children', []):
-            self.inherit_metadata(block_map, block_map[child], inheriting_metadata)
+        for child in block_fields.get('children', []):
+            self.inherit_settings(block_map, block_map[child], inheriting_settings)
 
     def descendants(self, block_map, usage_id, depth, descendent_map):
         """
@@ -1104,7 +1153,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
 
         if depth is None or depth > 0:
             depth = depth - 1 if depth is not None else None
-            for child in block_map[usage_id].get('children', []):
+            for child in block_map[usage_id]['fields'].get('children', []):
                 descendent_map = self.descendants(block_map, child, depth,
                     descendent_map)
 
@@ -1217,7 +1266,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
         del new_structure['_id']
         new_structure['previous_version'] = structure['_id']
         new_structure['edited_by'] = user_id
-        new_structure['edited_on'] = datetime.datetime.utcnow()
+        new_structure['edited_on'] = datetime.datetime.now(UTC)
         return new_structure
 
     def _find_local_root(self, element_to_find, possibility, tree):
@@ -1242,3 +1291,31 @@ class SplitMongoModuleStore(ModuleStoreBase):
         self.course_index.update(
             {"_id": index_entry["_id"]},
             {"$set": {"versions.{}".format(branch): new_id}})
+
+    def _partition_fields_by_scope(self, category, fields):
+        """
+        Return dictionary of {scope: {field1: val, ..}..} for the fields of this potential xblock
+
+        :param category: the xblock category
+        :param fields: the dictionary of {fieldname: value}
+        """
+        if fields is None:
+            return {}
+        cls = XModuleDescriptor.load_class(category)
+        result = collections.defaultdict(dict)
+        for field_name, value in fields.iteritems():
+            field = getattr(cls, field_name)
+            result[field.scope][field_name] = value
+        return result
+
+    def _filter_special_fields(self, fields):
+        """
+        Remove any fields which split or its kvs computes or adds but does not want persisted.
+
+        :param fields: a dict of fields
+        """
+        if 'location' in fields:
+            del fields['location']
+        if 'category' in fields:
+            del fields['category']
+        return fields

common/lib/xmodule/xmodule/modulestore/split_mongo/split_mongo_kvs.py

@@ -8,45 +8,49 @@ from .definition_lazy_loader import DefinitionLazyLoader
 SplitMongoKVSid = namedtuple('SplitMongoKVSid', 'id, def_id')
 
 
-# TODO should this be here or w/ x_module or ???
+PROVENANCE_LOCAL = 'local'
+PROVENANCE_DEFAULT = 'default'
+PROVENANCE_INHERITED = 'inherited'
+
 class SplitMongoKVS(KeyValueStore):
     """
     A KeyValueStore that maps keyed data access to one of the 3 data areas
     known to the MongoModuleStore (data, children, and metadata)
     """
-    def __init__(self, definition, children, metadata, _inherited_metadata, location, category):
+
+    def __init__(self, definition, fields, _inherited_settings, location, category):
         """
 
-        :param definition:
-        :param children:
-        :param metadata: the locally defined value for each metadata field
-        :param _inherited_metadata: the value of each inheritable field from above this.
-            Note, metadata may override and disagree w/ this b/c this says what the value
-            should be if metadata is undefined for this field.
+        :param definition: either a lazyloader or definition id for the definition
+        :param fields: a dictionary of the locally set fields
+        :param _inherited_settings: the value of each inheritable field from above this.
+            Note, local fields may override and disagree w/ this b/c this says what the value
+            should be if the field is undefined.
         """
         # ensure kvs's don't share objects w/ others so that changes can't appear in separate ones
         # the particular use case was that changes to kvs's were polluting caches. My thinking was
         # that kvs's should be independent thus responsible for the isolation.
-        if isinstance(definition, DefinitionLazyLoader):
-            self._definition = definition
-        else:
-            self._definition = copy.copy(definition)
-        self._children = copy.copy(children)
-        self._metadata = copy.copy(metadata)
-        self._inherited_metadata = _inherited_metadata
+        self._definition = definition  # either a DefinitionLazyLoader or the db id of the definition.
+        # if the db id, then the definition is presumed to be loaded into _fields
+        self._fields = copy.copy(fields)
+        self._inherited_settings = _inherited_settings
         self._location = location
         self._category = category
 
     def get(self, key):
-        if key.scope == Scope.children:
-            return self._children
-        elif key.scope == Scope.parent:
+        # simplest case, field is directly set
+        if key.field_name in self._fields:
+            return self._fields[key.field_name]
+
+        # parent undefined in editing runtime (I think)
+        if key.scope == Scope.parent:
             return None
+        if key.scope == Scope.children:
+            raise KeyError()
         elif key.scope == Scope.settings:
-            if key.field_name in self._metadata:
-                return self._metadata[key.field_name]
-            elif key.field_name in self._inherited_metadata:
-                return self._inherited_metadata[key.field_name]
+            # get from inheritance since not locally set
+            if key.field_name in self._inherited_settings:
+                return self._inherited_settings[key.field_name]
             else:
                 raise KeyError()
         elif key.scope == Scope.content:
@@ -54,110 +58,118 @@ class SplitMongoKVS(KeyValueStore):
                 return self._location
             elif key.field_name == 'category':
                 return self._category
-            else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    return self._definition.get('data')
-                elif 'data' not in self._definition or key.field_name not in self._definition['data']:
+            elif isinstance(self._definition, DefinitionLazyLoader):
+                self._load_definition()
+                if key.field_name in self._fields:
+                    return self._fields[key.field_name]
+
             raise KeyError()
         else:
-                    return self._definition['data'][key.field_name]
-        else:
             raise InvalidScopeError(key.scope)
 
     def set(self, key, value):
-        # TODO cache db update implications & add method to invoke
-        if key.scope == Scope.children:
-            self._children = value
-            # TODO remove inheritance from any orphaned exchildren
-            # TODO add inheritance to any new children
-        elif key.scope == Scope.settings:
-            # TODO if inheritable, push down to children who don't override
-            self._metadata[key.field_name] = value
-        elif key.scope == Scope.content:
+        # handle any special cases
+        if key.scope not in [Scope.children, Scope.settings, Scope.content]:
+            raise InvalidScopeError(key.scope)
+        if key.scope == Scope.content:
             if key.field_name == 'location':
-                self._location = value
+                self._location = value  # is changing this legal?
+                return
             elif key.field_name == 'category':
-                self._category = value
-            else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    self._definition.get('data')
+                # TODO should this raise an exception? that is, should xblock types be mungable?
+                return
             else:
-                    self._definition.setdefault('data', {})[key.field_name] = value
-        else:
-            raise InvalidScopeError(key.scope)
+                self._load_definition()
+
+        # set the field
+        self._fields[key.field_name] = value
+
+        # handle any side effects
+        # if key.scope == Scope.children:
+            # TODO remove inheritance from any exchildren
+            # TODO add inheritance to any new children
+        # if key.scope == Scope.settings:
+            # TODO if inheritable, push down to children
 
     def delete(self, key):
-        # TODO cache db update implications & add method to invoke
-        if key.scope == Scope.children:
-            self._children = []
-        elif key.scope == Scope.settings:
-            # TODO if inheritable, ensure _inherited_metadata has value from above and
-            # revert children to that value
-            if key.field_name in self._metadata:
-                del self._metadata[key.field_name]
-        elif key.scope == Scope.content:
-            # don't allow deletion of location nor category
+        # handle any special cases
+        if key.scope not in [Scope.children, Scope.settings, Scope.content]:
+            raise InvalidScopeError(key.scope)
+        if key.scope == Scope.content:
             if key.field_name == 'location':
-                pass
+                return  # noop
             elif key.field_name == 'category':
-                pass
+                # TODO should this raise an exception? that is, should xblock types be mungable?
+                return  # noop
             else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    self._definition.setdefault('data', None)
-                else:
-                    try:
-                        del self._definition['data'][key.field_name]
-                    except KeyError:
-                        pass
-        else:
-            raise InvalidScopeError(key.scope)
+                self._load_definition()
+
+        # delete the field value
+        if key.field_name in self._fields:
+            del self._fields[key.field_name]
+
+        # handle any side effects
+        # if key.scope == Scope.children:
+            # TODO remove inheritance from any exchildren
+        # if key.scope == Scope.settings:
+            # TODO if inheritable, push down _inherited_settings value to children
 
     def has(self, key):
-        if key.scope in (Scope.children, Scope.parent):
-            return True
-        elif key.scope == Scope.settings:
-            return key.field_name in self._metadata or key.field_name in self._inherited_metadata
-        elif key.scope == Scope.content:
+        # handle any special cases
+        if key.scope == Scope.content:
             if key.field_name == 'location':
                 return True
             elif key.field_name == 'category':
                 return self._category is not None
             else:
-                if isinstance(self._definition, DefinitionLazyLoader):
-                    self._definition = self._definition.fetch()
-                if (key.field_name == 'data' and
-                        not isinstance(self._definition.get('data'), dict)):
-                    return self._definition.get('data') is not None
-                else:
-                    return key.field_name in self._definition.get('data', {})
-        else:
-            return False
+                self._load_definition()
+        elif key.scope == Scope.parent:
+            return True
+
+        # it's not clear whether inherited values should return True. Right now they don't
+        # if someone changes it so that they do, then change any tests of field.name in xx._model_data
+        return key.field_name in self._fields
 
-    def get_data(self):
+    # would like to just take a key, but there's a bunch of magic in DbModel for constructing the key via
+    # a private method
+    def field_value_provenance(self, key_scope, key_name):
         """
-        Intended only for use by persistence layer to get the native definition['data'] rep
+        Where the field value comes from: one of [PROVENANCE_LOCAL, PROVENANCE_DEFAULT, PROVENANCE_INHERITED].
         """
-        if isinstance(self._definition, DefinitionLazyLoader):
-            self._definition = self._definition.fetch()
-        return self._definition.get('data')
+        # handle any special cases
+        if key_scope == Scope.content:
+            if key_name == 'location':
+                return PROVENANCE_LOCAL
+            elif key_name == 'category':
+                return PROVENANCE_LOCAL
+            else:
+                self._load_definition()
+                if key_name in self._fields:
+                    return PROVENANCE_LOCAL
+                else:
+                    return PROVENANCE_DEFAULT
+        elif key_scope == Scope.parent:
+            return PROVENANCE_DEFAULT
+        elif key_name in self._fields:
+            return PROVENANCE_LOCAL
+        elif key_scope == Scope.settings and key_name in self._inherited_settings:
+            return PROVENANCE_INHERITED
+        else:
+            return PROVENANCE_DEFAULT
 
-    def get_own_metadata(self):
+    def get_inherited_settings(self):
         """
-        Get the metadata explicitly set on this element.
+        Get the metadata set by the ancestors (which own metadata may override or not)
         """
-        return self._metadata
+        return self._inherited_settings
 
-    def get_inherited_metadata(self):
+    def _load_definition(self):
         """
-        Get the metadata set by the ancestors (which own metadata may override or not)
+        Update fields w/ the lazily loaded definitions
         """
-        return self._inherited_metadata
+        if isinstance(self._definition, DefinitionLazyLoader):
+            persisted_definition = self._definition.fetch()
+            if persisted_definition is not None:
+                self._fields.update(persisted_definition.get('fields'))
+                # do we want to cache any of the edit_info?
+            self._definition = None  # already loaded

common/lib/xmodule/xmodule/modulestore/tests/persistent_factories.py

@@ -16,8 +16,7 @@ class PersistentCourseFactory(factory.Factory):
     * prettyid: defaults to 999
     * display_name
     * user_id
-    * data (optional) the data payload to save in the course item
-    * metadata (optional) the metadata payload. If display_name is in the metadata, that takes
+    * fields (optional) the settings and content payloads. If display_name is in the metadata, that takes
     precedence over any display_name provided directly.
     """
     FACTORY_FOR = CourseDescriptor
@@ -28,7 +27,7 @@ class PersistentCourseFactory(factory.Factory):
     user_id = "test_user"
     data = None
     metadata = None
-    master_version = 'draft'
+    master_branch = 'draft'
 
     # pylint: disable=W0613
     @classmethod
@@ -38,17 +37,14 @@ class PersistentCourseFactory(factory.Factory):
         prettyid = kwargs.get('prettyid')
         display_name = kwargs.get('display_name')
         user_id = kwargs.get('user_id')
-        data = kwargs.get('data')
-        metadata = kwargs.get('metadata', {})
-        if metadata is None:
-            metadata = {}
-        if 'display_name' not in metadata:
-            metadata['display_name'] = display_name
+        fields = kwargs.get('fields', {})
+        if display_name and 'display_name' not in fields:
+            fields['display_name'] = display_name
 
         # Write the data to the mongo datastore
         new_course = modulestore('split').create_course(
-            org, prettyid, user_id, metadata=metadata, course_data=data, id_root=prettyid,
-            master_version=kwargs.get('master_version'))
+            org, prettyid, user_id, fields=fields, id_root=prettyid,
+            master_branch=kwargs.get('master_branch'))
 
         return new_course
 
@@ -70,26 +66,23 @@ class ItemFactory(factory.Factory):
         """
         Uses *kwargs*:
 
-        *parent_location* (required): the location of the course & possibly parent
+        :param parent_location: (required) the location of the course & possibly parent
 
-        *category* (defaults to 'chapter')
+        :param category: (defaults to 'chapter')
 
-        *data* (optional): the data for the item
+        :param fields: (optional) the data for the item
 
-        definition_locator (optional): the DescriptorLocator for the definition this uses or branches
+        :param definition_locator (optional): the DescriptorLocator for the definition this uses or branches
 
-        *display_name* (optional): the display name of the item
-
-        *metadata* (optional): dictionary of metadata attributes (display_name here takes
-        precedence over the above attr)
+        :param display_name (optional): the display name of the item
         """
-        metadata = kwargs.get('metadata', {})
-        if 'display_name' not in metadata and 'display_name' in kwargs:
-            metadata['display_name'] = kwargs['display_name']
+        fields = kwargs.get('fields', {})
+        if 'display_name' not in fields and 'display_name' in kwargs:
+            fields['display_name'] = kwargs['display_name']
 
         return modulestore('split').create_item(kwargs['parent_location'], kwargs['category'],
             kwargs['user_id'], definition_locator=kwargs.get('definition_locator'),
-            new_def_data=kwargs.get('data'), metadata=metadata)
+            fields=fields)
 
     @classmethod
     def _build(cls, target_class, *args, **kwargs):

common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py

@@ -187,6 +187,7 @@ class SplitModuleCourseTests(SplitModuleTest):
         self.assertEqual(course.category, 'course')
         self.assertEqual(len(course.tabs), 6)
         self.assertEqual(course.display_name, "The Ancient Greek Hero")
+        self.assertEqual(course.lms.graceperiod, datetime.timedelta(hours=2))
         self.assertIsNone(course.advertised_start)
         self.assertEqual(len(course.children), 0)
         self.assertEqual(course.definition_locator.definition_id, "head12345_11")
@@ -438,12 +439,12 @@ class SplitModuleItemTests(SplitModuleTest):
             qualifiers=
             {
                 'category': 'chapter',
-                'metadata': {'display_name': {'$regex': 'Hera'}}
+                'fields': {'display_name': {'$regex': 'Hera'}}
             }
         )
         self.assertEqual(len(matches), 2)
 
-        matches = modulestore().get_items(locator, qualifiers={'children': 'chapter2'})
+        matches = modulestore().get_items(locator, qualifiers={'fields': {'children': 'chapter2'}})
         self.assertEqual(len(matches), 1)
         self.assertEqual(matches[0].location.usage_id, 'head12345')
 
@@ -507,8 +508,7 @@ class TestItemCrud(SplitModuleTest):
 
     def test_create_minimal_item(self):
         """
-        create_item(course_or_parent_locator, category, user, definition_locator=None, new_def_data=None,
-        metadata=None): new_desciptor
+        create_item(course_or_parent_locator, category, user, definition_locator=None, fields): new_desciptor
         """
         # grab link to course to ensure new versioning works
         locator = CourseLocator(course_id="GreekHero", branch='draft')
@@ -518,7 +518,7 @@ class TestItemCrud(SplitModuleTest):
         category = 'sequential'
         new_module = modulestore().create_item(
             locator, category, 'user123',
-            metadata={'display_name': 'new sequential'}
+            fields={'display_name': 'new sequential'}
         )
         # check that course version changed and course's previous is the other one
         self.assertEqual(new_module.location.course_id, "GreekHero")
@@ -553,7 +553,7 @@ class TestItemCrud(SplitModuleTest):
         category = 'chapter'
         new_module = modulestore().create_item(
             locator, category, 'user123',
-            metadata={'display_name': 'new chapter'},
+            fields={'display_name': 'new chapter'},
             definition_locator=DescriptionLocator("chapter12345_2")
         )
         # check that course version changed and course's previous is the other one
@@ -574,15 +574,13 @@ class TestItemCrud(SplitModuleTest):
         new_payload = "<problem>empty</problem>"
         new_module = modulestore().create_item(
             locator, category, 'anotheruser',
-            metadata={'display_name': 'problem 1'},
-            new_def_data=new_payload
+            fields={'display_name': 'problem 1', 'data': new_payload},
         )
         another_payload = "<problem>not empty</problem>"
         another_module = modulestore().create_item(
             locator, category, 'anotheruser',
-            metadata={'display_name': 'problem 2'},
+            fields={'display_name': 'problem 2', 'data': another_payload},
             definition_locator=DescriptionLocator("problem12345_3_1"),
-            new_def_data=another_payload
         )
         # check that course version changed and course's previous is the other one
         parent = modulestore().get_item(locator)
@@ -616,6 +614,7 @@ class TestItemCrud(SplitModuleTest):
         self.assertNotEqual(problem.max_attempts, 4, "Invalidates rest of test")
 
         problem.max_attempts = 4
+        problem.save()  # decache above setting into the kvs
         updated_problem = modulestore().update_item(problem, 'changeMaven')
         # check that course version changed and course's previous is the other one
         self.assertEqual(updated_problem.definition_locator.definition_id, pre_def_id)
@@ -651,6 +650,7 @@ class TestItemCrud(SplitModuleTest):
         # reorder children
         self.assertGreater(len(block.children), 0, "meaningless test")
         moved_child = block.children.pop()
+        block.save()  # decache model changes
         updated_problem = modulestore().update_item(block, 'childchanger')
         # check that course version changed and course's previous is the other one
         self.assertEqual(updated_problem.definition_locator.definition_id, pre_def_id)
@@ -660,6 +660,7 @@ class TestItemCrud(SplitModuleTest):
         locator.usage_id = "chapter1"
         other_block = modulestore().get_item(locator)
         other_block.children.append(moved_child)
+        other_block.save()  # decache model changes
         other_updated = modulestore().update_item(other_block, 'childchanger')
         self.assertIn(moved_child, other_updated.children)
 
@@ -673,6 +674,7 @@ class TestItemCrud(SplitModuleTest):
         pre_version_guid = block.location.version_guid
 
         block.grading_policy['GRADER'][0]['min_count'] = 13
+        block.save()  # decache model changes
         updated_block = modulestore().update_item(block, 'definition_changer')
 
         self.assertNotEqual(updated_block.definition_locator.definition_id, pre_def_id)
@@ -689,15 +691,13 @@ class TestItemCrud(SplitModuleTest):
         new_payload = "<problem>empty</problem>"
         modulestore().create_item(
             locator, category, 'test_update_manifold',
-            metadata={'display_name': 'problem 1'},
-            new_def_data=new_payload
+            fields={'display_name': 'problem 1', 'data': new_payload},
         )
         another_payload = "<problem>not empty</problem>"
         modulestore().create_item(
             locator, category, 'test_update_manifold',
-            metadata={'display_name': 'problem 2'},
+            fields={'display_name': 'problem 2', 'data': another_payload},
             definition_locator=DescriptionLocator("problem12345_3_1"),
-            new_def_data=another_payload
         )
         # pylint: disable=W0212
         modulestore()._clear_cache()
@@ -712,6 +712,7 @@ class TestItemCrud(SplitModuleTest):
         block.children = block.children[1:] + [block.children[0]]
         block.advertised_start = "Soon"
 
+        block.save()  # decache model changes
         updated_block = modulestore().update_item(block, "test_update_manifold")
         self.assertNotEqual(updated_block.definition_locator.definition_id, pre_def_id)
         self.assertNotEqual(updated_block.location.version_guid, pre_version_guid)
@@ -733,7 +734,7 @@ class TestItemCrud(SplitModuleTest):
         # delete a leaf
         problems = modulestore().get_items(reusable_location, {'category': 'problem'})
         locn_to_del = problems[0].location
-        new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user')
+        new_course_loc = modulestore().delete_item(locn_to_del, 'deleting_user', delete_children=True)
         deleted = BlockUsageLocator(course_id=reusable_location.course_id,
                                     branch=reusable_location.branch,
                                     usage_id=locn_to_del.usage_id)
@@ -748,7 +749,7 @@ class TestItemCrud(SplitModuleTest):
 
         # delete a subtree
         nodes = modulestore().get_items(reusable_location, {'category': 'chapter'})
-        new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user')
+        new_course_loc = modulestore().delete_item(nodes[0].location, 'deleting_user', delete_children=True)
         # check subtree
 
         def check_subtree(node):
@@ -855,7 +856,7 @@ class TestCourseCreation(SplitModuleTest):
         # using new_draft.location will insert the chapter under the course root
         new_item = modulestore().create_item(
             new_draft.location, 'chapter', 'leech_master',
-            metadata={'display_name': 'new chapter'}
+            fields={'display_name': 'new chapter'}
         )
         new_draft_locator.version_guid = None
         new_index = modulestore().get_course_index_info(new_draft_locator)
@@ -887,20 +888,18 @@ class TestCourseCreation(SplitModuleTest):
         original_locator = CourseLocator(course_id="contender", branch='draft')
         original = modulestore().get_course(original_locator)
         original_index = modulestore().get_course_index_info(original_locator)
-        data_payload = {}
-        metadata_payload = {}
+        fields = {}
         for field in original.fields:
             if field.scope == Scope.content and field.name != 'location':
-                data_payload[field.name] = getattr(original, field.name)
+                fields[field.name] = getattr(original, field.name)
             elif field.scope == Scope.settings:
-                metadata_payload[field.name] = getattr(original, field.name)
-        data_payload['grading_policy']['GRADE_CUTOFFS'] = {'A': .9, 'B': .8, 'C': .65}
-        metadata_payload['display_name'] = 'Derivative'
+                fields[field.name] = getattr(original, field.name)
+        fields['grading_policy']['GRADE_CUTOFFS'] = {'A': .9, 'B': .8, 'C': .65}
+        fields['display_name'] = 'Derivative'
         new_draft = modulestore().create_course(
             'leech', 'derivative', 'leech_master', id_root='counter',
             versions_dict={'draft': original_index['versions']['draft']},
-            course_data=data_payload,
-            metadata=metadata_payload
+            fields=fields
         )
         new_draft_locator = new_draft.location
         self.assertRegexpMatches(new_draft_locator.course_id, r'counter.*')
@@ -913,10 +912,10 @@ class TestCourseCreation(SplitModuleTest):
         self.assertGreaterEqual(new_index["edited_on"], pre_time)
         self.assertLessEqual(new_index["edited_on"], datetime.datetime.now(UTC))
         self.assertEqual(new_index['edited_by'], 'leech_master')
-        self.assertEqual(new_draft.display_name, metadata_payload['display_name'])
+        self.assertEqual(new_draft.display_name, fields['display_name'])
         self.assertDictEqual(
             new_draft.grading_policy['GRADE_CUTOFFS'],
-            data_payload['grading_policy']['GRADE_CUTOFFS']
+            fields['grading_policy']['GRADE_CUTOFFS']
         )
 
     def test_update_course_index(self):

common/lib/xmodule/xmodule/x_module.py

@@ -587,33 +587,20 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         Creates an instance of this descriptor from the supplied json_data.
         This may be overridden by subclasses
 
-        json_data: A json object with the keys 'definition' and 'metadata',
-            definition: A json object with the keys 'data' and 'children'
-                data: A json value
-                children: A list of edX Location urls
-            metadata: A json object with any keys
-
-        This json_data is transformed to model_data using the following rules:
-            1) The model data contains all of the fields from metadata
-            2) The model data contains the 'children' array
-            3) If 'definition.data' is a json object, model data contains all of its fields
-               Otherwise, it contains the single field 'data'
-            4) Any value later in this list overrides a value earlier in this list
-
         json_data:
         - 'category': the xmodule category (required)
-        - 'metadata': a dict of locally set metadata (not inherited)
-        - 'children': a list of children's usage_ids w/in this course
-        - 'definition':
+        - 'fields': a dict of locally set fields (not inherited)
+        - 'definition': (optional) the db id for the definition record (not the definition content) or a
+        definitionLazyLoader
         - '_id' (optional): the usage_id of this. Will generate one if not given one.
         """
         usage_id = json_data.get('_id', None)
-        if not '_inherited_metadata' in json_data and parent_xblock is not None:
-            json_data['_inherited_metadata'] = parent_xblock.xblock_kvs.get_inherited_metadata().copy()
-            json_metadata = json_data.get('metadata', {})
+        if not '_inherited_settings' in json_data and parent_xblock is not None:
+            json_data['_inherited_settings'] = parent_xblock.xblock_kvs.get_inherited_settings().copy()
+            json_fields = json_data.get('fields', {})
             for field in inheritance.INHERITABLE_METADATA:
-                if field in json_metadata:
-                    json_data['_inherited_metadata'][field] = json_metadata[field]
+                if field in json_fields:
+                    json_data['_inherited_settings'][field] = json_fields[field]
 
         new_block = system.xblock_from_json(cls, usage_id, json_data)
         if parent_xblock is not None:
@@ -736,6 +723,27 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
         # We are not allowing editing of xblock tag and name fields at this time (for any component).
         return [XBlock.tags, XBlock.name]
 
+    def get_set_fields_by_scope(self, scope=Scope.content):
+        """
+        Get a dictionary of the fields for the given scope which are set explicitly on this xblock. (Including
+        any set to None.)
+        """
+        if scope == Scope.settings and hasattr(self, '_inherited_metadata'):
+            inherited_metadata = getattr(self, '_inherited_metadata')
+            result = {}
+            for field in self.fields:
+                if (field.scope == scope and
+                        field.name in self._model_data and
+                        field.name not in inherited_metadata):
+                    result[field.name] = getattr(self, field.name)
+            return result
+        else:
+            result = {}
+            for field in self.fields:
+                if (field.scope == scope and field.name in self._model_data):
+                    result[field.name] = getattr(self, field.name)
+            return result
+
     @property
     def editable_metadata_fields(self):
         """

common/test/data/splitmongo_json/definitions.json

@@ -2,7 +2,7 @@
     {
         "_id":"head12345_12",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -43,15 +43,17 @@
             },
             "wiki_slug":null
         },
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364481713238},
             "previous_version":"head12345_11",
             "original_version":"head12345_10"
+        }
     },
     {
         "_id":"head12345_11",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -92,15 +94,17 @@
             },
             "wiki_slug":null
         },
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364481713238},
             "previous_version":"head12345_10",
             "original_version":"head12345_10"
+        }
     },
     {
         "_id":"head12345_10",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -141,15 +145,17 @@
             },
             "wiki_slug":null
         },
+        "edit_info": {
             "edited_by":"test@edx.org",
             "edited_on":{"$date": 1364473713238},
             "previous_version":null,
             "original_version":"head12345_10"
+        }
     },
     {
         "_id":"head23456_1",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -190,15 +196,17 @@
             },
             "wiki_slug":null
         },
+        "edit_info": {
             "edited_by":"test@edx.org",
             "edited_on":{"$date": 1364481313238},
             "previous_version":"head23456_0",
             "original_version":"head23456_0"
+        }
     },
     {
         "_id":"head23456_0",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -239,15 +247,17 @@
             },
             "wiki_slug":null
         },
+        "edit_info": {
             "edited_by":"test@edx.org",
             "edited_on":{"$date" : 1364481313238},
             "previous_version":null,
             "original_version":"head23456_0"
+        }
     },
     {
         "_id":"head345679_1",
         "category":"course",
-        "data":{
+        "fields":{
             "textbooks":[
                 
             ],
@@ -281,54 +291,66 @@
             },
             "wiki_slug":null
         },
+        "edit_info": {
             "edited_by":"test@edx.org",
             "edited_on":{"$date" : 1364481313238},
             "previous_version":null,
             "original_version":"head23456_0"
+        }
     },
     {
         "_id":"chapter12345_1",
         "category":"chapter",
-        "data":null,
+        "fields":{},
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
             "original_version":"chapter12345_1"
+        }
     },
     {
         "_id":"chapter12345_2",
         "category":"chapter",
-        "data":null,
+        "fields":{},
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
             "original_version":"chapter12345_2"
+        }
     },
     {
         "_id":"chapter12345_3",
         "category":"chapter",
-        "data":null,
+        "fields":{},
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
             "original_version":"chapter12345_3"
+        }
     },
     {
         "_id":"problem12345_3_1",
         "category":"problem",
-        "data":"",
+        "fields": {"data": ""},
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
             "original_version":"problem12345_3_1"
+        }
     },
     {
         "_id":"problem12345_3_2",
         "category":"problem",
-        "data":"",
+        "fields": {"data": ""},
+        "edit_info": {
             "edited_by":"testassist@edx.org",
             "edited_on":{"$date" : 1364483713238},
             "previous_version":null,
             "original_version":"problem12345_3_2"
         }
+    }
 ]
\ No newline at end of file

common/test/data/splitmongo_json/structures.json

@@ -10,14 +10,14 @@
         },
         "blocks":{
             "head12345":{
+                "category":"course",
+                "definition":"head12345_12",
+                "fields":{
                     "children":[
                         "chapter1",
                         "chapter2",
                         "chapter3"
                     ],
-                "category":"course",
-                "definition":"head12345_12",
-                "metadata":{
                     "end":"2013-06-13T04:30",
                     "tabs":[
                         {
@@ -54,88 +54,99 @@
                     "advertised_start":"Fall 2013",
                     "display_name":"The Ancient Greek Hero"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd0000" },
                     "previous_version":{ "$oid" : "1d00000000000000dddd1111" },
                     "edited_by":"testassist@edx.org",
                     "edited_on":{
                         "$date":1364483713238
                     }
+                }
             },
             "chapter1":{
+                "category":"chapter",
+                "definition":"chapter12345_1",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"chapter",
-                "definition":"chapter12345_1",
-                "metadata":{
                     "display_name":"Hercules"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd0000" },
                     "previous_version":null,
                     "edited_by":"testassist@edx.org",
                     "edited_on":{
                         "$date":1364483713238
                     }
+                }
             },
             "chapter2":{
+                "category":"chapter",
+                "definition":"chapter12345_2",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"chapter",
-                "definition":"chapter12345_2",
-                "metadata":{
                     "display_name":"Hera heckles Hercules"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd0000" },
                     "previous_version":null,
                     "edited_by":"testassist@edx.org",
                     "edited_on":{
                         "$date":1364483713238
                     }
+                }
             },
             "chapter3":{
+                "category":"chapter",
+                "definition":"chapter12345_3",
+                "fields":{
                     "children":[
                         "problem1",
                         "problem3_2"
                     ],
-                "category":"chapter",
-                "definition":"chapter12345_3",
-                "metadata":{
                     "display_name":"Hera cuckolds Zeus"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd0000" },
                     "previous_version":null,
                     "edited_by":"testassist@edx.org",
                     "edited_on":{
                         "$date":1364483713238
                     }
+                }
             },
             "problem1":{
+                "category":"problem",
+                "definition":"problem12345_3_1",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"problem",
-                "definition":"problem12345_3_1",
-                "metadata":{
                     "display_name":"Problem 3.1",
                     "graceperiod":"4 hours 0 minutes 0 seconds"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd0000" },
                     "previous_version":null,
                     "edited_by":"testassist@edx.org",
                     "edited_on":{
                         "$date":1364483713238
                     }
+                }
             },
             "problem3_2":{
+                "category":"problem",
+                "definition":"problem12345_3_2",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"problem",
-                "definition":"problem12345_3_2",
-                "metadata":{
                     "display_name":"Problem 3.2"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd0000" },
                     "previous_version":null,
                     "edited_by":"testassist@edx.org",
@@ -144,6 +155,7 @@
                     }
                 }
             }
+        }
     },
     {
         "_id": { "$oid" : "1d00000000000000dddd1111"},
@@ -156,12 +168,12 @@
         },
         "blocks":{
             "head12345":{
+                "category":"course",
+                "definition":"head12345_11",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"course",
-                "definition":"head12345_11",
-                "metadata":{
                     "end":"2013-04-13T04:30",
                     "tabs":[
                         {
@@ -198,6 +210,7 @@
                     "advertised_start":null,
                     "display_name":"The Ancient Greek Hero"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd1111" },
                     "previous_version":{ "$oid" : "1d00000000000000dddd3333" },
                     "edited_by":"testassist@edx.org",
@@ -206,6 +219,7 @@
                     }
                 }
             }
+        }
     },
     {
         "_id": { "$oid" : "1d00000000000000dddd3333"},
@@ -218,12 +232,12 @@
         },
         "blocks":{
             "head12345":{
+                "category":"course",
+                "definition":"head12345_10",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"course",
-                "definition":"head12345_10",
-                "metadata":{
                     "end":null,
                     "tabs":[
                         {
@@ -250,6 +264,7 @@
                     "advertised_start":null,
                     "display_name":"The Ancient Greek Hero"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd3333" },
                     "previous_version":null,
                     "edited_by":"test@edx.org",
@@ -258,6 +273,7 @@
                     }
                 }
             }
+        }
     },
     {
         "_id": { "$oid" : "1d00000000000000dddd2222"},
@@ -270,12 +286,12 @@
         },
         "blocks":{
             "head23456":{
+                "category":"course",
+                "definition":"head23456_1",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"course",
-                "definition":"head23456_1",
-                "metadata":{
                     "end":null,
                     "tabs":[
                         {
@@ -302,12 +318,14 @@
                     "advertised_start":null,
                     "display_name":"The most wonderful course"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd2222" },
                     "previous_version":{ "$oid" : "1d00000000000000dddd4444" },
                     "edited_by":"test@edx.org",
                     "edited_on":{
                         "$date":1364481313238
                     }
+                }
                 
             }
         }
@@ -323,12 +341,12 @@
         },
         "blocks":{
             "head23456":{
+                "category":"course",
+                "definition":"head23456_0",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"course",
-                "definition":"head23456_0",
-                "metadata":{
                     "end":null,
                     "tabs":[
                         {
@@ -355,6 +373,7 @@
                     "advertised_start":null,
                     "display_name":"A wonderful course"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd4444" },
                     "previous_version":null,
                     "edited_by":"test@edx.org",
@@ -363,6 +382,7 @@
                     }
                 }
             }
+        }
     },
     {
         "_id": { "$oid" : "1d00000000000000eeee0000"},
@@ -375,12 +395,12 @@
         },
         "blocks":{
             "head23456":{
+                "category":"course",
+                "definition":"head23456_1",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"course",
-                "definition":"head23456_1",
-                "metadata":{
                     "end":null,
                     "tabs":[
                         {
@@ -407,6 +427,7 @@
                     "advertised_start":null,
                     "display_name":"The most wonderful course"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000eeee0000" },
                     "previous_version":null,
                     "edited_by":"test@edx.org",
@@ -415,6 +436,7 @@
                     }
                 }
             }
+        }
     },
     {
         "_id": { "$oid" : "1d00000000000000dddd5555"},
@@ -427,12 +449,12 @@
         },
         "blocks":{
             "head345679":{
+                "category":"course",
+                "definition":"head345679_1",
+                "fields":{
                     "children":[
                         
                     ],
-                "category":"course",
-                "definition":"head345679_1",
-                "metadata":{
                     "end":null,
                     "tabs":[
                         {
@@ -459,6 +481,7 @@
                     "advertised_start":null,
                     "display_name":"Yet another contender"
                 },
+                "edit_info": {
                     "update_version":{ "$oid" : "1d00000000000000dddd5555" },
                     "previous_version":null,
                     "edited_by":"test@guestx.edu",
@@ -468,4 +491,5 @@
                 }
             }
         }
+    }
 ]