Skip to content

E
edx-platform
Commit dda08930 by Don Mitchell
Options

Compute ObjectId locally rather than delegating to Mongo db 

Was following a std sql pattern, but non-sql db's are slow write
and pymongo's ObjectId() is supposed to generate a true GUID
equivalent to waiting for the db to do so.
parent b0689a4e
Showing with 56 additions and 65 deletions
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
...@@ -20,6 +20,7 @@ from .definition_lazy_loader import DefinitionLazyLoader ...@@ -20,6 +20,7 @@ from .definition_lazy_loader import DefinitionLazyLoader
from .caching_descriptor_system import CachingDescriptorSystem from .caching_descriptor_system import CachingDescriptorSystem
from xblock.fields import Scope from xblock.fields import Scope
from xblock.runtime import Mixologist from xblock.runtime import Mixologist
from bson.objectid import ObjectId
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
#============================================================================== #==============================================================================
...@@ -519,6 +520,7 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -519,6 +520,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
else: else:
result.setdefault(version['blocks'][usage_id]['edit_info']['previous_version'], set()).add( result.setdefault(version['blocks'][usage_id]['edit_info']['previous_version'], set()).add(
version['blocks'][usage_id]['edit_info']['update_version']) version['blocks'][usage_id]['edit_info']['update_version'])
# more than one possible_root means usage was added and deleted > 1x. # more than one possible_root means usage was added and deleted > 1x.
if len(possible_roots) > 1: if len(possible_roots) > 1:
# find the history segment including block_locator's version # find the history segment including block_locator's version
...@@ -552,20 +554,21 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -552,20 +554,21 @@ class SplitMongoModuleStore(ModuleStoreBase):
:param user_id: request.user object :param user_id: request.user object
""" """
new_def_data = self._filter_special_fields(new_def_data) new_def_data = self._filter_special_fields(new_def_data)
new_id = ObjectId()
document = { document = {
'_id': new_id,
"category" : category, "category" : category,
"fields": new_def_data, "fields": new_def_data,
"edit_info": { "edit_info": {
"_id": new_id,
"edited_by": user_id, "edited_by": user_id,
"edited_on": datetime.datetime.now(UTC), "edited_on": datetime.datetime.now(UTC),
"previous_version": None, "previous_version": None,
"original_version": None "original_version": new_id,
} }
} }
new_id = self.definitions.insert(document) self.definitions.insert(document)
definition_locator = DefinitionLocator(new_id) definition_locator = DefinitionLocator(new_id)
document['edit_info']['original_version'] = new_id
self.definitions.update({'_id': new_id}, {'$set': {"edit_info.original_version": new_id}})
return definition_locator return definition_locator
def update_definition_from_data(self, definition_locator, new_def_data, user_id): def update_definition_from_data(self, definition_locator, new_def_data, user_id):
...@@ -589,15 +592,17 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -589,15 +592,17 @@ class SplitMongoModuleStore(ModuleStoreBase):
old_definition = self.definitions.find_one({'_id': definition_locator.definition_id}) old_definition = self.definitions.find_one({'_id': definition_locator.definition_id})
if old_definition is None: if old_definition is None:
raise ItemNotFoundError(definition_locator.url()) raise ItemNotFoundError(definition_locator.url())
del old_definition['_id']
if needs_saved(): if needs_saved():
# new id to create new version
old_definition['_id'] = ObjectId()
old_definition['fields'] = new_def_data old_definition['fields'] = new_def_data
old_definition['edit_info']['edited_by'] = user_id old_definition['edit_info']['edited_by'] = user_id
old_definition['edit_info']['edited_on'] = datetime.datetime.now(UTC) old_definition['edit_info']['edited_on'] = datetime.datetime.now(UTC)
# previous version id
old_definition['edit_info']['previous_version'] = definition_locator.definition_id old_definition['edit_info']['previous_version'] = definition_locator.definition_id
new_id = self.definitions.insert(old_definition) self.definitions.insert(old_definition)
return DefinitionLocator(new_id), True return DefinitionLocator(old_definition['_id']), True
else: else:
return definition_locator, False return definition_locator, False
...@@ -707,7 +712,9 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -707,7 +712,9 @@ class SplitMongoModuleStore(ModuleStoreBase):
else: else:
new_structure = self._version_structure(structure, user_id) new_structure = self._version_structure(structure, user_id)
# generate an id new_id = new_structure['_id']
# generate usage id
if usage_id is not None: if usage_id is not None:
if usage_id in new_structure['blocks']: if usage_id in new_structure['blocks']:
raise DuplicateItemError(usage_id, self, 'structures') raise DuplicateItemError(usage_id, self, 'structures')
...@@ -716,7 +723,6 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -716,7 +723,6 @@ class SplitMongoModuleStore(ModuleStoreBase):
else: else:
new_usage_id = self._generate_usage_id(new_structure['blocks'], category) new_usage_id = self._generate_usage_id(new_structure['blocks'], category)
update_version_keys = ['blocks.{}.edit_info.update_version'.format(new_usage_id)]
block_fields = partitioned_fields.get(Scope.settings, {}) block_fields = partitioned_fields.get(Scope.settings, {})
if Scope.children in partitioned_fields: if Scope.children in partitioned_fields:
block_fields.update(partitioned_fields[Scope.children]) block_fields.update(partitioned_fields[Scope.children])
...@@ -727,9 +733,11 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -727,9 +733,11 @@ class SplitMongoModuleStore(ModuleStoreBase):
'edit_info': { 'edit_info': {
'edited_on': datetime.datetime.now(UTC), 'edited_on': datetime.datetime.now(UTC),
'edited_by': user_id, 'edited_by': user_id,
'previous_version': None 'previous_version': None,
'update_version': new_id,
} }
} }
# if given parent, add new block as child and update parent's version # if given parent, add new block as child and update parent's version
parent = None parent = None
if isinstance(course_or_parent_locator, BlockUsageLocator) and course_or_parent_locator.usage_id is not None: if isinstance(course_or_parent_locator, BlockUsageLocator) and course_or_parent_locator.usage_id is not None:
...@@ -739,22 +747,14 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -739,22 +747,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
parent['edit_info']['edited_on'] = datetime.datetime.now(UTC) parent['edit_info']['edited_on'] = datetime.datetime.now(UTC)
parent['edit_info']['edited_by'] = user_id parent['edit_info']['edited_by'] = user_id
parent['edit_info']['previous_version'] = parent['edit_info']['update_version'] parent['edit_info']['previous_version'] = parent['edit_info']['update_version']
update_version_keys.append( parent['edit_info']['update_version'] = new_id
'blocks.{}.edit_info.update_version'.format(course_or_parent_locator.usage_id)
)
if continue_version: if continue_version:
new_id = structure['_id']
# db update # db update
self.structures.update({'_id': new_id}, new_structure) self.structures.update({'_id': new_id}, new_structure)
# clear cache so things get refetched and inheritance recomputed # clear cache so things get refetched and inheritance recomputed
self._clear_cache(new_id) self._clear_cache(new_id)
else: else:
new_id = self.structures.insert(new_structure) self.structures.insert(new_structure)
update_version_payload = {key: new_id for key in update_version_keys}
self.structures.update(
{'_id': new_id},
{'$set': update_version_payload})
# update the index entry if appropriate # update the index entry if appropriate
if index_entry is not None: if index_entry is not None:
...@@ -815,22 +815,26 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -815,22 +815,26 @@ class SplitMongoModuleStore(ModuleStoreBase):
# if building a wholly new structure # if building a wholly new structure
if versions_dict is None or master_branch not in versions_dict: if versions_dict is None or master_branch not in versions_dict:
# create new definition and structure # create new definition and structure
definition_id = ObjectId()
definition_entry = { definition_entry = {
'_id': definition_id,
'category': root_category, 'category': root_category,
'fields': definition_fields, 'fields': definition_fields,
'edit_info': { 'edit_info': {
'edited_by': user_id, 'edited_by': user_id,
'edited_on': datetime.datetime.now(UTC), 'edited_on': datetime.datetime.now(UTC),
'previous_version': None, 'previous_version': None,
'original_version': definition_id,
} }
} }
definition_id = self.definitions.insert(definition_entry) self.definitions.insert(definition_entry)
definition_entry['edit_info']['original_version'] = definition_id
self.definitions.update({'_id': definition_id}, {'$set': {"edit_info.original_version": definition_id}})
new_id = ObjectId()
draft_structure = { draft_structure = {
'_id': new_id,
'root': root_usage_id, 'root': root_usage_id,
'previous_version': None, 'previous_version': None,
'original_version': new_id,
'edited_by': user_id, 'edited_by': user_id,
'edited_on': datetime.datetime.now(UTC), 'edited_on': datetime.datetime.now(UTC),
'blocks': { 'blocks': {
...@@ -841,18 +845,14 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -841,18 +845,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
'edit_info': { 'edit_info': {
'edited_on': datetime.datetime.now(UTC), 'edited_on': datetime.datetime.now(UTC),
'edited_by': user_id, 'edited_by': user_id,
'previous_version': None 'previous_version': None,
'update_version': new_id,
} }
} }
} }
} }
new_id = self.structures.insert(draft_structure) self.structures.insert(draft_structure)
draft_structure['original_version'] = new_id
self.structures.update(
{'_id': new_id},
{'$set': {"original_version": new_id,
'blocks.{}.edit_info.update_version'.format(root_usage_id): new_id}}
)
if versions_dict is None: if versions_dict is None:
versions_dict = {master_branch: new_id} versions_dict = {master_branch: new_id}
else: else:
...@@ -864,6 +864,7 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -864,6 +864,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
draft_structure = self._lookup_course(draft_version)['structure'] draft_structure = self._lookup_course(draft_version)['structure']
if definition_fields or block_fields: if definition_fields or block_fields:
draft_structure = self._version_structure(draft_structure, user_id) draft_structure = self._version_structure(draft_structure, user_id)
new_id = draft_structure['_id']
root_block = draft_structure['blocks'][draft_structure['root']] root_block = draft_structure['blocks'][draft_structure['root']]
if block_fields is not None: if block_fields is not None:
root_block['fields'].update(block_fields) root_block['fields'].update(block_fields)
...@@ -873,16 +874,17 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -873,16 +874,17 @@ class SplitMongoModuleStore(ModuleStoreBase):
definition['edit_info']['previous_version'] = definition['_id'] definition['edit_info']['previous_version'] = definition['_id']
definition['edit_info']['edited_by'] = user_id definition['edit_info']['edited_by'] = user_id
definition['edit_info']['edited_on'] = datetime.datetime.now(UTC) definition['edit_info']['edited_on'] = datetime.datetime.now(UTC)
del definition['_id'] definition['_id'] = ObjectId()
root_block['definition'] = self.definitions.insert(definition) self.definitions.insert(definition)
root_block['definition'] = definition['_id']
root_block['edit_info']['edited_on'] = datetime.datetime.now(UTC) root_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
root_block['edit_info']['edited_by'] = user_id root_block['edit_info']['edited_by'] = user_id
root_block['edit_info']['previous_version'] = root_block['edit_info'].get('update_version') root_block['edit_info']['previous_version'] = root_block['edit_info'].get('update_version')
# insert updates the '_id' in draft_structure root_block['edit_info']['update_version'] = new_id
new_id = self.structures.insert(draft_structure)
self.structures.insert(draft_structure)
versions_dict[master_branch] = new_id versions_dict[master_branch] = new_id
self.structures.update({'_id': new_id},
{'$set': {'blocks.{}.edit_info.update_version'.format(draft_structure['root']): new_id}})
# create the index entry # create the index entry
if id_root is None: if id_root is None:
id_root = org id_root = org
...@@ -895,7 +897,7 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -895,7 +897,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
'edited_by': user_id, 'edited_by': user_id,
'edited_on': datetime.datetime.now(UTC), 'edited_on': datetime.datetime.now(UTC),
'versions': versions_dict} 'versions': versions_dict}
new_id = self.course_index.insert(index_entry) self.course_index.insert(index_entry)
return self.get_course(CourseLocator(course_id=new_id, branch=master_branch)) return self.get_course(CourseLocator(course_id=new_id, branch=master_branch))
def update_item(self, descriptor, user_id, force=False): def update_item(self, descriptor, user_id, force=False):
...@@ -940,16 +942,14 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -940,16 +942,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
if descriptor.has_children: if descriptor.has_children:
block_data['fields']["children"] = [self._usage_id(child) for child in descriptor.children] block_data['fields']["children"] = [self._usage_id(child) for child in descriptor.children]
new_id = new_structure['_id']
block_data['edit_info'] = { block_data['edit_info'] = {
'edited_on': datetime.datetime.now(UTC), 'edited_on': datetime.datetime.now(UTC),
'edited_by': user_id, 'edited_by': user_id,
'previous_version': block_data['edit_info']['update_version'], 'previous_version': block_data['edit_info']['update_version'],
'update_version': new_id,
} }
new_id = self.structures.insert(new_structure) self.structures.insert(new_structure)
self.structures.update(
{'_id': new_id},
{'$set': {'blocks.{}.edit_info.update_version'.format(descriptor.location.usage_id): new_id}})
# update the index entry if appropriate # update the index entry if appropriate
if index_entry is not None: if index_entry is not None:
self._update_head(index_entry, descriptor.location.branch, new_id) self._update_head(index_entry, descriptor.location.branch, new_id)
...@@ -983,15 +983,11 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -983,15 +983,11 @@ class SplitMongoModuleStore(ModuleStoreBase):
index_entry = self._get_index_if_valid(xblock.location, force) index_entry = self._get_index_if_valid(xblock.location, force)
structure = self._lookup_course(xblock.location)['structure'] structure = self._lookup_course(xblock.location)['structure']
new_structure = self._version_structure(structure, user_id) new_structure = self._version_structure(structure, user_id)
new_id = new_structure['_id']
is_updated = self._persist_subdag(xblock, user_id, new_structure['blocks'], new_id)
changed_blocks = self._persist_subdag(xblock, user_id, new_structure['blocks']) if is_updated:
self.structures.insert(new_structure)
if changed_blocks:
new_id = self.structures.insert(new_structure)
update_command = {}
for usage_id in changed_blocks:
update_command['blocks.{}.edit_info.update_version'.format(usage_id)] = new_id
self.structures.update({'_id': new_id}, {'$set': update_command})
# update the index entry if appropriate # update the index entry if appropriate
if index_entry is not None: if index_entry is not None:
...@@ -1002,7 +998,7 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1002,7 +998,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
else: else:
return xblock return xblock
def _persist_subdag(self, xblock, user_id, structure_blocks): def _persist_subdag(self, xblock, user_id, structure_blocks, new_id):
# persist the definition if persisted != passed # persist the definition if persisted != passed
new_def_data = self._filter_special_fields(xblock.get_explicitly_set_fields_by_scope(Scope.content)) new_def_data = self._filter_special_fields(xblock.get_explicitly_set_fields_by_scope(Scope.content))
if (xblock.definition_locator is None or xblock.definition_locator.definition_id is None): if (xblock.definition_locator is None or xblock.definition_locator.definition_id is None):
...@@ -1027,17 +1023,15 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1027,17 +1023,15 @@ class SplitMongoModuleStore(ModuleStoreBase):
is_updated = True is_updated = True
children = [] children = []
updated_blocks = []
if xblock.has_children: if xblock.has_children:
for child in xblock.children: for child in xblock.children:
if isinstance(child.usage_id, LocalId): if isinstance(child.usage_id, LocalId):
child_block = xblock.system.get_block(child) child_block = xblock.system.get_block(child)
updated_blocks += self._persist_subdag(child_block, user_id, structure_blocks) is_updated = self._persist_subdag(child_block, user_id, structure_blocks, new_id) or is_updated
children.append(child_block.location.usage_id) children.append(child_block.location.usage_id)
else: else:
children.append(child) children.append(child)
is_updated = is_updated or updated_blocks
block_fields = xblock.get_explicitly_set_fields_by_scope(Scope.settings) block_fields = xblock.get_explicitly_set_fields_by_scope(Scope.settings)
if not is_new and not is_updated: if not is_new and not is_updated:
is_updated = self._compare_settings(block_fields, structure_blocks[usage_id]['fields']) is_updated = self._compare_settings(block_fields, structure_blocks[usage_id]['fields'])
...@@ -1052,13 +1046,13 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1052,13 +1046,13 @@ class SplitMongoModuleStore(ModuleStoreBase):
"fields": block_fields, "fields": block_fields,
'edit_info': { 'edit_info': {
'previous_version': previous_version, 'previous_version': previous_version,
'update_version': new_id,
'edited_by': user_id, 'edited_by': user_id,
'edited_on': datetime.datetime.now(UTC) 'edited_on': datetime.datetime.now(UTC)
} }
} }
updated_blocks.append(usage_id)
return updated_blocks return is_updated
def _compare_settings(self, settings, original_fields): def _compare_settings(self, settings, original_fields):
""" """
...@@ -1133,15 +1127,16 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1133,15 +1127,16 @@ class SplitMongoModuleStore(ModuleStoreBase):
index_entry = self._get_index_if_valid(usage_locator, force) index_entry = self._get_index_if_valid(usage_locator, force)
new_structure = self._version_structure(original_structure, user_id) new_structure = self._version_structure(original_structure, user_id)
new_blocks = new_structure['blocks'] new_blocks = new_structure['blocks']
new_id = new_structure['_id']
parents = self.get_parent_locations(usage_locator) parents = self.get_parent_locations(usage_locator)
update_version_keys = []
for parent in parents: for parent in parents:
parent_block = new_blocks[parent.usage_id] parent_block = new_blocks[parent.usage_id]
parent_block['fields']['children'].remove(usage_locator.usage_id) parent_block['fields']['children'].remove(usage_locator.usage_id)
parent_block['edit_info']['edited_on'] = datetime.datetime.now(UTC) parent_block['edit_info']['edited_on'] = datetime.datetime.now(UTC)
parent_block['edit_info']['edited_by'] = user_id parent_block['edit_info']['edited_by'] = user_id
parent_block['edit_info']['previous_version'] = parent_block['edit_info']['update_version'] parent_block['edit_info']['previous_version'] = parent_block['edit_info']['update_version']
update_version_keys.append('blocks.{}.edit_info.update_version'.format(parent.usage_id)) parent_block['edit_info']['update_version'] = new_id
# remove subtree # remove subtree
def remove_subtree(usage_id): def remove_subtree(usage_id):
for child in new_blocks[usage_id]['fields'].get('children', []): for child in new_blocks[usage_id]['fields'].get('children', []):
...@@ -1151,10 +1146,7 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1151,10 +1146,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
remove_subtree(usage_locator.usage_id) remove_subtree(usage_locator.usage_id)
# update index if appropriate and structures # update index if appropriate and structures
new_id = self.structures.insert(new_structure) self.structures.insert(new_structure)
if update_version_keys:
update_version_payload = {key: new_id for key in update_version_keys}
self.structures.update({'_id': new_id}, {'$set': update_version_payload})
result = CourseLocator(version_guid=new_id) result = CourseLocator(version_guid=new_id)
...@@ -1272,7 +1264,6 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1272,7 +1264,6 @@ class SplitMongoModuleStore(ModuleStoreBase):
# clear cache again b/c inheritance may be wrong over orphans # clear cache again b/c inheritance may be wrong over orphans
self._clear_cache(original_structure['_id']) self._clear_cache(original_structure['_id'])
def _block_matches(self, value, qualifiers): def _block_matches(self, value, qualifiers):
''' '''
Return True or False depending on whether the value (block contents) Return True or False depending on whether the value (block contents)
...@@ -1374,7 +1365,7 @@ class SplitMongoModuleStore(ModuleStoreBase): ...@@ -1374,7 +1365,7 @@ class SplitMongoModuleStore(ModuleStoreBase):
:param user_id: :param user_id:
""" """
new_structure = copy.deepcopy(structure) new_structure = copy.deepcopy(structure)
del new_structure['_id'] new_structure['_id'] = ObjectId()
new_structure['previous_version'] = structure['_id'] new_structure['previous_version'] = structure['_id']
new_structure['edited_by'] = user_id new_structure['edited_by'] = user_id
new_structure['edited_on'] = datetime.datetime.now(UTC) new_structure['edited_on'] = datetime.datetime.now(UTC)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment