Skip to content

E
edx-platform
Commit d0c9daa4 by John Eskew
Options

Merge pull request #6920 from edx/jeskew/eager_loading_xblocks_to_depth 

Improve performance of import/export and course traversal with Split modulestore.
parents 3a28c570 2a4a89a9
Showing with 683 additions and 554 deletions
common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
...@@ -105,49 +105,43 @@ class CrossStoreXMLRoundtrip(unittest.TestCase): ...@@ -105,49 +105,43 @@ class CrossStoreXMLRoundtrip(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH) make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import with source_ms.build() as (source_content, source_store):
with MongoContentstoreBuilder().build() as source_content: with dest_ms.build() as (dest_content, dest_store):
# Construct the modulestore for storing the first import (using the previously created contentstore) source_course_key = source_store.make_course_key('a', 'course', 'course')
with source_ms.build(source_content) as source_store: dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content: with CodeBlockTimer("initial_import"):
# Construct the modulestore for storing the second import (using the second contentstore) import_from_xml(
with dest_ms.build(dest_content) as dest_store: source_store,
source_course_key = source_store.make_course_key('a', 'course', 'course') 'test_user',
dest_course_key = dest_store.make_course_key('a', 'course', 'course') TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
with CodeBlockTimer("initial_import"): static_content_store=source_content,
import_from_xml( target_course_id=source_course_key,
source_store, create_course_if_not_present=True,
'test_user', raise_on_failure=True,
TEST_DATA_ROOT, )
course_dirs=TEST_COURSE,
static_content_store=source_content, with CodeBlockTimer("export"):
target_course_id=source_course_key, export_to_xml(
create_course_if_not_present=True, source_store,
raise_on_failure=True, source_content,
) source_course_key,
self.export_dir,
with CodeBlockTimer("export"): 'exported_source_course',
export_to_xml( )
source_store,
source_content, with CodeBlockTimer("second_import"):
source_course_key, import_from_xml(
self.export_dir, dest_store,
'exported_source_course', 'test_user',
) self.export_dir,
course_dirs=['exported_source_course'],
with CodeBlockTimer("second_import"): static_content_store=dest_content,
import_from_xml( target_course_id=dest_course_key,
dest_store, create_course_if_not_present=True,
'test_user', raise_on_failure=True,
self.export_dir, )
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
@ddt.ddt @ddt.ddt
...@@ -193,47 +187,44 @@ class FindAssetTest(unittest.TestCase): ...@@ -193,47 +187,44 @@ class FindAssetTest(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH) make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import with source_ms.build() as (source_content, source_store):
with MongoContentstoreBuilder().build() as source_content: source_course_key = source_store.make_course_key('a', 'course', 'course')
# Construct the modulestore for storing the first import (using the previously created contentstore) asset_key = source_course_key.make_asset_key(
with source_ms.build(source_content) as source_store: AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif'
source_course_key = source_store.make_course_key('a', 'course', 'course') )
asset_key = source_course_key.make_asset_key(
AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif' with CodeBlockTimer("initial_import"):
import_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
) )
with CodeBlockTimer("initial_import"): with CodeBlockTimer("find_nonexistent_asset"):
import_from_xml( # More correct would be using the AssetManager.find() - but since the test
source_store, # has created its own test modulestore, the AssetManager can't be used.
'test_user', __ = source_store.find_asset_metadata(asset_key)
TEST_DATA_ROOT,
course_dirs=TEST_COURSE, # Perform get_all_asset_metadata for each sort.
static_content_store=source_content, for sort in ALL_SORTS:
target_course_id=source_course_key, with CodeBlockTimer("get_asset_list:{}-{}".format(
create_course_if_not_present=True, sort[0],
raise_on_failure=True, 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
)):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=0, sort=sort, maxresults=50
)
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
) )
with CodeBlockTimer("find_nonexistent_asset"):
# More correct would be using the AssetManager.find() - but since the test
# has created its own test modulestore, the AssetManager can't be used.
__ = source_store.find_asset_metadata(asset_key)
# Perform get_all_asset_metadata for each sort.
for sort in ALL_SORTS:
with CodeBlockTimer("get_asset_list:{}-{}".format(
sort[0],
'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
)):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=0, sort=sort, maxresults=50
)
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
)
@ddt.ddt @ddt.ddt
...@@ -265,48 +256,45 @@ class TestModulestoreAssetSize(unittest.TestCase): ...@@ -265,48 +256,45 @@ class TestModulestoreAssetSize(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH) make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import with source_ms.build() as (source_content, source_store):
with MongoContentstoreBuilder().build() as source_content: source_course_key = source_store.make_course_key('a', 'course', 'course')
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store: import_from_xml(
source_course_key = source_store.make_course_key('a', 'course', 'course') source_store,
'test_user',
import_from_xml( TEST_DATA_ROOT,
source_store, course_dirs=TEST_COURSE,
'test_user', static_content_store=source_content,
TEST_DATA_ROOT, target_course_id=source_course_key,
course_dirs=TEST_COURSE, create_course_if_not_present=True,
static_content_store=source_content, raise_on_failure=True,
target_course_id=source_course_key, )
create_course_if_not_present=True,
raise_on_failure=True, asset_collection = source_ms.asset_collection()
) # Ensure the asset collection exists.
if asset_collection.name in asset_collection.database.collection_names():
# Map gets the size of each structure.
mapper = Code("""
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
""")
asset_collection = source_ms.asset_collection() # Reduce finds the largest structure size and returns only it.
# Ensure the asset collection exists. reducer = Code("""
if asset_collection.name in asset_collection.database.collection_names(): function(key, values) {
var max_size = 0;
# Map gets the size of each structure. for (var i=0; i < values.length; i++) {
mapper = Code(""" if (values[i] > max_size) {
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } max_size = values[i];
""")
# Reduce finds the largest structure size and returns only it.
reducer = Code("""
function(key, values) {
var max_size = 0;
for (var i=0; i < values.length; i++) {
if (values[i] > max_size) {
max_size = values[i];
}
} }
return max_size;
} }
""") return max_size;
}
""")
results = asset_collection.map_reduce(mapper, reducer, "size_results") results = asset_collection.map_reduce(mapper, reducer, "size_results")
result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
) )
with open("bson_sizes.txt", "a") as f: with open("bson_sizes.txt", "a") as f:
f.write(result_str) f.write(result_str)
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
...@@ -402,13 +402,18 @@ class SplitBulkWriteMixin(BulkOperationsMixin): ...@@ -402,13 +402,18 @@ class SplitBulkWriteMixin(BulkOperationsMixin):
bulk_write_record = self._get_bulk_ops_record(course_key) bulk_write_record = self._get_bulk_ops_record(course_key)
if bulk_write_record.active: if bulk_write_record.active:
# Only query for the definitions that aren't already cached.
for definition in bulk_write_record.definitions.values(): for definition in bulk_write_record.definitions.values():
definition_id = definition.get('_id') definition_id = definition.get('_id')
if definition_id in ids: if definition_id in ids:
ids.remove(definition_id) ids.remove(definition_id)
definitions.append(definition) definitions.append(definition)
definitions.extend(self.db_connection.get_definitions(list(ids))) # Query the db for the definitions.
defs_from_db = self.db_connection.get_definitions(list(ids))
# Add the retrieved definitions to the cache.
bulk_write_record.definitions.update({d.get('_id'): d for d in defs_from_db})
definitions.extend(defs_from_db)
return definitions return definitions
def update_definition(self, course_key, definition): def update_definition(self, course_key, definition):
...@@ -683,23 +688,29 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): ...@@ -683,23 +688,29 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
new_module_data new_module_data
) )
if not lazy: # This code supports lazy loading, where the descendent definitions aren't loaded
# Load all descendants by id # until they're actually needed.
# However, assume that depth == 0 means no depth is specified and depth != 0 means
# a depth *is* specified. If a non-zero depth is specified, force non-lazy definition
# loading in order to populate the definition cache for later access.
load_definitions_now = depth != 0 or not lazy
if load_definitions_now:
# Non-lazy loading: Load all descendants by id.
descendent_definitions = self.get_definitions( descendent_definitions = self.get_definitions(
course_key, course_key,
[ [
block['definition'] block.definition
for block in new_module_data.itervalues() for block in new_module_data.itervalues()
] ]
) )
# turn into a map # Turn definitions into a map.
definitions = {definition['_id']: definition definitions = {definition['_id']: definition
for definition in descendent_definitions} for definition in descendent_definitions}
for block in new_module_data.itervalues(): for block in new_module_data.itervalues():
if block.definition in definitions: if block.definition in definitions:
definition = definitions[block.definition] definition = definitions[block.definition]
# convert_fields was being done here, but it gets done later in the runtime's xblock_from_json # convert_fields gets done later in the runtime's xblock_from_json
block.fields.update(definition.get('fields')) block.fields.update(definition.get('fields'))
block.definition_loaded = True block.definition_loaded = True
......
common/lib/xmodule/xmodule/modulestore/tests/test_asides.py
...@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase): ...@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase):
""" """
Check that the xml modulestore read in all the asides with their values Check that the xml modulestore read in all the asides with their values
""" """
with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as store: with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as (__, store):
def check_block(block): def check_block(block):
""" """
Check whether block has the expected aside w/ its fields and then recurse to the block's children Check whether block has the expected aside w/ its fields and then recurse to the block's children
......
common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py
...@@ -161,155 +161,146 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -161,155 +161,146 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
""" """
Save the metadata in each store and retrieve it singularly, as all assets, and after deleting all. Save the metadata in each store and retrieve it singularly, as all assets, and after deleting all.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_filename = 'burnside.jpg'
asset_filename = 'burnside.jpg' new_asset_loc = course.id.make_asset_key('asset', asset_filename)
new_asset_loc = course.id.make_asset_key('asset', asset_filename) # Save the asset's metadata.
# Save the asset's metadata. new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) # Find the asset's metadata and confirm it's the same.
# Find the asset's metadata and confirm it's the same. found_asset_md = store.find_asset_metadata(new_asset_loc)
found_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(found_asset_md)
self.assertIsNotNone(found_asset_md) self.assertEquals(new_asset_md, found_asset_md)
self.assertEquals(new_asset_md, found_asset_md) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_delete(self, storebuilder): def test_delete(self, storebuilder):
""" """
Delete non-existent and existent metadata Delete non-existent and existent metadata
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') # Attempt to delete an asset that doesn't exist.
# Attempt to delete an asset that doesn't exist. self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
new_asset_md = self._make_asset_metadata(new_asset_loc) new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1) self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_find_non_existing_assets(self, storebuilder): def test_find_non_existing_assets(self, storebuilder):
""" """
Find a non-existent asset in an existing course. Find a non-existent asset in an existing course.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') # Find existing asset metadata.
# Find existing asset metadata. asset_md = store.find_asset_metadata(new_asset_loc)
asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNone(asset_md)
self.assertIsNone(asset_md)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_get_all_non_existing_assets(self, storebuilder): def test_get_all_non_existing_assets(self, storebuilder):
""" """
Get all assets in an existing course when no assets exist. Get all assets in an existing course when no assets exist.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) # Find existing asset metadata.
# Find existing asset metadata. asset_md = store.get_all_asset_metadata(course.id, 'asset')
asset_md = store.get_all_asset_metadata(course.id, 'asset') self.assertEquals(asset_md, [])
self.assertEquals(asset_md, [])
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_find_assets_in_non_existent_course(self, storebuilder): def test_find_assets_in_non_existent_course(self, storebuilder):
""" """
Find asset metadata from a non-existent course. Find asset metadata from a non-existent course.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format(
fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format( course.id.org, course.id.course, course.id.run
course.id.org, course.id.course, course.id.run ))
)) new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg') # Find asset metadata from non-existent course.
# Find asset metadata from non-existent course. with self.assertRaises(ItemNotFoundError):
with self.assertRaises(ItemNotFoundError): store.find_asset_metadata(new_asset_loc)
store.find_asset_metadata(new_asset_loc) with self.assertRaises(ItemNotFoundError):
with self.assertRaises(ItemNotFoundError): store.get_all_asset_metadata(fake_course_id, 'asset')
store.get_all_asset_metadata(fake_course_id, 'asset')
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_add_same_asset_twice(self, storebuilder): def test_add_same_asset_twice(self, storebuilder):
""" """
Add an asset's metadata, then add it again. Add an asset's metadata, then add it again.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) # Add asset metadata.
# Add asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) # Add *the same* asset metadata.
# Add *the same* asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) # Still one here?
# Still one here? self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_different_asset_types(self, storebuilder): def test_different_asset_types(self, storebuilder):
""" """
Test saving assets with other asset types. Test saving assets with other asset types.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('vrml', 'pyramid.vrml')
new_asset_loc = course.id.make_asset_key('vrml', 'pyramid.vrml') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) # Add asset metadata.
# Add asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_asset_types_with_other_field_names(self, storebuilder): def test_asset_types_with_other_field_names(self, storebuilder):
""" """
Test saving assets using an asset type of 'course_id'. Test saving assets using an asset type of 'course_id'.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('course_id', 'just_to_see_if_it_still_works.jpg')
new_asset_loc = course.id.make_asset_key('course_id', 'just_to_see_if_it_still_works.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) # Add asset metadata.
# Add asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) all_assets = store.get_all_asset_metadata(course.id, 'course_id')
all_assets = store.get_all_asset_metadata(course.id, 'course_id') self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path)
self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_lock_unlock_assets(self, storebuilder): def test_lock_unlock_assets(self, storebuilder):
""" """
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
locked_state = new_asset_md.locked
locked_state = new_asset_md.locked # Flip the course asset's locked status.
# Flip the course asset's locked status. store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test) # Find the same course and check its locked status.
# Find the same course and check its locked status. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) self.assertEquals(updated_asset_md.locked, not locked_state)
self.assertEquals(updated_asset_md.locked, not locked_state) # Now flip it back.
# Now flip it back. store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test) reupdated_asset_md = store.find_asset_metadata(new_asset_loc)
reupdated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(reupdated_asset_md)
self.assertIsNotNone(reupdated_asset_md) self.assertEquals(reupdated_asset_md.locked, locked_state)
self.assertEquals(reupdated_asset_md.locked, locked_state)
ALLOWED_ATTRS = ( ALLOWED_ATTRS = (
('pathname', '/new/path'), ('pathname', '/new/path'),
...@@ -340,98 +331,93 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -340,98 +331,93 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
""" """
Save setting each attr one at a time Save setting each attr one at a time
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) for attribute, value in self.ALLOWED_ATTRS:
for attr, value in self.ALLOWED_ATTRS: # Set the course asset's attribute.
# Set the course asset's attr. store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) # Find the same course asset and check its changed attribute.
# Find the same course asset and check its changed attr. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) self.assertIsNotNone(getattr(updated_asset_md, attribute, None))
self.assertIsNotNone(getattr(updated_asset_md, attr, None)) self.assertEquals(getattr(updated_asset_md, attribute, None), value)
self.assertEquals(getattr(updated_asset_md, attr, None), value)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_set_disallowed_attrs(self, storebuilder): def test_set_disallowed_attrs(self, storebuilder):
""" """
setting disallowed attrs should fail setting disallowed attrs should fail
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) for attribute, value in self.DISALLOWED_ATTRS:
for attr, value in self.DISALLOWED_ATTRS: original_attr_val = getattr(new_asset_md, attribute)
original_attr_val = getattr(new_asset_md, attr) # Set the course asset's attribute.
# Set the course asset's attr. store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) # Find the same course and check its changed attribute.
# Find the same course and check its changed attr. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) self.assertIsNotNone(getattr(updated_asset_md, attribute, None))
self.assertIsNotNone(getattr(updated_asset_md, attr, None)) # Make sure that the attribute is unchanged from its original value.
# Make sure that the attr is unchanged from its original value. self.assertEquals(getattr(updated_asset_md, attribute, None), original_attr_val)
self.assertEquals(getattr(updated_asset_md, attr, None), original_attr_val)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_set_unknown_attrs(self, storebuilder): def test_set_unknown_attrs(self, storebuilder):
""" """
setting unknown attrs should fail setting unknown attrs should fail
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) for attribute, value in self.UNKNOWN_ATTRS:
for attr, value in self.UNKNOWN_ATTRS: # Set the course asset's attribute.
# Set the course asset's attr. store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) # Find the same course and check its changed attribute.
# Find the same course and check its changed attr. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) # Make sure the unknown field was *not* added.
# Make sure the unknown field was *not* added. with self.assertRaises(AttributeError):
with self.assertRaises(AttributeError): self.assertEquals(getattr(updated_asset_md, attribute), value)
self.assertEquals(getattr(updated_asset_md, attr), value)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_save_one_different_asset(self, storebuilder): def test_save_one_different_asset(self, storebuilder):
""" """
saving and deleting things which are not 'asset' saving and deleting things which are not 'asset'
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_key = course.id.make_asset_key('different', 'burn.jpg')
asset_key = course.id.make_asset_key('different', 'burn.jpg') new_asset_thumbnail = self._make_asset_thumbnail_metadata(
new_asset_thumbnail = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) )
) store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_find_different(self, storebuilder): def test_find_different(self, storebuilder):
""" """
finding things which are of type other than 'asset' finding things which are of type other than 'asset'
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_key = course.id.make_asset_key('different', 'burn.jpg')
asset_key = course.id.make_asset_key('different', 'burn.jpg') new_asset_thumbnail = self._make_asset_thumbnail_metadata(
new_asset_thumbnail = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) )
) store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertIsNotNone(store.find_asset_metadata(asset_key)) self.assertIsNotNone(store.find_asset_metadata(asset_key))
unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg') unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg')
self.assertIsNone(store.find_asset_metadata(unknown_asset_key)) self.assertIsNone(store.find_asset_metadata(unknown_asset_key))
def _check_asset_values(self, assets, orig): def _check_asset_values(self, assets, orig):
""" """
...@@ -447,37 +433,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -447,37 +433,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
getting all things which are of type other than 'asset' getting all things which are of type other than 'asset'
""" """
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store)
# Save 'em.
# Save 'em. for asset_type, filename in self.alls:
for asset_type, filename in self.alls: asset_key = course.id.make_asset_key(asset_type, filename)
asset_key = course.id.make_asset_key(asset_type, filename) new_asset = self._make_asset_thumbnail_metadata(
new_asset = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key)
)
store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test)
# Check 'em.
for asset_type, asset_list in (
('different', self.differents),
('vrml', self.vrmls),
('asset', self.regular_assets),
):
assets = store.get_all_asset_metadata(course.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
course.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
) )
self.assertEquals(len(assets), len(self.alls)) store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test)
self._check_asset_values(assets, self.alls)
# Check 'em.
for asset_type, asset_list in (
('different', self.differents),
('vrml', self.vrmls),
('asset', self.regular_assets),
):
assets = store.get_all_asset_metadata(course.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
course.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(assets), len(self.alls))
self._check_asset_values(assets, self.alls)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_save_metadata_list(self, storebuilder): def test_save_metadata_list(self, storebuilder):
...@@ -485,40 +470,39 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -485,40 +470,39 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
Save a list of asset metadata all at once. Save a list of asset metadata all at once.
""" """
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store)
# Make a list of AssetMetadata objects.
# Make a list of AssetMetadata objects. md_list = []
md_list = [] for asset_type, filename in self.alls:
for asset_type, filename in self.alls: asset_key = course.id.make_asset_key(asset_type, filename)
asset_key = course.id.make_asset_key(asset_type, filename) md_list.append(self._make_asset_thumbnail_metadata(
md_list.append(self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) ))
))
# Save 'em.
# Save 'em. store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
# Check 'em.
# Check 'em. for asset_type, asset_list in (
for asset_type, asset_list in ( ('different', self.differents),
('different', self.differents), ('vrml', self.vrmls),
('vrml', self.vrmls), ('asset', self.regular_assets),
('asset', self.regular_assets), ):
): assets = store.get_all_asset_metadata(course.id, asset_type)
assets = store.get_all_asset_metadata(course.id, asset_type) self.assertEquals(len(assets), len(asset_list))
self.assertEquals(len(assets), len(asset_list)) self._check_asset_values(assets, asset_list)
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0) self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
assets = store.get_all_asset_metadata( course.id, None, start=0, maxresults=-1,
course.id, None, start=0, maxresults=-1, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(assets), len(self.alls))
self.assertEquals(len(assets), len(self.alls)) self._check_asset_values(assets, self.alls)
self._check_asset_values(assets, self.alls)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_save_metadata_list_with_mismatched_asset(self, storebuilder): def test_save_metadata_list_with_mismatched_asset(self, storebuilder):
...@@ -526,140 +510,137 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -526,140 +510,137 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
Save a list of asset metadata all at once - but with one asset's metadata from a different course. Save a list of asset metadata all at once - but with one asset's metadata from a different course.
""" """
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store)
# Make a list of AssetMetadata objects.
# Make a list of AssetMetadata objects. md_list = []
md_list = [] for asset_type, filename in self.alls:
for asset_type, filename in self.alls: if asset_type == 'asset':
if asset_type == 'asset': asset_key = course2.id.make_asset_key(asset_type, filename)
asset_key = course2.id.make_asset_key(asset_type, filename) else:
else: asset_key = course1.id.make_asset_key(asset_type, filename)
asset_key = course1.id.make_asset_key(asset_type, filename) md_list.append(self._make_asset_thumbnail_metadata(
md_list.append(self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) ))
))
# Save 'em.
# Save 'em. store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
# Check 'em.
# Check 'em. for asset_type, asset_list in (
for asset_type, asset_list in ( ('different', self.differents),
('different', self.differents), ('vrml', self.vrmls),
('vrml', self.vrmls), ):
): assets = store.get_all_asset_metadata(course1.id, asset_type)
assets = store.get_all_asset_metadata(course1.id, asset_type) self.assertEquals(len(assets), len(asset_list))
self.assertEquals(len(assets), len(asset_list)) self._check_asset_values(assets, asset_list)
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3)
assets = store.get_all_asset_metadata(
assets = store.get_all_asset_metadata( course1.id, None, start=0, maxresults=-1,
course1.id, None, start=0, maxresults=-1, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(assets), len(self.differents + self.vrmls))
self.assertEquals(len(assets), len(self.differents + self.vrmls)) self._check_asset_values(assets, self.differents + self.vrmls)
self._check_asset_values(assets, self.differents + self.vrmls)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_delete_all_different_type(self, storebuilder): def test_delete_all_different_type(self, storebuilder):
""" """
deleting all assets of a given but not 'asset' type deleting all assets of a given but not 'asset' type
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg')
asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg') new_asset_thumbnail = self._make_asset_thumbnail_metadata(
new_asset_thumbnail = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) )
) store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_get_all_assets_with_paging(self, storebuilder): def test_get_all_assets_with_paging(self, storebuilder):
""" """
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store) self.setup_assets(course1.id, course2.id, store)
self.setup_assets(course1.id, course2.id, store)
expected_sorts_by_2 = (
expected_sorts_by_2 = ( (
( ('displayname', ModuleStoreEnum.SortOrder.ascending),
('displayname', ModuleStoreEnum.SortOrder.ascending), ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'),
('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'), (2, 2, 1)
(2, 2, 1) ),
), (
( ('displayname', ModuleStoreEnum.SortOrder.descending),
('displayname', ModuleStoreEnum.SortOrder.descending), ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'),
('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'), (2, 2, 1)
(2, 2, 1) ),
), (
( ('uploadDate', ModuleStoreEnum.SortOrder.ascending),
('uploadDate', ModuleStoreEnum.SortOrder.ascending), ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'),
('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'), (2, 2, 1)
(2, 2, 1) ),
), (
( ('uploadDate', ModuleStoreEnum.SortOrder.descending),
('uploadDate', ModuleStoreEnum.SortOrder.descending), ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'),
('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'), (2, 2, 1)
(2, 2, 1) ),
), )
) # First, with paging across all sorts.
# First, with paging across all sorts. for sort_test in expected_sorts_by_2:
for sort_test in expected_sorts_by_2: for i in xrange(3):
for i in xrange(3): asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0]
course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0] )
) num_expected_results = sort_test[2][i]
num_expected_results = sort_test[2][i] expected_filename = sort_test[1][2 * i]
expected_filename = sort_test[1][2 * i] self.assertEquals(len(asset_page), num_expected_results)
self.assertEquals(len(asset_page), num_expected_results) self.assertEquals(asset_page[0].asset_id.path, expected_filename)
self.assertEquals(asset_page[0].asset_id.path, expected_filename) if num_expected_results == 2:
if num_expected_results == 2: expected_filename = sort_test[1][(2 * i) + 1]
expected_filename = sort_test[1][(2 * i) + 1] self.assertEquals(asset_page[1].asset_id.path, expected_filename)
self.assertEquals(asset_page[1].asset_id.path, expected_filename)
# Now fetch everything.
# Now fetch everything. asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(asset_page), 5)
self.assertEquals(len(asset_page), 5) self.assertEquals(asset_page[0].asset_id.path, 'code.tgz')
self.assertEquals(asset_page[0].asset_id.path, 'code.tgz') self.assertEquals(asset_page[1].asset_id.path, 'demo.swf')
self.assertEquals(asset_page[1].asset_id.path, 'demo.swf') self.assertEquals(asset_page[2].asset_id.path, 'dog.png')
self.assertEquals(asset_page[2].asset_id.path, 'dog.png') self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf')
self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf') self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
# Some odd conditions.
# Some odd conditions. asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(asset_page), 0)
self.assertEquals(len(asset_page), 0) asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=3, maxresults=0,
course2.id, 'asset', start=3, maxresults=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(asset_page), 0)
self.assertEquals(len(asset_page), 0) asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=3, maxresults=-12345,
course2.id, 'asset', start=3, maxresults=-12345, sort=('displayname', ModuleStoreEnum.SortOrder.descending)
sort=('displayname', ModuleStoreEnum.SortOrder.descending) )
) self.assertEquals(len(asset_page), 2)
self.assertEquals(len(asset_page), 2)
@ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())])) @ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())]))
def test_xml_not_yet_implemented(self, storebuilder): def test_xml_not_yet_implemented(self, storebuilder):
""" """
Test coverage which shows that for now xml read operations are not implemented Test coverage which shows that for now xml read operations are not implemented
""" """
with storebuilder.build(None) as store: with storebuilder.build(contentstore=None) as (__, store):
course_key = store.make_course_key("org", "course", "run") course_key = store.make_course_key("org", "course", "run")
asset_key = course_key.make_asset_key('asset', 'foo.jpg') asset_key = course_key.make_asset_key('asset', 'foo.jpg')
self.assertEquals(store.find_asset_metadata(asset_key), None) self.assertEquals(store.find_asset_metadata(asset_key), None)
...@@ -670,38 +651,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -670,38 +651,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
""" """
Create a course with assets, copy them all to another course in the same modulestore, and check on it. Create a course with assets, copy them all to another course in the same modulestore, and check on it.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store) self.setup_assets(course1.id, None, store)
self.setup_assets(course1.id, None, store) self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) all_assets = store.get_all_asset_metadata(
all_assets = store.get_all_asset_metadata( course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(all_assets), 2)
self.assertEquals(len(all_assets), 2) self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_copy_all_assets_from_course_with_no_assets(self, storebuilder): def test_copy_all_assets_from_course_with_no_assets(self, storebuilder):
""" """
Create a course with *no* assets, and try copy them all to another course in the same modulestore. Create a course with *no* assets, and try copy them all to another course in the same modulestore.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store) store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) all_assets = store.get_all_asset_metadata(
all_assets = store.get_all_asset_metadata( course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(all_assets), 0)
self.assertEquals(len(all_assets), 0)
@ddt.data( @ddt.data(
('mongo', 'split'), ('mongo', 'split'),
...@@ -713,19 +692,18 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -713,19 +692,18 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
Create a course with assets, copy them all to another course in a different modulestore, and check on it. Create a course with assets, copy them all to another course in a different modulestore, and check on it.
""" """
mixed_builder = MIXED_MODULESTORE_BOTH_SETUP mixed_builder = MIXED_MODULESTORE_BOTH_SETUP
with MongoContentstoreBuilder().build() as contentstore: with mixed_builder.build() as (__, mixed_store):
with mixed_builder.build(contentstore) as mixed_store: with mixed_store.default_store(from_store):
with mixed_store.default_store(from_store): course1 = CourseFactory.create(modulestore=mixed_store)
course1 = CourseFactory.create(modulestore=mixed_store) with mixed_store.default_store(to_store):
with mixed_store.default_store(to_store): course2 = CourseFactory.create(modulestore=mixed_store)
course2 = CourseFactory.create(modulestore=mixed_store) self.setup_assets(course1.id, None, mixed_store)
self.setup_assets(course1.id, None, mixed_store) self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2) self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0) mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102)
mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102) all_assets = mixed_store.get_all_asset_metadata(
all_assets = mixed_store.get_all_asset_metadata( course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(all_assets), 2)
self.assertEquals(len(all_assets), 2) self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
...@@ -76,12 +76,66 @@ class MemoryCache(object): ...@@ -76,12 +76,66 @@ class MemoryCache(object):
self._data[key] = value self._data[key] = value
class MongoModulestoreBuilder(object): class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database() # pylint: disable=protected-access
def __repr__(self):
return 'MongoContentstoreBuilder()'
class StoreBuilderBase(object):
"""
Base class for all modulestore builders.
"""
@contextmanager
def build(self, **kwargs):
"""
Build the modulstore, optionally building the contentstore as well.
"""
contentstore = kwargs.pop('contentstore', None)
if not contentstore:
with self.build_without_contentstore() as (contentstore, modulestore):
yield contentstore, modulestore
else:
with self.build_with_contentstore(contentstore) as modulestore:
yield modulestore
@contextmanager
def build_without_contentstore(self):
"""
Build both the contentstore and the modulestore.
"""
with MongoContentstoreBuilder().build() as contentstore:
with self.build_with_contentstore(contentstore) as modulestore:
yield contentstore, modulestore
class MongoModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a DraftModuleStore. A builder class for a DraftModuleStore.
""" """
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns an isolated mongo modulestore, and then deletes A contextmanager that returns an isolated mongo modulestore, and then deletes
all of its data at the end of the context. all of its data at the end of the context.
...@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object): ...@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object):
return 'MongoModulestoreBuilder()' return 'MongoModulestoreBuilder()'
class VersioningModulestoreBuilder(object): class VersioningModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a VersioningModuleStore. A builder class for a VersioningModuleStore.
""" """
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns an isolated versioning modulestore, and then deletes A contextmanager that returns an isolated versioning modulestore, and then deletes
all of its data at the end of the context. all of its data at the end of the context.
...@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object): ...@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object):
return 'SplitModulestoreBuilder()' return 'SplitModulestoreBuilder()'
class XmlModulestoreBuilder(object): class XmlModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a XMLModuleStore. A builder class for a XMLModuleStore.
""" """
# pylint: disable=unused-argument # pylint: disable=unused-argument
@contextmanager @contextmanager
def build(self, contentstore=None, course_ids=None): def build_with_contentstore(self, contentstore=None, course_ids=None):
""" """
A contextmanager that returns an isolated xml modulestore A contextmanager that returns an isolated xml modulestore
...@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object): ...@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object):
yield modulestore yield modulestore
class MixedModulestoreBuilder(object): class MixedModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a MixedModuleStore. A builder class for a MixedModuleStore.
""" """
...@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object): ...@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object):
self.mixed_modulestore = None self.mixed_modulestore = None
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns a mixed modulestore built on top of modulestores A contextmanager that returns a mixed modulestore built on top of modulestores
generated by other builder classes. generated by other builder classes.
...@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object): ...@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object):
""" """
names, generators = zip(*self.store_builders) names, generators = zip(*self.store_builders)
with nested(*(gen.build(contentstore) for gen in generators)) as modulestores: with nested(*(gen.build_with_contentstore(contentstore) for gen in generators)) as modulestores:
# Make the modulestore creation function just return the already-created modulestores # Make the modulestore creation function just return the already-created modulestores
store_iterator = iter(modulestores) store_iterator = iter(modulestores)
create_modulestore_instance = lambda *args, **kwargs: store_iterator.next() create_modulestore_instance = lambda *args, **kwargs: store_iterator.next()
...@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object): ...@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object):
return store.db_connection.structures return store.db_connection.structures
class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database()
def __repr__(self):
return 'MongoContentstoreBuilder()'
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([ MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
('draft', MongoModulestoreBuilder()), ('draft', MongoModulestoreBuilder()),
('split', VersioningModulestoreBuilder()) ('split', VersioningModulestoreBuilder())
...@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase): ...@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
# Construct the contentstore for storing the first import # Construct the contentstore for storing the first import
with source_content_builder.build() as source_content: with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore) # Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store: with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import # Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content: with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore) # Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store: with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course') source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course')
......
common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py 0 → 100644
"""
Tests to verify correct number of MongoDB calls during course import/export and traversal
when using the Split modulestore.
"""
from tempfile import mkdtemp
from shutil import rmtree
from unittest import TestCase
import ddt
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.tests.factories import check_mongo_calls
from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MixedModulestoreBuilder, VersioningModulestoreBuilder,
MongoModulestoreBuilder, TEST_DATA_DIR
)
MIXED_OLD_MONGO_MODULESTORE_BUILDER = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())])
MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())])
@ddt.ddt
class CountMongoCallsXMLRoundtrip(TestCase):
"""
This class exists to test XML import and export to/from Split.
"""
def setUp(self):
super(CountMongoCallsXMLRoundtrip, self).setUp()
self.export_dir = mkdtemp()
self.addCleanup(rmtree, self.export_dir, ignore_errors=True)
@ddt.data(
(MIXED_OLD_MONGO_MODULESTORE_BUILDER, 287, 780, 702, 702),
(MIXED_SPLIT_MODULESTORE_BUILDER, 37, 16, 190, 189),
)
@ddt.unpack
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
course_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
with check_mongo_calls(export_reads):
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with check_mongo_calls(import_reads, second_import_writes):
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
@ddt.ddt
class CountMongoCallsCourseTraversal(TestCase):
"""
Tests the number of Mongo calls made when traversing a course tree from the top course root
to the leaf nodes.
"""
@ddt.data(
(MIXED_OLD_MONGO_MODULESTORE_BUILDER, None, 189), # The way this traversal *should* be done.
(MIXED_OLD_MONGO_MODULESTORE_BUILDER, 0, 387), # The pathological case - do *not* query a course this way!
(MIXED_SPLIT_MODULESTORE_BUILDER, None, 7), # The way this traversal *should* be done.
(MIXED_SPLIT_MODULESTORE_BUILDER, 0, 145) # The pathological case - do *not* query a course this way!
)
@ddt.unpack
def test_number_mongo_calls(self, store, depth, num_mongo_calls):
with store.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
# First, import a course.
import_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
course_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
# Course traversal modeled after the traversal done here:
# lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# Starting at the root course block, do a breadth-first traversal using
# get_children() to retrieve each block's children.
with check_mongo_calls(num_mongo_calls):
start_block = source_store.get_course(source_course_key, depth=depth)
stack = [start_block]
while stack:
curr_block = stack.pop()
if curr_block.has_children:
for block in reversed(curr_block.get_children()):
stack.append(block)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment