Skip to content

E
edx-platform
Commit c3d2962b by John Eskew
Options

Refactor contentstore/modulestore builders to wrap both in a single 

contextmananger. Add ddt to mongo call count testing.
parent 3565435e
Showing with 630 additions and 633 deletions
common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
...@@ -105,49 +105,43 @@ class CrossStoreXMLRoundtrip(unittest.TestCase): ...@@ -105,49 +105,43 @@ class CrossStoreXMLRoundtrip(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH) make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import with source_ms.build() as (source_content, source_store):
with MongoContentstoreBuilder().build() as source_content: with dest_ms.build() as (dest_content, dest_store):
# Construct the modulestore for storing the first import (using the previously created contentstore) source_course_key = source_store.make_course_key('a', 'course', 'course')
with source_ms.build(source_content) as source_store: dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content: with CodeBlockTimer("initial_import"):
# Construct the modulestore for storing the second import (using the second contentstore) import_from_xml(
with dest_ms.build(dest_content) as dest_store: source_store,
source_course_key = source_store.make_course_key('a', 'course', 'course') 'test_user',
dest_course_key = dest_store.make_course_key('a', 'course', 'course') TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
with CodeBlockTimer("initial_import"): static_content_store=source_content,
import_from_xml( target_course_id=source_course_key,
source_store, create_course_if_not_present=True,
'test_user', raise_on_failure=True,
TEST_DATA_ROOT, )
course_dirs=TEST_COURSE,
static_content_store=source_content, with CodeBlockTimer("export"):
target_course_id=source_course_key, export_to_xml(
create_course_if_not_present=True, source_store,
raise_on_failure=True, source_content,
) source_course_key,
self.export_dir,
with CodeBlockTimer("export"): 'exported_source_course',
export_to_xml( )
source_store,
source_content, with CodeBlockTimer("second_import"):
source_course_key, import_from_xml(
self.export_dir, dest_store,
'exported_source_course', 'test_user',
) self.export_dir,
course_dirs=['exported_source_course'],
with CodeBlockTimer("second_import"): static_content_store=dest_content,
import_from_xml( target_course_id=dest_course_key,
dest_store, create_course_if_not_present=True,
'test_user', raise_on_failure=True,
self.export_dir, )
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
@ddt.ddt @ddt.ddt
...@@ -193,47 +187,44 @@ class FindAssetTest(unittest.TestCase): ...@@ -193,47 +187,44 @@ class FindAssetTest(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH) make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import with source_ms.build() as (source_content, source_store):
with MongoContentstoreBuilder().build() as source_content: source_course_key = source_store.make_course_key('a', 'course', 'course')
# Construct the modulestore for storing the first import (using the previously created contentstore) asset_key = source_course_key.make_asset_key(
with source_ms.build(source_content) as source_store: AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif'
source_course_key = source_store.make_course_key('a', 'course', 'course') )
asset_key = source_course_key.make_asset_key(
AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif' with CodeBlockTimer("initial_import"):
import_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
) )
with CodeBlockTimer("initial_import"): with CodeBlockTimer("find_nonexistent_asset"):
import_from_xml( # More correct would be using the AssetManager.find() - but since the test
source_store, # has created its own test modulestore, the AssetManager can't be used.
'test_user', __ = source_store.find_asset_metadata(asset_key)
TEST_DATA_ROOT,
course_dirs=TEST_COURSE, # Perform get_all_asset_metadata for each sort.
static_content_store=source_content, for sort in ALL_SORTS:
target_course_id=source_course_key, with CodeBlockTimer("get_asset_list:{}-{}".format(
create_course_if_not_present=True, sort[0],
raise_on_failure=True, 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
)):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=0, sort=sort, maxresults=50
)
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
) )
with CodeBlockTimer("find_nonexistent_asset"):
# More correct would be using the AssetManager.find() - but since the test
# has created its own test modulestore, the AssetManager can't be used.
__ = source_store.find_asset_metadata(asset_key)
# Perform get_all_asset_metadata for each sort.
for sort in ALL_SORTS:
with CodeBlockTimer("get_asset_list:{}-{}".format(
sort[0],
'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
)):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=0, sort=sort, maxresults=50
)
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
)
@ddt.ddt @ddt.ddt
...@@ -265,48 +256,45 @@ class TestModulestoreAssetSize(unittest.TestCase): ...@@ -265,48 +256,45 @@ class TestModulestoreAssetSize(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH) make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import with source_ms.build() as (source_content, source_store):
with MongoContentstoreBuilder().build() as source_content: source_course_key = source_store.make_course_key('a', 'course', 'course')
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store: import_from_xml(
source_course_key = source_store.make_course_key('a', 'course', 'course') source_store,
'test_user',
import_from_xml( TEST_DATA_ROOT,
source_store, course_dirs=TEST_COURSE,
'test_user', static_content_store=source_content,
TEST_DATA_ROOT, target_course_id=source_course_key,
course_dirs=TEST_COURSE, create_course_if_not_present=True,
static_content_store=source_content, raise_on_failure=True,
target_course_id=source_course_key, )
create_course_if_not_present=True,
raise_on_failure=True, asset_collection = source_ms.asset_collection()
) # Ensure the asset collection exists.
if asset_collection.name in asset_collection.database.collection_names():
# Map gets the size of each structure.
mapper = Code("""
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
""")
asset_collection = source_ms.asset_collection() # Reduce finds the largest structure size and returns only it.
# Ensure the asset collection exists. reducer = Code("""
if asset_collection.name in asset_collection.database.collection_names(): function(key, values) {
var max_size = 0;
# Map gets the size of each structure. for (var i=0; i < values.length; i++) {
mapper = Code(""" if (values[i] > max_size) {
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } max_size = values[i];
""")
# Reduce finds the largest structure size and returns only it.
reducer = Code("""
function(key, values) {
var max_size = 0;
for (var i=0; i < values.length; i++) {
if (values[i] > max_size) {
max_size = values[i];
}
} }
return max_size;
} }
""") return max_size;
}
""")
results = asset_collection.map_reduce(mapper, reducer, "size_results") results = asset_collection.map_reduce(mapper, reducer, "size_results")
result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
) )
with open("bson_sizes.txt", "a") as f: with open("bson_sizes.txt", "a") as f:
f.write(result_str) f.write(result_str)
common/lib/xmodule/xmodule/modulestore/tests/test_asides.py
...@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase): ...@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase):
""" """
Check that the xml modulestore read in all the asides with their values Check that the xml modulestore read in all the asides with their values
""" """
with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as store: with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as (__, store):
def check_block(block): def check_block(block):
""" """
Check whether block has the expected aside w/ its fields and then recurse to the block's children Check whether block has the expected aside w/ its fields and then recurse to the block's children
......
common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py
...@@ -161,155 +161,146 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -161,155 +161,146 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
""" """
Save the metadata in each store and retrieve it singularly, as all assets, and after deleting all. Save the metadata in each store and retrieve it singularly, as all assets, and after deleting all.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_filename = 'burnside.jpg'
asset_filename = 'burnside.jpg' new_asset_loc = course.id.make_asset_key('asset', asset_filename)
new_asset_loc = course.id.make_asset_key('asset', asset_filename) # Save the asset's metadata.
# Save the asset's metadata. new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) # Find the asset's metadata and confirm it's the same.
# Find the asset's metadata and confirm it's the same. found_asset_md = store.find_asset_metadata(new_asset_loc)
found_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(found_asset_md)
self.assertIsNotNone(found_asset_md) self.assertEquals(new_asset_md, found_asset_md)
self.assertEquals(new_asset_md, found_asset_md) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_delete(self, storebuilder): def test_delete(self, storebuilder):
""" """
Delete non-existent and existent metadata Delete non-existent and existent metadata
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') # Attempt to delete an asset that doesn't exist.
# Attempt to delete an asset that doesn't exist. self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
new_asset_md = self._make_asset_metadata(new_asset_loc) new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1) self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_find_non_existing_assets(self, storebuilder): def test_find_non_existing_assets(self, storebuilder):
""" """
Find a non-existent asset in an existing course. Find a non-existent asset in an existing course.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') # Find existing asset metadata.
# Find existing asset metadata. asset_md = store.find_asset_metadata(new_asset_loc)
asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNone(asset_md)
self.assertIsNone(asset_md)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_get_all_non_existing_assets(self, storebuilder): def test_get_all_non_existing_assets(self, storebuilder):
""" """
Get all assets in an existing course when no assets exist. Get all assets in an existing course when no assets exist.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) # Find existing asset metadata.
# Find existing asset metadata. asset_md = store.get_all_asset_metadata(course.id, 'asset')
asset_md = store.get_all_asset_metadata(course.id, 'asset') self.assertEquals(asset_md, [])
self.assertEquals(asset_md, [])
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_find_assets_in_non_existent_course(self, storebuilder): def test_find_assets_in_non_existent_course(self, storebuilder):
""" """
Find asset metadata from a non-existent course. Find asset metadata from a non-existent course.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format(
fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format( course.id.org, course.id.course, course.id.run
course.id.org, course.id.course, course.id.run ))
)) new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg') # Find asset metadata from non-existent course.
# Find asset metadata from non-existent course. with self.assertRaises(ItemNotFoundError):
with self.assertRaises(ItemNotFoundError): store.find_asset_metadata(new_asset_loc)
store.find_asset_metadata(new_asset_loc) with self.assertRaises(ItemNotFoundError):
with self.assertRaises(ItemNotFoundError): store.get_all_asset_metadata(fake_course_id, 'asset')
store.get_all_asset_metadata(fake_course_id, 'asset')
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_add_same_asset_twice(self, storebuilder): def test_add_same_asset_twice(self, storebuilder):
""" """
Add an asset's metadata, then add it again. Add an asset's metadata, then add it again.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) # Add asset metadata.
# Add asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) # Add *the same* asset metadata.
# Add *the same* asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) # Still one here?
# Still one here? self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_different_asset_types(self, storebuilder): def test_different_asset_types(self, storebuilder):
""" """
Test saving assets with other asset types. Test saving assets with other asset types.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('vrml', 'pyramid.vrml')
new_asset_loc = course.id.make_asset_key('vrml', 'pyramid.vrml') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) # Add asset metadata.
# Add asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_asset_types_with_other_field_names(self, storebuilder): def test_asset_types_with_other_field_names(self, storebuilder):
""" """
Test saving assets using an asset type of 'course_id'. Test saving assets using an asset type of 'course_id'.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('course_id', 'just_to_see_if_it_still_works.jpg')
new_asset_loc = course.id.make_asset_key('course_id', 'just_to_see_if_it_still_works.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) # Add asset metadata.
# Add asset metadata. store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) all_assets = store.get_all_asset_metadata(course.id, 'course_id')
all_assets = store.get_all_asset_metadata(course.id, 'course_id') self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path)
self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_lock_unlock_assets(self, storebuilder): def test_lock_unlock_assets(self, storebuilder):
""" """
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
locked_state = new_asset_md.locked
locked_state = new_asset_md.locked # Flip the course asset's locked status.
# Flip the course asset's locked status. store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test) # Find the same course and check its locked status.
# Find the same course and check its locked status. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) self.assertEquals(updated_asset_md.locked, not locked_state)
self.assertEquals(updated_asset_md.locked, not locked_state) # Now flip it back.
# Now flip it back. store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test) reupdated_asset_md = store.find_asset_metadata(new_asset_loc)
reupdated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(reupdated_asset_md)
self.assertIsNotNone(reupdated_asset_md) self.assertEquals(reupdated_asset_md.locked, locked_state)
self.assertEquals(reupdated_asset_md.locked, locked_state)
ALLOWED_ATTRS = ( ALLOWED_ATTRS = (
('pathname', '/new/path'), ('pathname', '/new/path'),
...@@ -340,98 +331,93 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -340,98 +331,93 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
""" """
Save setting each attr one at a time Save setting each attr one at a time
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) for attribute, value in self.ALLOWED_ATTRS:
for attr, value in self.ALLOWED_ATTRS: # Set the course asset's attribute.
# Set the course asset's attr. store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) # Find the same course asset and check its changed attribute.
# Find the same course asset and check its changed attr. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) self.assertIsNotNone(getattr(updated_asset_md, attribute, None))
self.assertIsNotNone(getattr(updated_asset_md, attr, None)) self.assertEquals(getattr(updated_asset_md, attribute, None), value)
self.assertEquals(getattr(updated_asset_md, attr, None), value)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_set_disallowed_attrs(self, storebuilder): def test_set_disallowed_attrs(self, storebuilder):
""" """
setting disallowed attrs should fail setting disallowed attrs should fail
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) for attribute, value in self.DISALLOWED_ATTRS:
for attr, value in self.DISALLOWED_ATTRS: original_attr_val = getattr(new_asset_md, attribute)
original_attr_val = getattr(new_asset_md, attr) # Set the course asset's attribute.
# Set the course asset's attr. store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) # Find the same course and check its changed attribute.
# Find the same course and check its changed attr. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) self.assertIsNotNone(getattr(updated_asset_md, attribute, None))
self.assertIsNotNone(getattr(updated_asset_md, attr, None)) # Make sure that the attribute is unchanged from its original value.
# Make sure that the attr is unchanged from its original value. self.assertEquals(getattr(updated_asset_md, attribute, None), original_attr_val)
self.assertEquals(getattr(updated_asset_md, attr, None), original_attr_val)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_set_unknown_attrs(self, storebuilder): def test_set_unknown_attrs(self, storebuilder):
""" """
setting unknown attrs should fail setting unknown attrs should fail
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') new_asset_md = self._make_asset_metadata(new_asset_loc)
new_asset_md = self._make_asset_metadata(new_asset_loc) store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) for attribute, value in self.UNKNOWN_ATTRS:
for attr, value in self.UNKNOWN_ATTRS: # Set the course asset's attribute.
# Set the course asset's attr. store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test)
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) # Find the same course and check its changed attribute.
# Find the same course and check its changed attr. updated_asset_md = store.find_asset_metadata(new_asset_loc)
updated_asset_md = store.find_asset_metadata(new_asset_loc) self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(updated_asset_md) # Make sure the unknown field was *not* added.
# Make sure the unknown field was *not* added. with self.assertRaises(AttributeError):
with self.assertRaises(AttributeError): self.assertEquals(getattr(updated_asset_md, attribute), value)
self.assertEquals(getattr(updated_asset_md, attr), value)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_save_one_different_asset(self, storebuilder): def test_save_one_different_asset(self, storebuilder):
""" """
saving and deleting things which are not 'asset' saving and deleting things which are not 'asset'
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_key = course.id.make_asset_key('different', 'burn.jpg')
asset_key = course.id.make_asset_key('different', 'burn.jpg') new_asset_thumbnail = self._make_asset_thumbnail_metadata(
new_asset_thumbnail = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) )
) store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_find_different(self, storebuilder): def test_find_different(self, storebuilder):
""" """
finding things which are of type other than 'asset' finding things which are of type other than 'asset'
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_key = course.id.make_asset_key('different', 'burn.jpg')
asset_key = course.id.make_asset_key('different', 'burn.jpg') new_asset_thumbnail = self._make_asset_thumbnail_metadata(
new_asset_thumbnail = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) )
) store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertIsNotNone(store.find_asset_metadata(asset_key)) self.assertIsNotNone(store.find_asset_metadata(asset_key))
unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg') unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg')
self.assertIsNone(store.find_asset_metadata(unknown_asset_key)) self.assertIsNone(store.find_asset_metadata(unknown_asset_key))
def _check_asset_values(self, assets, orig): def _check_asset_values(self, assets, orig):
""" """
...@@ -447,37 +433,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -447,37 +433,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
getting all things which are of type other than 'asset' getting all things which are of type other than 'asset'
""" """
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store)
# Save 'em.
# Save 'em. for asset_type, filename in self.alls:
for asset_type, filename in self.alls: asset_key = course.id.make_asset_key(asset_type, filename)
asset_key = course.id.make_asset_key(asset_type, filename) new_asset = self._make_asset_thumbnail_metadata(
new_asset = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key)
)
store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test)
# Check 'em.
for asset_type, asset_list in (
('different', self.differents),
('vrml', self.vrmls),
('asset', self.regular_assets),
):
assets = store.get_all_asset_metadata(course.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
course.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
) )
self.assertEquals(len(assets), len(self.alls)) store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test)
self._check_asset_values(assets, self.alls)
# Check 'em.
for asset_type, asset_list in (
('different', self.differents),
('vrml', self.vrmls),
('asset', self.regular_assets),
):
assets = store.get_all_asset_metadata(course.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
course.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(assets), len(self.alls))
self._check_asset_values(assets, self.alls)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_save_metadata_list(self, storebuilder): def test_save_metadata_list(self, storebuilder):
...@@ -485,40 +470,39 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -485,40 +470,39 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
Save a list of asset metadata all at once. Save a list of asset metadata all at once.
""" """
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store)
# Make a list of AssetMetadata objects.
# Make a list of AssetMetadata objects. md_list = []
md_list = [] for asset_type, filename in self.alls:
for asset_type, filename in self.alls: asset_key = course.id.make_asset_key(asset_type, filename)
asset_key = course.id.make_asset_key(asset_type, filename) md_list.append(self._make_asset_thumbnail_metadata(
md_list.append(self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) ))
))
# Save 'em.
# Save 'em. store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
# Check 'em.
# Check 'em. for asset_type, asset_list in (
for asset_type, asset_list in ( ('different', self.differents),
('different', self.differents), ('vrml', self.vrmls),
('vrml', self.vrmls), ('asset', self.regular_assets),
('asset', self.regular_assets), ):
): assets = store.get_all_asset_metadata(course.id, asset_type)
assets = store.get_all_asset_metadata(course.id, asset_type) self.assertEquals(len(assets), len(asset_list))
self.assertEquals(len(assets), len(asset_list)) self._check_asset_values(assets, asset_list)
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0) self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
assets = store.get_all_asset_metadata( course.id, None, start=0, maxresults=-1,
course.id, None, start=0, maxresults=-1, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(assets), len(self.alls))
self.assertEquals(len(assets), len(self.alls)) self._check_asset_values(assets, self.alls)
self._check_asset_values(assets, self.alls)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_save_metadata_list_with_mismatched_asset(self, storebuilder): def test_save_metadata_list_with_mismatched_asset(self, storebuilder):
...@@ -526,140 +510,137 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -526,140 +510,137 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
Save a list of asset metadata all at once - but with one asset's metadata from a different course. Save a list of asset metadata all at once - but with one asset's metadata from a different course.
""" """
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store)
# Make a list of AssetMetadata objects.
# Make a list of AssetMetadata objects. md_list = []
md_list = [] for asset_type, filename in self.alls:
for asset_type, filename in self.alls: if asset_type == 'asset':
if asset_type == 'asset': asset_key = course2.id.make_asset_key(asset_type, filename)
asset_key = course2.id.make_asset_key(asset_type, filename) else:
else: asset_key = course1.id.make_asset_key(asset_type, filename)
asset_key = course1.id.make_asset_key(asset_type, filename) md_list.append(self._make_asset_thumbnail_metadata(
md_list.append(self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) ))
))
# Save 'em.
# Save 'em. store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test)
# Check 'em.
# Check 'em. for asset_type, asset_list in (
for asset_type, asset_list in ( ('different', self.differents),
('different', self.differents), ('vrml', self.vrmls),
('vrml', self.vrmls), ):
): assets = store.get_all_asset_metadata(course1.id, asset_type)
assets = store.get_all_asset_metadata(course1.id, asset_type) self.assertEquals(len(assets), len(asset_list))
self.assertEquals(len(assets), len(asset_list)) self._check_asset_values(assets, asset_list)
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3)
assets = store.get_all_asset_metadata(
assets = store.get_all_asset_metadata( course1.id, None, start=0, maxresults=-1,
course1.id, None, start=0, maxresults=-1, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(assets), len(self.differents + self.vrmls))
self.assertEquals(len(assets), len(self.differents + self.vrmls)) self._check_asset_values(assets, self.differents + self.vrmls)
self._check_asset_values(assets, self.differents + self.vrmls)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_delete_all_different_type(self, storebuilder): def test_delete_all_different_type(self, storebuilder):
""" """
deleting all assets of a given but not 'asset' type deleting all assets of a given but not 'asset' type
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course = CourseFactory.create(modulestore=store)
course = CourseFactory.create(modulestore=store) asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg')
asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg') new_asset_thumbnail = self._make_asset_thumbnail_metadata(
new_asset_thumbnail = self._make_asset_thumbnail_metadata( self._make_asset_metadata(asset_key)
self._make_asset_metadata(asset_key) )
) store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_get_all_assets_with_paging(self, storebuilder): def test_get_all_assets_with_paging(self, storebuilder):
""" """
Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store) self.setup_assets(course1.id, course2.id, store)
self.setup_assets(course1.id, course2.id, store)
expected_sorts_by_2 = (
expected_sorts_by_2 = ( (
( ('displayname', ModuleStoreEnum.SortOrder.ascending),
('displayname', ModuleStoreEnum.SortOrder.ascending), ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'),
('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'), (2, 2, 1)
(2, 2, 1) ),
), (
( ('displayname', ModuleStoreEnum.SortOrder.descending),
('displayname', ModuleStoreEnum.SortOrder.descending), ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'),
('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'), (2, 2, 1)
(2, 2, 1) ),
), (
( ('uploadDate', ModuleStoreEnum.SortOrder.ascending),
('uploadDate', ModuleStoreEnum.SortOrder.ascending), ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'),
('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'), (2, 2, 1)
(2, 2, 1) ),
), (
( ('uploadDate', ModuleStoreEnum.SortOrder.descending),
('uploadDate', ModuleStoreEnum.SortOrder.descending), ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'),
('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'), (2, 2, 1)
(2, 2, 1) ),
), )
) # First, with paging across all sorts.
# First, with paging across all sorts. for sort_test in expected_sorts_by_2:
for sort_test in expected_sorts_by_2: for i in xrange(3):
for i in xrange(3): asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0]
course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0] )
) num_expected_results = sort_test[2][i]
num_expected_results = sort_test[2][i] expected_filename = sort_test[1][2 * i]
expected_filename = sort_test[1][2 * i] self.assertEquals(len(asset_page), num_expected_results)
self.assertEquals(len(asset_page), num_expected_results) self.assertEquals(asset_page[0].asset_id.path, expected_filename)
self.assertEquals(asset_page[0].asset_id.path, expected_filename) if num_expected_results == 2:
if num_expected_results == 2: expected_filename = sort_test[1][(2 * i) + 1]
expected_filename = sort_test[1][(2 * i) + 1] self.assertEquals(asset_page[1].asset_id.path, expected_filename)
self.assertEquals(asset_page[1].asset_id.path, expected_filename)
# Now fetch everything.
# Now fetch everything. asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(asset_page), 5)
self.assertEquals(len(asset_page), 5) self.assertEquals(asset_page[0].asset_id.path, 'code.tgz')
self.assertEquals(asset_page[0].asset_id.path, 'code.tgz') self.assertEquals(asset_page[1].asset_id.path, 'demo.swf')
self.assertEquals(asset_page[1].asset_id.path, 'demo.swf') self.assertEquals(asset_page[2].asset_id.path, 'dog.png')
self.assertEquals(asset_page[2].asset_id.path, 'dog.png') self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf')
self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf') self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
# Some odd conditions.
# Some odd conditions. asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(asset_page), 0)
self.assertEquals(len(asset_page), 0) asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=3, maxresults=0,
course2.id, 'asset', start=3, maxresults=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(asset_page), 0)
self.assertEquals(len(asset_page), 0) asset_page = store.get_all_asset_metadata(
asset_page = store.get_all_asset_metadata( course2.id, 'asset', start=3, maxresults=-12345,
course2.id, 'asset', start=3, maxresults=-12345, sort=('displayname', ModuleStoreEnum.SortOrder.descending)
sort=('displayname', ModuleStoreEnum.SortOrder.descending) )
) self.assertEquals(len(asset_page), 2)
self.assertEquals(len(asset_page), 2)
@ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())])) @ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())]))
def test_xml_not_yet_implemented(self, storebuilder): def test_xml_not_yet_implemented(self, storebuilder):
""" """
Test coverage which shows that for now xml read operations are not implemented Test coverage which shows that for now xml read operations are not implemented
""" """
with storebuilder.build(None) as store: with storebuilder.build(contentstore=None) as (__, store):
course_key = store.make_course_key("org", "course", "run") course_key = store.make_course_key("org", "course", "run")
asset_key = course_key.make_asset_key('asset', 'foo.jpg') asset_key = course_key.make_asset_key('asset', 'foo.jpg')
self.assertEquals(store.find_asset_metadata(asset_key), None) self.assertEquals(store.find_asset_metadata(asset_key), None)
...@@ -670,38 +651,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -670,38 +651,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
""" """
Create a course with assets, copy them all to another course in the same modulestore, and check on it. Create a course with assets, copy them all to another course in the same modulestore, and check on it.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store) self.setup_assets(course1.id, None, store)
self.setup_assets(course1.id, None, store) self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) all_assets = store.get_all_asset_metadata(
all_assets = store.get_all_asset_metadata( course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(all_assets), 2)
self.assertEquals(len(all_assets), 2) self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
@ddt.data(*MODULESTORE_SETUPS) @ddt.data(*MODULESTORE_SETUPS)
def test_copy_all_assets_from_course_with_no_assets(self, storebuilder): def test_copy_all_assets_from_course_with_no_assets(self, storebuilder):
""" """
Create a course with *no* assets, and try copy them all to another course in the same modulestore. Create a course with *no* assets, and try copy them all to another course in the same modulestore.
""" """
with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build() as (__, store):
with storebuilder.build(contentstore) as store: course1 = CourseFactory.create(modulestore=store)
course1 = CourseFactory.create(modulestore=store) course2 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store) store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) all_assets = store.get_all_asset_metadata(
all_assets = store.get_all_asset_metadata( course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(all_assets), 0)
self.assertEquals(len(all_assets), 0)
@ddt.data( @ddt.data(
('mongo', 'split'), ('mongo', 'split'),
...@@ -713,19 +692,18 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): ...@@ -713,19 +692,18 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
Create a course with assets, copy them all to another course in a different modulestore, and check on it. Create a course with assets, copy them all to another course in a different modulestore, and check on it.
""" """
mixed_builder = MIXED_MODULESTORE_BOTH_SETUP mixed_builder = MIXED_MODULESTORE_BOTH_SETUP
with MongoContentstoreBuilder().build() as contentstore: with mixed_builder.build() as (__, mixed_store):
with mixed_builder.build(contentstore) as mixed_store: with mixed_store.default_store(from_store):
with mixed_store.default_store(from_store): course1 = CourseFactory.create(modulestore=mixed_store)
course1 = CourseFactory.create(modulestore=mixed_store) with mixed_store.default_store(to_store):
with mixed_store.default_store(to_store): course2 = CourseFactory.create(modulestore=mixed_store)
course2 = CourseFactory.create(modulestore=mixed_store) self.setup_assets(course1.id, None, mixed_store)
self.setup_assets(course1.id, None, mixed_store) self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2) self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0) mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102)
mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102) all_assets = mixed_store.get_all_asset_metadata(
all_assets = mixed_store.get_all_asset_metadata( course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) )
) self.assertEquals(len(all_assets), 2)
self.assertEquals(len(all_assets), 2) self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
...@@ -76,12 +76,66 @@ class MemoryCache(object): ...@@ -76,12 +76,66 @@ class MemoryCache(object):
self._data[key] = value self._data[key] = value
class MongoModulestoreBuilder(object): class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database() # pylint: disable=protected-access
def __repr__(self):
return 'MongoContentstoreBuilder()'
class StoreBuilderBase(object):
"""
Base class for all modulestore builders.
"""
@contextmanager
def build(self, **kwargs):
"""
Build the modulstore, optionally building the contentstore as well.
"""
contentstore = kwargs.pop('contentstore', None)
if not contentstore:
with self.build_without_contentstore() as (contentstore, modulestore):
yield contentstore, modulestore
else:
with self.build_with_contentstore(contentstore) as modulestore:
yield modulestore
@contextmanager
def build_without_contentstore(self):
"""
Build both the contentstore and the modulestore.
"""
with MongoContentstoreBuilder().build() as contentstore:
with self.build_with_contentstore(contentstore) as modulestore:
yield contentstore, modulestore
class MongoModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a DraftModuleStore. A builder class for a DraftModuleStore.
""" """
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns an isolated mongo modulestore, and then deletes A contextmanager that returns an isolated mongo modulestore, and then deletes
all of its data at the end of the context. all of its data at the end of the context.
...@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object): ...@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object):
return 'MongoModulestoreBuilder()' return 'MongoModulestoreBuilder()'
class VersioningModulestoreBuilder(object): class VersioningModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a VersioningModuleStore. A builder class for a VersioningModuleStore.
""" """
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns an isolated versioning modulestore, and then deletes A contextmanager that returns an isolated versioning modulestore, and then deletes
all of its data at the end of the context. all of its data at the end of the context.
...@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object): ...@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object):
return 'SplitModulestoreBuilder()' return 'SplitModulestoreBuilder()'
class XmlModulestoreBuilder(object): class XmlModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a XMLModuleStore. A builder class for a XMLModuleStore.
""" """
# pylint: disable=unused-argument # pylint: disable=unused-argument
@contextmanager @contextmanager
def build(self, contentstore=None, course_ids=None): def build_with_contentstore(self, contentstore=None, course_ids=None):
""" """
A contextmanager that returns an isolated xml modulestore A contextmanager that returns an isolated xml modulestore
...@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object): ...@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object):
yield modulestore yield modulestore
class MixedModulestoreBuilder(object): class MixedModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a MixedModuleStore. A builder class for a MixedModuleStore.
""" """
...@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object): ...@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object):
self.mixed_modulestore = None self.mixed_modulestore = None
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns a mixed modulestore built on top of modulestores A contextmanager that returns a mixed modulestore built on top of modulestores
generated by other builder classes. generated by other builder classes.
...@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object): ...@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object):
""" """
names, generators = zip(*self.store_builders) names, generators = zip(*self.store_builders)
with nested(*(gen.build(contentstore) for gen in generators)) as modulestores: with nested(*(gen.build_with_contentstore(contentstore) for gen in generators)) as modulestores:
# Make the modulestore creation function just return the already-created modulestores # Make the modulestore creation function just return the already-created modulestores
store_iterator = iter(modulestores) store_iterator = iter(modulestores)
create_modulestore_instance = lambda *args, **kwargs: store_iterator.next() create_modulestore_instance = lambda *args, **kwargs: store_iterator.next()
...@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object): ...@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object):
return store.db_connection.structures return store.db_connection.structures
class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database()
def __repr__(self):
return 'MongoContentstoreBuilder()'
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([ MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
('draft', MongoModulestoreBuilder()), ('draft', MongoModulestoreBuilder()),
('split', VersioningModulestoreBuilder()) ('split', VersioningModulestoreBuilder())
...@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase): ...@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
# Construct the contentstore for storing the first import # Construct the contentstore for storing the first import
with source_content_builder.build() as source_content: with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore) # Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store: with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import # Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content: with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore) # Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store: with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course') source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course')
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_call_count.py
...@@ -6,18 +6,21 @@ when using the Split modulestore. ...@@ -6,18 +6,21 @@ when using the Split modulestore.
from tempfile import mkdtemp from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
from unittest import TestCase from unittest import TestCase
import ddt
from xmodule.modulestore.xml_importer import import_from_xml from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.tests.factories import check_mongo_calls from xmodule.modulestore.tests.factories import check_mongo_calls
from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MongoContentstoreBuilder, MixedModulestoreBuilder, VersioningModulestoreBuilder, MixedModulestoreBuilder, VersioningModulestoreBuilder,
TEST_DATA_DIR MongoModulestoreBuilder, TEST_DATA_DIR
) )
MIXED_OLD_MONGO_MODULESTORE_BUILDER = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())])
MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]) MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())])
@ddt.ddt
class CountMongoCallsXMLRoundtrip(TestCase): class CountMongoCallsXMLRoundtrip(TestCase):
""" """
This class exists to test XML import and export to/from Split. This class exists to test XML import and export to/from Split.
...@@ -28,92 +31,92 @@ class CountMongoCallsXMLRoundtrip(TestCase): ...@@ -28,92 +31,92 @@ class CountMongoCallsXMLRoundtrip(TestCase):
self.export_dir = mkdtemp() self.export_dir = mkdtemp()
self.addCleanup(rmtree, self.export_dir, ignore_errors=True) self.addCleanup(rmtree, self.export_dir, ignore_errors=True)
def test_import_export(self): @ddt.data(
# Construct the contentstore for storing the first import (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 287, 780, 702, 702),
with MongoContentstoreBuilder().build() as source_content: (MIXED_SPLIT_MODULESTORE_BUILDER, 37, 16, 190, 189),
# Construct the modulestore for storing the first import (using the previously created contentstore) )
with MIXED_SPLIT_MODULESTORE_BUILDER.build(source_content) as source_store: @ddt.unpack
# Construct the contentstore for storing the second import def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with MongoContentstoreBuilder().build() as dest_content: with store_builder.build() as (source_content, source_store):
# Construct the modulestore for storing the second import (using the second contentstore) with store_builder.build() as (dest_content, dest_store):
with MIXED_SPLIT_MODULESTORE_BUILDER.build(dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course')
source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
with check_mongo_calls(16, 190): # the course id and the wiki_slug in the test XML course. The course must be updated
import_from_xml( # with the correct wiki_slug during import.
source_store, with check_mongo_calls(import_reads, first_import_writes):
'test_user', import_from_xml(
TEST_DATA_DIR, source_store,
course_dirs=['manual-testing-complete'], 'test_user',
static_content_store=source_content, TEST_DATA_DIR,
target_course_id=source_course_key, course_dirs=['manual-testing-complete'],
create_course_if_not_present=True, static_content_store=source_content,
raise_on_failure=True, target_course_id=source_course_key,
) create_course_if_not_present=True,
raise_on_failure=True,
with check_mongo_calls(37): )
export_to_xml(
source_store, with check_mongo_calls(export_reads):
source_content, export_to_xml(
source_course_key, source_store,
self.export_dir, source_content,
'exported_source_course', source_course_key,
) self.export_dir,
'exported_source_course',
with check_mongo_calls(16, 189): )
import_from_xml(
dest_store, with check_mongo_calls(import_reads, second_import_writes):
'test_user', import_from_xml(
self.export_dir, dest_store,
course_dirs=['exported_source_course'], 'test_user',
static_content_store=dest_content, self.export_dir,
target_course_id=dest_course_key, course_dirs=['exported_source_course'],
create_course_if_not_present=True, static_content_store=dest_content,
raise_on_failure=True, target_course_id=dest_course_key,
) create_course_if_not_present=True,
raise_on_failure=True,
)
@ddt.ddt
class CountMongoCallsCourseTraversal(TestCase): class CountMongoCallsCourseTraversal(TestCase):
""" """
Tests the number of Mongo calls made when traversing a course tree from the top course root Tests the number of Mongo calls made when traversing a course tree from the top course root
to the leaf nodes. to the leaf nodes.
""" """
def test_number_mongo_calls(self): @ddt.data(
# Construct the contentstore for storing the course import (None, 7), # The way this traversal *should* be done.
with MongoContentstoreBuilder().build() as source_content: (0, 145) # The pathological case - do *not* query a course this way!
# Construct the modulestore for storing the course import (using the previously created contentstore) )
with MIXED_SPLIT_MODULESTORE_BUILDER.build(source_content) as source_store: @ddt.unpack
def test_number_mongo_calls(self, depth, num_mongo_calls):
source_course_key = source_store.make_course_key('a', 'course', 'course') with MIXED_SPLIT_MODULESTORE_BUILDER.build() as (source_content, source_store):
# First, import a course. source_course_key = source_store.make_course_key('a', 'course', 'course')
import_from_xml(
source_store, # First, import a course.
'test_user', import_from_xml(
TEST_DATA_DIR, source_store,
course_dirs=['manual-testing-complete'], 'test_user',
static_content_store=source_content, TEST_DATA_DIR,
target_course_id=source_course_key, course_dirs=['manual-testing-complete'],
create_course_if_not_present=True, static_content_store=source_content,
raise_on_failure=True, target_course_id=source_course_key,
) create_course_if_not_present=True,
raise_on_failure=True,
# Course traversal modeled after the traversal done here: )
# lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# Starting at the root course block, do a breadth-first traversal using # Course traversal modeled after the traversal done here:
# get_children() to retrieve each block's children. # lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# pylint: disable=bad-continuation # Starting at the root course block, do a breadth-first traversal using
for depth, num_calls in ( # get_children() to retrieve each block's children.
(None, 7), # The way this traversal *should* be done. with check_mongo_calls(num_mongo_calls):
(0, 145) # The pathological case - do *not* query a course this way! start_block = source_store.get_course(source_course_key, depth=depth)
): stack = [start_block]
with check_mongo_calls(num_calls): while stack:
start_block = source_store.get_course(source_course_key, depth=depth) curr_block = stack.pop()
stack = [start_block] if curr_block.has_children:
while stack: for block in reversed(curr_block.get_children()):
curr_block = stack.pop() stack.append(block)
if curr_block.has_children:
for block in reversed(curr_block.get_children()):
stack.append(block)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment