Skip to content

E
edx-platform
Commit c3d2962b by John Eskew
Options

Refactor contentstore/modulestore builders to wrap both in a single 

contextmananger. Add ddt to mongo call count testing.
parent 3565435e
Showing with 98 additions and 79 deletions
common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
......@@ -105,14 +105,8 @@ class CrossStoreXMLRoundtrip(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_ms.build(dest_content) as dest_store:
with source_ms.build() as (source_content, source_store):
with dest_ms.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
......@@ -193,10 +187,7 @@ class FindAssetTest(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store:
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
asset_key = source_course_key.make_asset_key(
AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif'
......@@ -265,10 +256,7 @@ class TestModulestoreAssetSize(unittest.TestCase):
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store:
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
import_from_xml(
......
common/lib/xmodule/xmodule/modulestore/tests/test_asides.py
......@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase):
"""
Check that the xml modulestore read in all the asides with their values
"""
with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as store:
with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as (__, store):
def check_block(block):
"""
Check whether block has the expected aside w/ its fields and then recurse to the block's children
......
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
......@@ -76,12 +76,66 @@ class MemoryCache(object):
self._data[key] = value
class MongoModulestoreBuilder(object):
class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database() # pylint: disable=protected-access
def __repr__(self):
return 'MongoContentstoreBuilder()'
class StoreBuilderBase(object):
"""
Base class for all modulestore builders.
"""
@contextmanager
def build(self, **kwargs):
"""
Build the modulstore, optionally building the contentstore as well.
"""
contentstore = kwargs.pop('contentstore', None)
if not contentstore:
with self.build_without_contentstore() as (contentstore, modulestore):
yield contentstore, modulestore
else:
with self.build_with_contentstore(contentstore) as modulestore:
yield modulestore
@contextmanager
def build_without_contentstore(self):
"""
Build both the contentstore and the modulestore.
"""
with MongoContentstoreBuilder().build() as contentstore:
with self.build_with_contentstore(contentstore) as modulestore:
yield contentstore, modulestore
class MongoModulestoreBuilder(StoreBuilderBase):
"""
A builder class for a DraftModuleStore.
"""
@contextmanager
def build(self, contentstore):
def build_with_contentstore(self, contentstore):
"""
A contextmanager that returns an isolated mongo modulestore, and then deletes
all of its data at the end of the context.
......@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object):
return 'MongoModulestoreBuilder()'
class VersioningModulestoreBuilder(object):
class VersioningModulestoreBuilder(StoreBuilderBase):
"""
A builder class for a VersioningModuleStore.
"""
@contextmanager
def build(self, contentstore):
def build_with_contentstore(self, contentstore):
"""
A contextmanager that returns an isolated versioning modulestore, and then deletes
all of its data at the end of the context.
......@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object):
return 'SplitModulestoreBuilder()'
class XmlModulestoreBuilder(object):
class XmlModulestoreBuilder(StoreBuilderBase):
"""
A builder class for a XMLModuleStore.
"""
# pylint: disable=unused-argument
@contextmanager
def build(self, contentstore=None, course_ids=None):
def build_with_contentstore(self, contentstore=None, course_ids=None):
"""
A contextmanager that returns an isolated xml modulestore
......@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object):
yield modulestore
class MixedModulestoreBuilder(object):
class MixedModulestoreBuilder(StoreBuilderBase):
"""
A builder class for a MixedModuleStore.
"""
......@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object):
self.mixed_modulestore = None
@contextmanager
def build(self, contentstore):
def build_with_contentstore(self, contentstore):
"""
A contextmanager that returns a mixed modulestore built on top of modulestores
generated by other builder classes.
......@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object):
"""
names, generators = zip(*self.store_builders)
with nested(*(gen.build(contentstore) for gen in generators)) as modulestores:
with nested(*(gen.build_with_contentstore(contentstore) for gen in generators)) as modulestores:
# Make the modulestore creation function just return the already-created modulestores
store_iterator = iter(modulestores)
create_modulestore_instance = lambda *args, **kwargs: store_iterator.next()
......@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object):
return store.db_connection.structures
class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database()
def __repr__(self):
return 'MongoContentstoreBuilder()'
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
('draft', MongoModulestoreBuilder()),
('split', VersioningModulestoreBuilder())
......@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store:
with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store:
with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_call_count.py
......@@ -6,18 +6,21 @@ when using the Split modulestore.
from tempfile import mkdtemp
from shutil import rmtree
from unittest import TestCase
import ddt
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.tests.factories import check_mongo_calls
from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MongoContentstoreBuilder, MixedModulestoreBuilder, VersioningModulestoreBuilder,
TEST_DATA_DIR
MixedModulestoreBuilder, VersioningModulestoreBuilder,
MongoModulestoreBuilder, TEST_DATA_DIR
)
MIXED_OLD_MONGO_MODULESTORE_BUILDER = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())])
MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())])
@ddt.ddt
class CountMongoCallsXMLRoundtrip(TestCase):
"""
This class exists to test XML import and export to/from Split.
......@@ -28,19 +31,21 @@ class CountMongoCallsXMLRoundtrip(TestCase):
self.export_dir = mkdtemp()
self.addCleanup(rmtree, self.export_dir, ignore_errors=True)
def test_import_export(self):
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with MIXED_SPLIT_MODULESTORE_BUILDER.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with MIXED_SPLIT_MODULESTORE_BUILDER.build(dest_content) as dest_store:
@ddt.data(
(MIXED_OLD_MONGO_MODULESTORE_BUILDER, 287, 780, 702, 702),
(MIXED_SPLIT_MODULESTORE_BUILDER, 37, 16, 190, 189),
)
@ddt.unpack
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
with check_mongo_calls(16, 190):
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_from_xml(
source_store,
'test_user',
......@@ -52,7 +57,7 @@ class CountMongoCallsXMLRoundtrip(TestCase):
raise_on_failure=True,
)
with check_mongo_calls(37):
with check_mongo_calls(export_reads):
export_to_xml(
source_store,
source_content,
......@@ -61,7 +66,7 @@ class CountMongoCallsXMLRoundtrip(TestCase):
'exported_source_course',
)
with check_mongo_calls(16, 189):
with check_mongo_calls(import_reads, second_import_writes):
import_from_xml(
dest_store,
'test_user',
......@@ -74,17 +79,20 @@ class CountMongoCallsXMLRoundtrip(TestCase):
)
@ddt.ddt
class CountMongoCallsCourseTraversal(TestCase):
"""
Tests the number of Mongo calls made when traversing a course tree from the top course root
to the leaf nodes.
"""
def test_number_mongo_calls(self):
# Construct the contentstore for storing the course import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the course import (using the previously created contentstore)
with MIXED_SPLIT_MODULESTORE_BUILDER.build(source_content) as source_store:
@ddt.data(
(None, 7), # The way this traversal *should* be done.
(0, 145) # The pathological case - do *not* query a course this way!
)
@ddt.unpack
def test_number_mongo_calls(self, depth, num_mongo_calls):
with MIXED_SPLIT_MODULESTORE_BUILDER.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
......@@ -104,12 +112,7 @@ class CountMongoCallsCourseTraversal(TestCase):
# lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# Starting at the root course block, do a breadth-first traversal using
# get_children() to retrieve each block's children.
# pylint: disable=bad-continuation
for depth, num_calls in (
(None, 7), # The way this traversal *should* be done.
(0, 145) # The pathological case - do *not* query a course this way!
):
with check_mongo_calls(num_calls):
with check_mongo_calls(num_mongo_calls):
start_block = source_store.get_course(source_course_key, depth=depth)
stack = [start_block]
while stack:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment