Skip to content

E
edx-platform
Commit c3d2962b by John Eskew
Options

Refactor contentstore/modulestore builders to wrap both in a single 

contextmananger. Add ddt to mongo call count testing.
parent 3565435e
Showing with 154 additions and 123 deletions
common/lib/xmodule/xmodule/modulestore/tests/test_asides.py
...@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase): ...@@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase):
""" """
Check that the xml modulestore read in all the asides with their values Check that the xml modulestore read in all the asides with their values
""" """
with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as store: with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as (__, store):
def check_block(block): def check_block(block):
""" """
Check whether block has the expected aside w/ its fields and then recurse to the block's children Check whether block has the expected aside w/ its fields and then recurse to the block's children
......
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
...@@ -76,12 +76,66 @@ class MemoryCache(object): ...@@ -76,12 +76,66 @@ class MemoryCache(object):
self._data[key] = value self._data[key] = value
class MongoModulestoreBuilder(object): class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database() # pylint: disable=protected-access
def __repr__(self):
return 'MongoContentstoreBuilder()'
class StoreBuilderBase(object):
"""
Base class for all modulestore builders.
"""
@contextmanager
def build(self, **kwargs):
"""
Build the modulstore, optionally building the contentstore as well.
"""
contentstore = kwargs.pop('contentstore', None)
if not contentstore:
with self.build_without_contentstore() as (contentstore, modulestore):
yield contentstore, modulestore
else:
with self.build_with_contentstore(contentstore) as modulestore:
yield modulestore
@contextmanager
def build_without_contentstore(self):
"""
Build both the contentstore and the modulestore.
"""
with MongoContentstoreBuilder().build() as contentstore:
with self.build_with_contentstore(contentstore) as modulestore:
yield contentstore, modulestore
class MongoModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a DraftModuleStore. A builder class for a DraftModuleStore.
""" """
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns an isolated mongo modulestore, and then deletes A contextmanager that returns an isolated mongo modulestore, and then deletes
all of its data at the end of the context. all of its data at the end of the context.
...@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object): ...@@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object):
return 'MongoModulestoreBuilder()' return 'MongoModulestoreBuilder()'
class VersioningModulestoreBuilder(object): class VersioningModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a VersioningModuleStore. A builder class for a VersioningModuleStore.
""" """
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns an isolated versioning modulestore, and then deletes A contextmanager that returns an isolated versioning modulestore, and then deletes
all of its data at the end of the context. all of its data at the end of the context.
...@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object): ...@@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object):
return 'SplitModulestoreBuilder()' return 'SplitModulestoreBuilder()'
class XmlModulestoreBuilder(object): class XmlModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a XMLModuleStore. A builder class for a XMLModuleStore.
""" """
# pylint: disable=unused-argument # pylint: disable=unused-argument
@contextmanager @contextmanager
def build(self, contentstore=None, course_ids=None): def build_with_contentstore(self, contentstore=None, course_ids=None):
""" """
A contextmanager that returns an isolated xml modulestore A contextmanager that returns an isolated xml modulestore
...@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object): ...@@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object):
yield modulestore yield modulestore
class MixedModulestoreBuilder(object): class MixedModulestoreBuilder(StoreBuilderBase):
""" """
A builder class for a MixedModuleStore. A builder class for a MixedModuleStore.
""" """
...@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object): ...@@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object):
self.mixed_modulestore = None self.mixed_modulestore = None
@contextmanager @contextmanager
def build(self, contentstore): def build_with_contentstore(self, contentstore):
""" """
A contextmanager that returns a mixed modulestore built on top of modulestores A contextmanager that returns a mixed modulestore built on top of modulestores
generated by other builder classes. generated by other builder classes.
...@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object): ...@@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object):
""" """
names, generators = zip(*self.store_builders) names, generators = zip(*self.store_builders)
with nested(*(gen.build(contentstore) for gen in generators)) as modulestores: with nested(*(gen.build_with_contentstore(contentstore) for gen in generators)) as modulestores:
# Make the modulestore creation function just return the already-created modulestores # Make the modulestore creation function just return the already-created modulestores
store_iterator = iter(modulestores) store_iterator = iter(modulestores)
create_modulestore_instance = lambda *args, **kwargs: store_iterator.next() create_modulestore_instance = lambda *args, **kwargs: store_iterator.next()
...@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object): ...@@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object):
return store.db_connection.structures return store.db_connection.structures
class MongoContentstoreBuilder(object):
"""
A builder class for a MongoContentStore.
"""
@contextmanager
def build(self):
"""
A contextmanager that returns a MongoContentStore, and deletes its contents
when the context closes.
"""
contentstore = MongoContentStore(
db='contentstore{}'.format(random.randint(0, 10000)),
collection='content',
**COMMON_DOCSTORE_CONFIG
)
contentstore.ensure_indexes()
try:
yield contentstore
finally:
# Delete the created database
contentstore._drop_database()
def __repr__(self):
return 'MongoContentstoreBuilder()'
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([ MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
('draft', MongoModulestoreBuilder()), ('draft', MongoModulestoreBuilder()),
('split', VersioningModulestoreBuilder()) ('split', VersioningModulestoreBuilder())
...@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase): ...@@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase):
# Construct the contentstore for storing the first import # Construct the contentstore for storing the first import
with source_content_builder.build() as source_content: with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore) # Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store: with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import # Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content: with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore) # Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store: with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course') source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course')
......
common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_call_count.py
...@@ -6,18 +6,21 @@ when using the Split modulestore. ...@@ -6,18 +6,21 @@ when using the Split modulestore.
from tempfile import mkdtemp from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
from unittest import TestCase from unittest import TestCase
import ddt
from xmodule.modulestore.xml_importer import import_from_xml from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.tests.factories import check_mongo_calls from xmodule.modulestore.tests.factories import check_mongo_calls
from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MongoContentstoreBuilder, MixedModulestoreBuilder, VersioningModulestoreBuilder, MixedModulestoreBuilder, VersioningModulestoreBuilder,
TEST_DATA_DIR MongoModulestoreBuilder, TEST_DATA_DIR
) )
MIXED_OLD_MONGO_MODULESTORE_BUILDER = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())])
MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]) MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())])
@ddt.ddt
class CountMongoCallsXMLRoundtrip(TestCase): class CountMongoCallsXMLRoundtrip(TestCase):
""" """
This class exists to test XML import and export to/from Split. This class exists to test XML import and export to/from Split.
...@@ -28,92 +31,92 @@ class CountMongoCallsXMLRoundtrip(TestCase): ...@@ -28,92 +31,92 @@ class CountMongoCallsXMLRoundtrip(TestCase):
self.export_dir = mkdtemp() self.export_dir = mkdtemp()
self.addCleanup(rmtree, self.export_dir, ignore_errors=True) self.addCleanup(rmtree, self.export_dir, ignore_errors=True)
def test_import_export(self): @ddt.data(
# Construct the contentstore for storing the first import (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 287, 780, 702, 702),
with MongoContentstoreBuilder().build() as source_content: (MIXED_SPLIT_MODULESTORE_BUILDER, 37, 16, 190, 189),
# Construct the modulestore for storing the first import (using the previously created contentstore) )
with MIXED_SPLIT_MODULESTORE_BUILDER.build(source_content) as source_store: @ddt.unpack
# Construct the contentstore for storing the second import def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with MongoContentstoreBuilder().build() as dest_content: with store_builder.build() as (source_content, source_store):
# Construct the modulestore for storing the second import (using the second contentstore) with store_builder.build() as (dest_content, dest_store):
with MIXED_SPLIT_MODULESTORE_BUILDER.build(dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course')
source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
with check_mongo_calls(16, 190): # the course id and the wiki_slug in the test XML course. The course must be updated
import_from_xml( # with the correct wiki_slug during import.
source_store, with check_mongo_calls(import_reads, first_import_writes):
'test_user', import_from_xml(
TEST_DATA_DIR, source_store,
course_dirs=['manual-testing-complete'], 'test_user',
static_content_store=source_content, TEST_DATA_DIR,
target_course_id=source_course_key, course_dirs=['manual-testing-complete'],
create_course_if_not_present=True, static_content_store=source_content,
raise_on_failure=True, target_course_id=source_course_key,
) create_course_if_not_present=True,
raise_on_failure=True,
with check_mongo_calls(37): )
export_to_xml(
source_store, with check_mongo_calls(export_reads):
source_content, export_to_xml(
source_course_key, source_store,
self.export_dir, source_content,
'exported_source_course', source_course_key,
) self.export_dir,
'exported_source_course',
with check_mongo_calls(16, 189): )
import_from_xml(
dest_store, with check_mongo_calls(import_reads, second_import_writes):
'test_user', import_from_xml(
self.export_dir, dest_store,
course_dirs=['exported_source_course'], 'test_user',
static_content_store=dest_content, self.export_dir,
target_course_id=dest_course_key, course_dirs=['exported_source_course'],
create_course_if_not_present=True, static_content_store=dest_content,
raise_on_failure=True, target_course_id=dest_course_key,
) create_course_if_not_present=True,
raise_on_failure=True,
)
@ddt.ddt
class CountMongoCallsCourseTraversal(TestCase): class CountMongoCallsCourseTraversal(TestCase):
""" """
Tests the number of Mongo calls made when traversing a course tree from the top course root Tests the number of Mongo calls made when traversing a course tree from the top course root
to the leaf nodes. to the leaf nodes.
""" """
def test_number_mongo_calls(self): @ddt.data(
# Construct the contentstore for storing the course import (None, 7), # The way this traversal *should* be done.
with MongoContentstoreBuilder().build() as source_content: (0, 145) # The pathological case - do *not* query a course this way!
# Construct the modulestore for storing the course import (using the previously created contentstore) )
with MIXED_SPLIT_MODULESTORE_BUILDER.build(source_content) as source_store: @ddt.unpack
def test_number_mongo_calls(self, depth, num_mongo_calls):
source_course_key = source_store.make_course_key('a', 'course', 'course') with MIXED_SPLIT_MODULESTORE_BUILDER.build() as (source_content, source_store):
# First, import a course. source_course_key = source_store.make_course_key('a', 'course', 'course')
import_from_xml(
source_store, # First, import a course.
'test_user', import_from_xml(
TEST_DATA_DIR, source_store,
course_dirs=['manual-testing-complete'], 'test_user',
static_content_store=source_content, TEST_DATA_DIR,
target_course_id=source_course_key, course_dirs=['manual-testing-complete'],
create_course_if_not_present=True, static_content_store=source_content,
raise_on_failure=True, target_course_id=source_course_key,
) create_course_if_not_present=True,
raise_on_failure=True,
# Course traversal modeled after the traversal done here: )
# lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# Starting at the root course block, do a breadth-first traversal using # Course traversal modeled after the traversal done here:
# get_children() to retrieve each block's children. # lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# pylint: disable=bad-continuation # Starting at the root course block, do a breadth-first traversal using
for depth, num_calls in ( # get_children() to retrieve each block's children.
(None, 7), # The way this traversal *should* be done. with check_mongo_calls(num_mongo_calls):
(0, 145) # The pathological case - do *not* query a course this way! start_block = source_store.get_course(source_course_key, depth=depth)
): stack = [start_block]
with check_mongo_calls(num_calls): while stack:
start_block = source_store.get_course(source_course_key, depth=depth) curr_block = stack.pop()
stack = [start_block] if curr_block.has_children:
while stack: for block in reversed(curr_block.get_children()):
curr_block = stack.pop() stack.append(block)
if curr_block.has_children:
for block in reversed(curr_block.get_children()):
stack.append(block)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment