Skip to content

E
edx-platform
Commit 62495847 by John Eskew
Options

Merge pull request #6659 from edx/jeskew/asset_md_size_of_mongo_docs 

Performance test to find asset metadata size.
parents 11428fa6 a4bbed3b
Showing with 154 additions and 43 deletions
common/lib/xmodule/xmodule/modulestore/perf_tests/generate_asset_xml.py
......@@ -6,12 +6,15 @@ Generates fake XML for asset metadata.
"""
import random
#import click
from lxml import etree
from datetime import datetime, timedelta
from xmodule.assetstore import AssetMetadata
from opaque_keys.edx.keys import CourseKey
try:
import click
except ImportError:
click = None
# Name of the asset metadata XML schema definition file.
ASSET_XSD_FILE = 'assets.xsd'
......@@ -145,7 +148,7 @@ def generate_random_asset_md():
return AssetMetadata(
asset_key,
pathname=pathname(),
internal_name=filename(),
internal_name=str([filename() for __ in xrange(10)]),
locked=locked(),
contenttype=contenttype(),
thumbnail=filename(),
......@@ -199,33 +202,37 @@ def validate_xml(xsd_filename, xml_filename):
with open(xml_filename, 'r') as f:
etree.fromstring(f.read(), xmlparser)
# @click.command()
# @click.option('--numAssets',
# type=click.INT,
# default=10,
# help="Number of assets to be generated by the script.",
# required=False
# )
# @click.option('--outputXml',
# type=click.File('w'),
# default=AssetMetadata.EXPORTED_ASSET_FILENAME,
# help="Filename for the output XML file.",
# required=False
# )
# @click.option('--inputXsd',
# type=click.File('r'),
# default=ASSET_XSD_FILE,
# help="Filename for the XSD (schema) file to read in.",
# required=False
# )
# def cli(numAssets, outputXml, inputXsd):
# """
# Generates a number of fake asset metadata items as XML - and validates the XML against the schema.
# """
# make_asset_xml(numAssets, outputXml)
# # Now - validate the XML against the XSD.
# validate_xml(inputXsd, outputXml)
# if __name__ == '__main__':
# cli()
if click is not None:
# pylint: disable=bad-continuation
@click.command()
@click.option('--num_assets',
type=click.INT,
default=10,
help="Number of assets to be generated by the script.",
required=False
)
@click.option('--output_xml',
type=click.File('w'),
default=AssetMetadata.EXPORTED_ASSET_FILENAME,
help="Filename for the output XML file.",
required=False
)
@click.option('--input_xsd',
type=click.File('r'),
default=ASSET_XSD_FILE,
help="Filename for the XSD (schema) file to read in.",
required=False
)
def cli(num_assets, output_xml, input_xsd):
"""
Generates a number of fake asset metadata items as XML - and validates the XML against the schema.
"""
make_asset_xml(num_assets, output_xml)
# Now - validate the XML against the XSD.
validate_xml(input_xsd, output_xml)
if __name__ == '__main__':
if click is not None:
cli() # pylint: disable=no-value-for-parameter
else:
print "Aborted! Module 'click' is not installed."
common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
......@@ -6,7 +6,8 @@ import unittest
from tempfile import mkdtemp
import itertools
from shutil import rmtree
from bson.code import Code
import datetime
import ddt
#from nose.plugins.attrib import attr
......@@ -31,7 +32,7 @@ except ImportError:
CodeBlockTimer = None
# Number of assets saved in the modulestore per test run.
ASSET_AMOUNT_PER_TEST = (1, 10, 100, 1000, 10000)
ASSET_AMOUNT_PER_TEST = (0, 1, 10, 100, 1000, 10000)
# Use only this course in asset metadata performance testing.
COURSE_NAME = 'manual-testing-complete'
......@@ -160,7 +161,7 @@ class FindAssetTest(unittest.TestCase):
classes with different amounts of asset metadata.
"""
# Use this attribute to skip this test on regular unittest CI runs.
# Use this attr to skip this test on regular unittest CI runs.
perf_test = True
def setUp(self):
......@@ -233,3 +234,79 @@ class FindAssetTest(unittest.TestCase):
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
)
@ddt.ddt
# Eventually, exclude this attribute from regular unittests while running *only* tests
# with this attribute during regular performance tests.
# @attr("perf_test")
@unittest.skip
class TestModulestoreAssetSize(unittest.TestCase):
"""
This class exists to measure the size of asset metadata in ifferent modulestore
classes with different amount of asset metadata.
"""
# Use this attribute to skip this test on regular unittest CI runs.
perf_test = True
test_run_time = datetime.datetime.now()
@ddt.data(*itertools.product(
MODULESTORE_SETUPS,
ASSET_AMOUNT_PER_TEST
))
@ddt.unpack
def test_asset_sizes(self, source_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store:
source_course_key = source_store.make_course_key('a', 'course', 'course')
import_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
asset_collection = source_ms.asset_collection()
# Ensure the asset collection exists.
if asset_collection.name in asset_collection.database.collection_names():
# Map gets the size of each structure.
mapper = Code("""
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
""")
# Reduce finds the largest structure size and returns only it.
reducer = Code("""
function(key, values) {
var max_size = 0;
for (var i=0; i < values.length; i++) {
if (values[i] > max_size) {
max_size = values[i];
}
}
return max_size;
}
""")
results = asset_collection.map_reduce(mapper, reducer, "size_results")
result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
)
with open("bson_sizes.txt", "a") as f:
f.write(result_str)
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
......@@ -100,7 +100,8 @@ class MongoModulestoreBuilder(object):
# Set up a temp directory for storing filesystem content created during import
fs_root = mkdtemp()
modulestore = DraftModuleStore(
# pylint: disable=attribute-defined-outside-init
self.modulestore = DraftModuleStore(
contentstore,
doc_store_config,
fs_root,
......@@ -109,13 +110,13 @@ class MongoModulestoreBuilder(object):
metadata_inheritance_cache_subsystem=MemoryCache(),
xblock_mixins=XBLOCK_MIXINS,
)
modulestore.ensure_indexes()
self.modulestore.ensure_indexes()
try:
yield modulestore
yield self.modulestore
finally:
# Delete the created database
modulestore._drop_database()
self.modulestore._drop_database() # pylint: disable=protected-access
# Delete the created directory on the filesystem
rmtree(fs_root, ignore_errors=True)
......@@ -123,6 +124,12 @@ class MongoModulestoreBuilder(object):
def __repr__(self):
return 'MongoModulestoreBuilder()'
def asset_collection(self):
"""
Returns the collection storing the asset metadata.
"""
return self.modulestore.asset_collection
class VersioningModulestoreBuilder(object):
"""
......@@ -160,7 +167,7 @@ class VersioningModulestoreBuilder(object):
yield modulestore
finally:
# Delete the created database
modulestore._drop_database()
modulestore._drop_database() # pylint: disable=protected-access
# Delete the created directory on the filesystem
rmtree(fs_root, ignore_errors=True)
......@@ -206,6 +213,7 @@ class MixedModulestoreBuilder(object):
"""
self.store_builders = store_builders
self.mappings = mappings or {}
self.modulestore = None
@contextmanager
def build(self, contentstore):
......@@ -227,7 +235,7 @@ class MixedModulestoreBuilder(object):
# Generate a fake list of stores to give the already generated stores appropriate names
stores = [{'NAME': name, 'ENGINE': 'This space deliberately left blank'} for name in names]
modulestore = MixedModuleStore(
self.modulestore = MixedModuleStore(
contentstore,
self.mappings,
stores,
......@@ -235,11 +243,29 @@ class MixedModulestoreBuilder(object):
xblock_mixins=XBLOCK_MIXINS,
)
yield modulestore
yield self.modulestore
def __repr__(self):
return 'MixedModulestoreBuilder({!r}, {!r})'.format(self.store_builders, self.mappings)
def asset_collection(self):
"""
Returns the collection storing the asset metadata.
"""
all_stores = self.modulestore.modulestores
if len(all_stores) > 1:
return None
store = all_stores[0]
if hasattr(store, 'asset_collection'):
# Mongo modulestore beneath mixed.
# Returns the entire collection with *all* courses' asset metadata.
return store.asset_collection
else:
# Split modulestore beneath mixed.
# Split stores all asset metadata in the structure collection.
return store.db_connection.structures
class MongoContentstoreBuilder(object):
"""
......@@ -276,7 +302,8 @@ MIXED_MODULESTORE_SETUPS = (
MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]),
)
MIXED_MS_SETUPS_SHORT = (
'mixed_mongo', 'mixed_split'
'mixed_mongo',
'mixed_split',
)
DIRECT_MODULESTORE_SETUPS = (
MongoModulestoreBuilder(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment