Skip to content

E
edx-platform
Commit fbee3014 by David Ormsbee
Options

A first cut at storing data differently.

parent 396c7de8
Showing with 171 additions and 164 deletions
lms/djangoapps/course_blocks/transformers/visibility.py
......@@ -19,6 +19,10 @@ class VisibilityTransformer(BlockStructureTransformer):
)
@classmethod
def collect_2(cls, block_cache_unit):
pass
@classmethod
def collect(cls, block_structure):
"""
Collects any information that's necessary to execute this transformer's
......
openedx/core/lib/block_cache/block_cache.py
"""
...
.
"""
from block_structure import BlockStructureFactory
from transformer import BlockStructureTransformers
def get_blocks(root_block_key, transformers, user_info):
# Load the cached block structure. This will first try to find the exact
# block in ephemeral storage, then fall back to the root course block it
# belongs to in ephemeral storage, and then fall back to the root course
# block stored in permanent storage.
bcu = BlockCacheUnit.load(root_block_key, transformers)
# Note that each transform may mutate the
for transformer in transformers:
with bcu.collected_data_for(transformer) as collected_data:
transformer.transform(user_info, collected_data)
return bcu.structure
def get_blocks(cache, modulestore, user_info, root_block_key, transformers):
unregistered_transformers = BlockStructureTransformers.find_unregistered(transformers)
if unregistered_transformers:
......
openedx/core/lib/block_cache/block_structure.py
......@@ -11,8 +11,6 @@ from transformer import BlockStructureTransformers
logger = getLogger(__name__) # pylint: disable=C0103
TRANSFORMER_VERSION_KEY = '_version'
class BlockRelations(object):
def __init__(self):
......@@ -32,6 +30,9 @@ class BlockStructure(object):
def __iter__(self):
return self.topological_traversal()
def __contains__(self, usage_key):
return usage_key in self._block_relations
def add_relation(self, parent_key, child_key):
self._add_relation(self._block_relations, parent_key, child_key)
......@@ -88,58 +89,6 @@ class BlockStructure(object):
_ = block_relations[block_key]
class BlockData(object):
def __init__(self):
# dictionary mapping xblock field names to their values.
self._xblock_fields = {}
# dictionary mapping transformers' IDs to their collected data.
self._transformer_data = defaultdict(dict)
class BlockStructureBlockData(BlockStructure):
"""
A sub-class of BlockStructure that encapsulates data captured about the blocks.
"""
def __init__(self, root_block_key):
super(BlockStructureBlockData, self).__init__(root_block_key)
# dictionary mapping usage keys to BlockData
self._block_data_map = defaultdict(BlockData)
# dictionary mapping transformer IDs to block-structure-wide transformer data
self._transformer_data = defaultdict(dict)
def get_xblock_field(self, usage_key, field_name, default=None):
block_data = self._block_data_map.get(usage_key)
return block_data._xblock_fields.get(field_name, default) if block_data else default
def get_transformer_data(self, transformer, key, default=None):
return self._transformer_data.get(transformer.name(), {}).get(key, default)
def set_transformer_data(self, transformer, key, value):
self._transformer_data[transformer.name()][key] = value
def get_transformer_data_version(self, transformer):
return self.get_transformer_data(transformer, TRANSFORMER_VERSION_KEY, 0)
def get_transformer_block_data(self, usage_key, transformer, key=None, default=None):
block_data = self._block_data_map.get(usage_key)
if not block_data:
return default
else:
transformer_data = block_data._transformer_data.get(transformer.name(), {})
if key:
return transformer_data.get(key, default)
else:
return transformer_data or default
def set_transformer_block_data(self, usage_key, transformer, key, value):
self._block_data_map[usage_key]._transformer_data[transformer.name()][key] = value
def remove_transformer_block_data(self, usage_key, transformer, key):
self._block_data_map[usage_key]._transformer_data.get(transformer.name(), {}).pop(key, None)
def remove_block(self, usage_key, keep_descendants):
children = self._block_relations[usage_key].children
parents = self._block_relations[usage_key].parents
......@@ -168,48 +117,10 @@ class BlockStructureBlockData(BlockStructure):
return True
list(self.topological_traversal(predicate=predicate, **kwargs))
class BlockStructureCollectedData(BlockStructureBlockData):
"""
A sub-class of BlockStructure that encapsulates information about the blocks during the collect phase.
"""
def __init__(self, root_block_key):
super(BlockStructureCollectedData, self).__init__(root_block_key)
self._xblock_map = {} # dict[UsageKey: XBlock]
self._requested_xblock_fields = set()
def request_xblock_fields(self, *field_names):
self._requested_xblock_fields.update(set(field_names))
def collect_requested_xblock_fields(self):
if not self._requested_xblock_fields:
return
for xblock in self._xblock_map.itervalues():
for field_name in self._requested_xblock_fields:
self._set_xblock_field(xblock, field_name)
def _set_xblock_field(self, xblock, field_name):
if hasattr(xblock, field_name):
self._block_data_map[xblock.location]._xblock_fields[field_name] = getattr(xblock, field_name)
def add_xblock(self, xblock):
self._xblock_map[xblock.location] = xblock
def get_xblock(self, usage_key):
return self._xblock_map[usage_key]
def add_transformer(self, transformer):
if transformer.VERSION == 0:
raise Exception('VERSION attribute is not set on transformer {0}.', transformer.name())
self.set_transformer_data(transformer, TRANSFORMER_VERSION_KEY, transformer.VERSION)
class BlockStructureFactory(object):
@classmethod
def create_from_modulestore(cls, root_block_key, modulestore):
block_structure = BlockStructureCollectedData(root_block_key)
def load_from_xblock(cls, root_xblock):
root_block_key = root_xblock.location
block_structure = BlockStructure(root_block_key)
blocks_visited = set()
def build_block_structure(xblock):
......@@ -224,73 +135,6 @@ class BlockStructureFactory(object):
block_structure.add_relation(xblock.location, child.location)
build_block_structure(child)
root_xblock = modulestore.get_item(root_block_key, depth=None)
build_block_structure(root_xblock)
return block_structure
@classmethod
def serialize_to_cache(cls, block_structure, cache):
data_to_cache = (
block_structure._block_relations,
block_structure._transformer_data,
block_structure._block_data_map
)
zp_data_to_cache = zpickle(data_to_cache)
cache.set(
cls._encode_root_cache_key(block_structure.root_block_key),
zp_data_to_cache
)
logger.debug(
"Wrote BlockStructure {} to cache, size: {}".format(
block_structure.root_block_key, len(zp_data_to_cache)
)
)
@classmethod
def create_from_cache(cls, root_block_key, cache):
"""
Returns:
BlockStructure, if the block structure is in the cache, and
NoneType otherwise.
"""
zp_data_from_cache = cache.get(cls._encode_root_cache_key(root_block_key))
if not zp_data_from_cache:
return None
logger.debug(
"Read BlockStructure {} from cache, size: {}".format(
root_block_key, len(zp_data_from_cache)
)
)
block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache)
block_structure = BlockStructureBlockData(root_block_key)
block_structure._block_relations = block_relations
block_structure._transformer_data = transformer_data
block_structure._block_data_map = block_data_map
transformer_issues = {}
for transformer in BlockStructureTransformers.get_registered_transformers():
cached_transformer_version = block_structure.get_transformer_data_version(transformer)
if transformer.VERSION != cached_transformer_version:
transformer_issues[transformer.name()] = "version: {}, cached: {}".format(
transformer.VERSION,
cached_transformer_version,
)
if transformer_issues:
logger.info(
"Collected data for the following transformers have issues:\n{}."
).format('\n'.join([t_name + ": " + t_value for t_name, t_value in transformer_issues.iteritems()]))
return None
return block_structure
@classmethod
def remove_from_cache(cls, root_block_key, cache):
cache.delete(cls._encode_root_cache_key(root_block_key))
# TODO also remove all block data?
@classmethod
def _encode_root_cache_key(cls, root_block_key):
return "root.key." + unicode(root_block_key)
openedx/core/lib/block_cache/tests/test_bcu.py 0 → 100644
"""
Tests for block_cache.py
def transform(user_info, structure, collected_data):
field_values = collected_data.xblock_field_values
"""
from mock import patch
from unittest import TestCase
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locator import CourseLocator, LibraryLocator, BlockUsageLocator
from .test_utils import (
MockModulestoreFactory, MockCache, MockUserInfo, MockTransformer, ChildrenMapTestMixin
)
from ..block_cache import get_blocks
from ..bcu import BlockCacheUnit, BlockFieldValues, BlockIndexMapping, CollectionData
TEST_COURSE_KEY = CourseLocator(org="BCU", course="Fast", run="101")
class TestBlockCacheUnit(TestCase):
def setUp(self):
self.bcu = BlockCacheUnit(
block_structure,
xblock_field_values,
transformers_to_field_values,
transformers_to_data,
)
class TestBlockFieldValues(TestCase):
def setUp(self):
self.mapping = BlockIndexMapping(
make_locators(TEST_COURSE_KEY, chapter=2, vertical=2, html=5, problem=5)
)
self.field_values = BlockFieldValues(
self.mapping,
{
'block_id': [key.block_id for key in self.mapping],
'block_type': [key.block_type for key in self.mapping],
'has_score': [key.block_type == 'problem' for key in self.mapping],
'horribly_named': [key.block_type == 'vertical' for key in self.mapping],
}
)
def test_get(self):
# Check some values we expect
self.assertEqual(
'chapter',
self.field_values.get('block_type', BlockUsageLocator(TEST_COURSE_KEY, 'chapter', 'chapter_0'))
)
self.assertEqual(
'html_4',
self.field_values.get('block_id', BlockUsageLocator(TEST_COURSE_KEY, 'html', 'html_4'))
)
self.assertTrue(
self.field_values.get('horribly_named', BlockUsageLocator(TEST_COURSE_KEY, 'vertical', 'vertical_1'))
)
self.assertEqual(
self.field_values[BlockUsageLocator(TEST_COURSE_KEY, 'problem', 'problem_4')],
{
'block_type': 'problem',
'block_id': 'problem_4',
'has_score': True,
'horribly_named': False,
}
)
# Make sure we throw key errors for non-existent fields or block keys
with self.assertRaises(KeyError):
self.field_values.get('no_such_field', BlockUsageLocator(TEST_COURSE_KEY, 'html', 'html_1'))
with self.assertRaises(KeyError):
self.field_values.get('block_id', TEST_COURSE_KEY)
def test_slice_by_fields(self):
self.assertEqual(
['block_id', 'block_type', 'has_score', 'horribly_named'],
self.field_values.fields
)
chapter_key = BlockUsageLocator(TEST_COURSE_KEY, 'chapter', 'chapter_0')
empty = self.field_values.slice_by_fields([])
self.assertEqual([], empty.fields)
self.assertEqual({}, empty[chapter_key])
grading = self.field_values.slice_by_fields(['block_id', 'has_score'])
self.assertEqual(
{'block_id': 'chapter_0', 'has_score': False},
grading[chapter_key]
)
self.assertEqual('chapter_0', grading.get('block_id', chapter_key))
# Now test mutation -- these are supposed to point to the same underlying
# lists (or XBlock field mutations wouldn't carry across Transformers)
self.assertFalse(grading.get('has_score', chapter_key))
self.assertFalse(self.field_values.get('has_score', chapter_key))
grading.set('has_score', chapter_key, True)
self.assertTrue(grading.get('has_score', chapter_key))
self.assertTrue(self.field_values.get('has_score', chapter_key))
class TestBlockIndexMapping(TestCase):
def setUp(self):
self.locators = make_locators(
TEST_COURSE_KEY, chapter=2, vertical=3, html=5, problem=5, video=5
)
self.mapping = BlockIndexMapping(self.locators)
def test_locator_ordering(self):
"""Locators should iterate in sorted order."""
sorted_locators = sorted(self.locators)
self.assertEqual(sorted_locators, list(self.mapping))
def test_index_lookup(self):
self.assertEqual(0, self.mapping.index_for(BlockUsageLocator(TEST_COURSE_KEY, 'chapter', 'chapter_0')))
self.assertEqual(2, self.mapping.index_for(BlockUsageLocator(TEST_COURSE_KEY, 'course', '2015')))
self.assertEqual(20, self.mapping.index_for(BlockUsageLocator(TEST_COURSE_KEY, 'video', 'video_4')))
with self.assertRaises(KeyError):
self.mapping.index_for(TEST_COURSE_KEY)
def make_locators(course_key, **block_types_to_qty):
locators = [BlockUsageLocator(TEST_COURSE_KEY, 'course', '2015')]
for block_type, qty in block_types_to_qty.items():
for i in xrange(qty):
block_id = "{}_{}".format(block_type, i)
locators.append(BlockUsageLocator(TEST_COURSE_KEY, block_type, block_id))
return locators
openedx/core/lib/block_cache/transformer.py
"""
...
How many things affect a Transformer's output?
1. Collected Data for this Transform (both its own and any dependent XBlock fields)
2. Block Structure as it exists at this point in the chain (others could have modified it for their own reasons)
2. User info
3. Time
4. ???
"""
from abc import abstractmethod
from openedx.core.lib.api.plugins import PluginManager
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment