Commit eb88d9da by Nimisha Asthagiri

Block Cache Framework

parent d3f39b2f
"""
...
"""
from django.core.cache import get_cache
from openedx.core.lib.block_cache.block_cache import get_blocks, clear_block_cache
from xmodule.modulestore.django import modulestore
from .transformers import (
start_date,
user_partitions,
visibility,
)
from .user_info import CourseUserInfo
LMS_COURSE_TRANSFORMERS = [
visibility.VisibilityTransformer(),
start_date.StartDateTransformer(),
user_partitions.UserPartitionTransformer(),
]
_cache = None
def _get_cache():
global _cache
if not _cache:
_cache = get_cache('lms.course_blocks')
return _cache
def get_course_blocks(
user,
root_usage_key,
transformers=LMS_COURSE_TRANSFORMERS,
):
if transformers is None:
transformers = LMS_COURSE_TRANSFORMERS
return get_blocks(
_get_cache(), modulestore(), CourseUserInfo(root_usage_key.course_key, user), root_usage_key, transformers,
)
def clear_course_from_cache(course_key):
course_usage_key = modulestore().make_course_usage_key(course_key)
return clear_block_cache(_get_cache(), course_usage_key)
"""
...
"""
# TODO 8874: Write docstrings
# TODO 8874: Think about encoding of cache (compression? pickle vs. JSON?). David Baumgold wants us to consider using JSON.
"""
...
"""
from block_structure import BlockStructureFactory
from transformer import BlockStructureTransformers
def get_blocks(cache, modulestore, user_info, root_block_key, transformers):
if not BlockStructureTransformers.are_all_registered(transformers):
raise Exception("One or more requested transformers are not registered.")
# Load the cached block structure.
root_block_structure = BlockStructureFactory.create_from_cache(root_block_key, cache)
if not root_block_structure:
# Create the block structure from the modulestore
root_block_structure = BlockStructureFactory.create_from_modulestore(root_block_key, modulestore)
# Collect data from each registered transformer
for transformer in BlockStructureTransformers.get_registered_transformers():
root_block_structure.add_transformer(transformer)
transformer.collect(root_block_structure)
# Collect all fields that were requested by the transformers
root_block_structure.collect_requested_xblock_fields()
# Cache this information
BlockStructureFactory.serialize_to_cache(root_block_structure, cache)
# Execute requested transforms on block structure
for transformer in transformers:
transformer.transform(user_info, root_block_structure)
# Prune block structure
root_block_structure.prune()
return root_block_structure
def clear_block_cache(cache, root_block_key):
BlockStructureFactory.remove_from_cache(root_block_key, cache)
"""
...
"""
from collections import defaultdict
from graph_traversals import traverse_topologically, traverse_post_order
from logging import getLogger
from transformer import BlockStructureTransformers
logger = getLogger(__name__) # pylint: disable=C0103
TRANSFORMER_VERSION_KEY = '_version'
class BlockStructure(object):
"""
A class to encapsulate a structure of blocks, a directed acyclic graph of blocks.
"""
class BlockRelations(object):
def __init__(self):
self.parents = []
self.children = []
def __init__(self, root_block_key):
self.root_block_key = root_block_key
self._block_relations = defaultdict(self.BlockRelations)
self._add_block(self._block_relations, root_block_key)
def add_relation(self, parent_key, child_key):
self._add_relation(self._block_relations, parent_key, child_key)
def get_parents(self, usage_key):
return self._block_relations.get(usage_key).parents if self.has_block(usage_key) else []
def get_children(self, usage_key):
return self._block_relations[usage_key].children if self.has_block(usage_key) else []
def has_block(self, usage_key):
return usage_key in self._block_relations
def get_block_keys(self):
return self._block_relations.iterkeys()
def topological_traversal(self, get_result=None, predicate=None):
return traverse_topologically(
start_node=self.root_block_key,
get_parents=self.get_parents,
get_children=self.get_children,
get_result=get_result,
predicate=predicate,
)
def prune(self):
# create a new block relations map with only those blocks that are still linked
pruned_block_relations = defaultdict(self.BlockRelations)
old_block_relations = self._block_relations
# def do_for_each_block(block_key):
# if block_key in old_block_relations:
# self._add_block(pruned_block_relations, block_key)
#
# for parent in old_block_relations[block_key].parents:
# if parent in pruned_block_relations:
# self._add_relation(pruned_block_relations, parent, block_key)
def do_for_each_block(block_key):
if block_key in old_block_relations:
self._add_block(pruned_block_relations, block_key)
for child in old_block_relations[block_key].children:
if child in pruned_block_relations:
self._add_relation(pruned_block_relations, block_key, child)
list(traverse_post_order(
start_node=self.root_block_key,
get_children=self.get_children,
get_result=do_for_each_block
))
self._block_relations = pruned_block_relations
@classmethod
def _add_relation(cls, block_relations, parent_key, child_key):
block_relations[child_key].parents.append(parent_key)
block_relations[parent_key].children.append(child_key)
@classmethod
def _add_block(cls, block_relations, block_key):
_ = block_relations[block_key]
class BlockStructureBlockData(BlockStructure):
"""
A sub-class of BlockStructure that encapsulates data captured about the blocks.
"""
class BlockData(object):
def __init__(self):
# dictionary mapping xblock field names to their values.
self._xblock_fields = {}
# dictionary mapping transformers' IDs to their collected data.
self._transformer_data = defaultdict(dict)
def __init__(self, root_block_key):
super(BlockStructureBlockData, self).__init__(root_block_key)
# dictionary mapping usage keys to BlockData
self._block_data_map = defaultdict(self.BlockData)
# dictionary mapping transformer IDs to block-structure-wide transformer data
self._transformer_data = defaultdict(dict)
def get_xblock_field(self, usage_key, field_name, default=None):
block_data = self._block_data_map.get(usage_key)
return block_data._xblock_fields.get(field_name, default) if block_data else default
def get_transformer_data(self, transformer, key, default=None):
return self._transformer_data.get(transformer.name(), {}).get(key, default)
def get_transformer_data_version(self, transformer):
return self.get_transformer_data(transformer, TRANSFORMER_VERSION_KEY, 0)
def get_transformer_block_data(self, usage_key, transformer, key, default=None):
block_data = self._block_data_map.get(usage_key)
return block_data._transformer_data.get(
transformer.name(), {}
).get(key, default) if block_data else default
def remove_block(self, usage_key):
# Remove block from its children.
for child in self._block_relations[usage_key].children:
self._block_relations[child].parents.remove(usage_key)
# Remove block from its parents.
for parent_key in self._block_relations[usage_key].parents:
self._block_relations[parent_key].children.remove(usage_key)
# Remove block.
if usage_key in self._block_relations:
del self._block_relations[usage_key]
if usage_key in self._block_data_map:
del self._block_data_map[usage_key]
def remove_block_if(self, removal_condition):
def predicate(block_key):
if removal_condition(block_key):
self.remove_block(block_key)
return False
return True
list(self.topological_traversal(predicate=predicate))
class BlockStructureCollectedData(BlockStructureBlockData):
"""
A sub-class of BlockStructure that encapsulates information about the blocks during the collect phase.
"""
def __init__(self, root_block_key):
super(BlockStructureCollectedData, self).__init__(root_block_key)
self._xblock_map = {} # dict[UsageKey: XBlock]
self._requested_xblock_fields = set()
def request_xblock_fields(self, *field_names):
self._requested_xblock_fields.update(set(field_names))
def collect_requested_xblock_fields(self):
if not self._requested_xblock_fields:
return
for xblock in self._xblock_map.itervalues():
for field_name in self._requested_xblock_fields:
self._set_xblock_field(xblock, field_name)
def _set_xblock_field(self, xblock, field_name):
if hasattr(xblock, field_name):
self._block_data_map[xblock.location]._xblock_fields[field_name] = getattr(xblock, field_name)
def add_xblock(self, xblock):
self._xblock_map[xblock.location] = xblock
def get_xblock(self, usage_key):
return self._xblock_map[usage_key]
def add_transformer(self, transformer):
if transformer.VERSION == 0:
raise Exception('VERSION attribute is not set on transformer {0}.', transformer.name())
self.set_transformer_data(transformer, TRANSFORMER_VERSION_KEY, transformer.VERSION)
def set_transformer_data(self, transformer, key, value):
self._transformer_data[transformer.name()][key] = value
def set_transformer_block_data(self, usage_key, transformer, key, value):
self._block_data_map[usage_key]._transformer_data[transformer.name()][key] = value
class BlockStructureFactory(object):
@classmethod
def create_from_modulestore(cls, root_block_key, modulestore):
block_structure = BlockStructureCollectedData(root_block_key)
def build_block_structure(xblock):
"""
Helper function to recursively walk block structure
"""
block_structure.add_xblock(xblock)
for child in xblock.get_children():
block_structure.add_relation(xblock.location, child.location)
build_block_structure(child)
root_xblock = modulestore.get_item(root_block_key, depth=None)
build_block_structure(root_xblock)
return block_structure
@classmethod
def serialize_to_cache(cls, block_structure, cache):
cache.set(
cls._encode_root_cache_key(block_structure.root_block_key),
(block_structure._block_relations, block_structure._transformer_data)
)
cache.set_many({
unicode(usage_key): block_data
for usage_key, block_data
in block_structure._block_data_map.iteritems()
})
@classmethod
def create_from_cache(cls, root_block_key, cache):
"""
Returns:
BlockStructure, if the block structure is in the cache, and
NoneType otherwise.
"""
block_relations, transformer_data = cache.get(cls._encode_root_cache_key(root_block_key), (None, None))
if block_relations:
block_structure = BlockStructureBlockData(root_block_key)
block_structure._block_relations = block_relations
block_structure._transformer_data = transformer_data
if all(
transformer.VERSION == block_structure.get_transformer_data_version(transformer)
for transformer in BlockStructureTransformers.get_registered_transformers()
):
block_structure._block_data_map = cache.get_many(block_relations.iterkeys())
return block_structure
else:
logger.info("Collected data for transformer is outdated.")
return None
@classmethod
def remove_from_cache(cls, root_block_key, cache):
cache.delete(cls._encode_root_cache_key(root_block_key))
# TODO also remove all block data?
@classmethod
def _encode_root_cache_key(cls, root_block_key):
return "root.key." + unicode(root_block_key)
"""
...
"""
from collections import deque
def _traverse_generic(start_node, get_parents, get_children, get_result=None, predicate=None):
"""
Helper function to avoid duplicating functionality between
traverse_depth_first and traverse_topologically.
If get_parents is None, do a depth first traversal.
Else, do a topological traversal.
The topological traversal has a worse time complexity than depth-first does,
as it needs to check whether each node's parents have been visited.
Arguments:
start_node - the starting node for the traversal
get_parents - function that returns a list of parent nodes for the given node
get_children - function that returns a list of children nodes for the given node
get_result - function that computes and returns the resulting value to be yielded for the given node
predicate - function that returns whether or not to yield the given node
"""
# If get_result or predicate aren't provided, just make them to no-ops.
get_result = get_result or (lambda node_: node_)
predicate = predicate or (lambda __: True)
# For our stack, we use the deque type, which is O(1) for pop and append.
stack = deque([start_node])
yield_results = {}
# While there are more nodes on the stack...
while stack:
# Take a node off the top of the stack.
curr_node = stack.pop()
# If we're doing a topological traversal, then make sure all the node's
# parents have been visited. If they haven't, then skip the node for
# now; we'll encounter it again later through another one of its
# parents.
if get_parents and curr_node != start_node:
parents = get_parents(curr_node)
all_parents_visited = all(parent in yield_results for parent in parents)
any_parent_yielded = any(yield_results[parent] for parent in parents) if all_parents_visited else False
if not all_parents_visited or not any_parent_yielded:
continue
# Add its unvisited children to the stack in reverse order so that
# they are popped off in their original order.
# It's important that we visit the children even if the parent isn't yielded
# in case a child has multiple parents and this is its last parent.
unvisited_children = get_children(curr_node)
# If we're not doing a topological traversal, check whether the child has been visited.
if not get_parents:
unvisited_children = list(
child
for child in unvisited_children
if child not in yield_results
)
unvisited_children.reverse()
stack.extend(unvisited_children)
# Return the result if it satisfies the predicate.
# Keep track of the result so we know whether to yield its children.
should_yield_node = predicate(curr_node)
yield_results[curr_node] = should_yield_node
if should_yield_node:
yield get_result(curr_node)
def traverse_topologically(start_node, get_parents, get_children, get_result=None, predicate=None):
return _traverse_generic(
start_node,
get_parents=get_parents,
get_children=get_children,
get_result=get_result,
predicate=predicate
)
def traverse_pre_order(start_node, get_children, get_result=None, predicate=None):
return _traverse_generic(
start_node,
get_parents=None,
get_children=get_children,
get_result=get_result,
predicate=predicate
)
class BlockAndChildIndexStackItem(object):
"""
Class for items in the stack.
"""
def __init__(self, block):
self.block = block
self.children = None
self.child_index = 0
def next_child(self, get_children):
"""
Returns the next child of the block for this item in the stack.
"""
if self.children is None:
self.children = get_children(self.block)
child = None
if self.child_index < len(self.children):
child = self.children[self.child_index]
self.child_index += 1
return child
def traverse_post_order(start_node, get_children, get_result=None, predicate=None):
# If get_result or predicate aren't provided, just make them to no-ops.
get_result = get_result or (lambda node_: node_)
predicate = predicate or (lambda __: True)
# For our stack, we use the deque type, which is O(1) for pop and append.
stack = deque([BlockAndChildIndexStackItem(start_node)])
visited = set()
while stack:
# peek at the next item in the stack
current_stack_item = stack[len(stack)-1]
# verify the block wasn't already visited and the block satisfies the predicate
if current_stack_item.block in visited or not predicate(current_stack_item.block):
stack.pop()
continue
next_child = current_stack_item.next_child(get_children)
if next_child:
stack.append(BlockAndChildIndexStackItem(next_child))
else:
yield get_result(current_stack_item.block)
visited.add(current_stack_item.block)
stack.pop()
"""
Tests for block_cache.py
"""
from mock import patch
from unittest import TestCase
from .test_utils import (
MockModulestoreFactory, MockCache, MockUserInfo, MockTransformer, SIMPLE_CHILDREN_MAP, BlockStructureTestMixin
)
from ..block_cache import get_blocks
class TestBlockCache(TestCase, BlockStructureTestMixin):
class TestTransformer1(MockTransformer):
@classmethod
def block_key(cls):
return 't1.key1'
@classmethod
def block_val(cls, block_key):
return 't1.val1.' + unicode(block_key)
@classmethod
def collect(self, block_structure):
list(
block_structure.topological_traversal(
get_result=lambda block_key: block_structure.set_transformer_block_data(
block_key, self, self.block_key(), self.block_val(block_key)
)))
def transform(self, user_info, block_structure):
def assert_collected_value(block_key):
assert (
block_structure.get_transformer_block_data(
block_key,
self,
self.block_key()
) == self.block_val(block_key)
)
list(
block_structure.topological_traversal(
get_result=lambda block_key: assert_collected_value(block_key)
))
@patch('openedx.core.lib.block_cache.transformer.BlockStructureTransformers.get_available_plugins')
def test_get_blocks(self, mock_available_transforms):
children_map = SIMPLE_CHILDREN_MAP
cache = MockCache()
user_info = MockUserInfo()
modulestore = MockModulestoreFactory.create(children_map)
transformers = [self.TestTransformer1()]
mock_available_transforms.return_value = {}
with self.assertRaisesRegexp(Exception, "requested transformers are not registered"):
get_blocks(cache, modulestore, user_info, root_block_key=0, transformers=transformers)
mock_available_transforms.return_value = {transformer.name(): transformer for transformer in transformers}
block_structure = get_blocks(cache, modulestore, user_info, root_block_key=0, transformers=transformers)
self.verify_block_structure(block_structure, children_map)
"""
Tests for block_structure.py
"""
from collections import namedtuple
import ddt
from mock import patch
from unittest import TestCase
from ..block_structure import (
BlockStructure, BlockStructureCollectedData, BlockStructureBlockData, BlockStructureFactory
)
from ..transformer import BlockStructureTransformer, BlockStructureTransformers
from .test_utils import (
MockCache, MockXBlock, MockModulestoreFactory, MockTransformer, SIMPLE_CHILDREN_MAP, BlockStructureTestMixin
)
@ddt.ddt
class TestBlockStructure(TestCase):
"""
Tests for BlockStructure
"""
def get_parents_map(self, children_map):
parent_map = [[] for node in children_map]
for parent, children in enumerate(children_map):
for child in children:
parent_map[child].append(parent)
return parent_map
@ddt.data(
[],
# 0
# / \
# 1 2
# / \
# 3 4
SIMPLE_CHILDREN_MAP,
# 0
# /
# 1
# /
# 2
# /
# 3
[[1], [2], [3], []],
# 0
# / \
# 1 2
# \ /
# 3
[[1, 2], [3], [3], []],
)
def test_relations(self, children_map):
# create block structure
block_structure = BlockStructure(root_block_key=0)
# add_relation
for parent, children in enumerate(children_map):
for child in children:
block_structure.add_relation(parent, child)
# get_children
for parent, children in enumerate(children_map):
self.assertSetEqual(set(block_structure.get_children(parent)), set(children))
# get_parents
for child, parents in enumerate(self.get_parents_map(children_map)):
self.assertSetEqual(set(block_structure.get_parents(child)), set(parents))
# has_block
for node in range(len(children_map)):
self.assertTrue(block_structure.has_block(node))
self.assertFalse(block_structure.has_block(len(children_map) + 1))
class TestBlockStructureData(TestCase):
"""
Tests for BlockStructureBlockData and BlockStructureCollectedData
"""
def test_non_versioned_transformer(self):
class TestNonVersionedTransformer(BlockStructureTransformer):
def transform(self, user_info, block_structure):
pass
block_structure = BlockStructureCollectedData(root_block_key=0)
with self.assertRaisesRegexp(Exception, "VERSION attribute is not set"):
block_structure.add_transformer(TestNonVersionedTransformer())
def test_transformer_data(self):
# transformer test cases
TransformerInfo = namedtuple("TransformerInfo", "transformer structure_wide_data block_specific_data")
transformers_info = [
TransformerInfo(
transformer=MockTransformer(),
structure_wide_data=[("t1.global1", "t1.g.val1"), ("t1.global2", "t1.g.val2"),],
block_specific_data={
"B1": [("t1.key1", "t1.b1.val1"), ("t1.key2", "t1.b1.val2")],
"B2": [("t1.key1", "t1.b2.val1"), ("t1.key2", "t1.b2.val2")],
"B3": [("t1.key1", True), ("t1.key2", False)],
"B4": [("t1.key1", None), ("t1.key2", False)],
},
),
TransformerInfo(
transformer=MockTransformer(),
structure_wide_data=[("t2.global1", "t2.g.val1"), ("t2.global2", "t2.g.val2"),],
block_specific_data={
"B1": [("t2.key1", "t2.b1.val1"), ("t2.key2", "t2.b1.val2")],
"B2": [("t2.key1", "t2.b2.val1"), ("t2.key2", "t2.b2.val2")],
},
),
]
# create block structure
block_structure = BlockStructureCollectedData(root_block_key=0)
# set transformer data
for t_info in transformers_info:
block_structure.add_transformer(t_info.transformer)
for key, val in t_info.structure_wide_data:
block_structure.set_transformer_data(t_info.transformer, key, val)
for block, block_data in t_info.block_specific_data.iteritems():
for key, val in block_data:
block_structure.set_transformer_block_data(block, t_info.transformer, key, val)
# verify transformer data
for t_info in transformers_info:
self.assertEquals(
block_structure.get_transformer_data_version(t_info.transformer),
MockTransformer.VERSION
)
for key, val in t_info.structure_wide_data:
self.assertEquals(
block_structure.get_transformer_data(t_info.transformer, key),
val,
)
for block, block_data in t_info.block_specific_data.iteritems():
for key, val in block_data:
self.assertEquals(
block_structure.get_transformer_block_data(block, t_info.transformer, key),
val,
)
def test_xblock_data(self):
# block test cases
blocks = [
MockXBlock("A", {}),
MockXBlock("B", {"field1": "B.val1"}),
MockXBlock("C", {"field1": "C.val1", "field2": "C.val2"}),
MockXBlock("D", {"field1": True, "field2": False}),
MockXBlock("E", {"field1": None, "field2": False}),
]
# add each block
block_structure = BlockStructureCollectedData(root_block_key=0)
for block in blocks:
block_structure.add_xblock(block)
# request fields
fields = ["field1", "field2", "field3"]
block_structure.request_xblock_fields(*fields)
# verify fields have not been collected yet
for block in blocks:
for field in fields:
self.assertIsNone(block_structure.get_xblock_field(block.location, field))
# collect fields
block_structure.collect_requested_xblock_fields()
# verify values of collected fields
for block in blocks:
for field in fields:
self.assertEquals(
block_structure.get_xblock_field(block.location, field),
block.field_map.get(field),
)
def test_remove_block(self):
block_structure = BlockStructureBlockData(root_block_key=0)
for parent, children in enumerate(SIMPLE_CHILDREN_MAP):
for child in children:
block_structure.add_relation(parent, child)
self.assertTrue(block_structure.has_block(1))
self.assertTrue(1 in block_structure.get_children(0))
block_structure.remove_block(1)
self.assertFalse(block_structure.has_block(1))
self.assertFalse(1 in block_structure.get_children(0))
self.assertTrue(block_structure.has_block(3))
self.assertTrue(block_structure.has_block(4))
block_structure.prune()
self.assertFalse(block_structure.has_block(3))
self.assertFalse(block_structure.has_block(4))
class TestBlockStructureFactory(TestCase, BlockStructureTestMixin):
"""
Tests for BlockStructureFactory
"""
def test_factory_methods(self):
children_map = SIMPLE_CHILDREN_MAP
modulestore = MockModulestoreFactory.create(children_map)
cache = MockCache()
# test create from modulestore
block_structure = BlockStructureFactory.create_from_modulestore(root_block_key=0, modulestore=modulestore)
self.verify_block_structure(block_structure, children_map)
# test not in cache
self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
# test transformers outdated
BlockStructureFactory.serialize_to_cache(block_structure, cache)
with patch('openedx.core.lib.block_cache.block_structure.logger.info') as mock_logger:
self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
self.assertTrue(mock_logger.called)
# update transformers
for transformer in BlockStructureTransformers.get_registered_transformers():
block_structure.add_transformer(transformer)
block_structure.set_transformer_block_data(
usage_key=0, transformer=transformer, key='test', value='test.val'
)
BlockStructureFactory.serialize_to_cache(block_structure, cache)
# test re-create from cache
from_cache_block_structure = BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache)
self.assertIsNotNone(from_cache_block_structure)
self.verify_block_structure(from_cache_block_structure, children_map)
# test remove from cache
BlockStructureFactory.remove_from_cache(root_block_key=0, cache=cache)
self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
"""
...
"""
# TODO 8874: Test graph_traversals more comprehensively.
from collections import defaultdict
from unittest import TestCase
from ..graph_traversals import (
traverse_pre_order, traverse_post_order, traverse_topologically
)
class GraphTraversalsTestCase(TestCase):
"""
...
"""
def setUp(self):
"""
...
"""
# b1
# / \
# c1 c2
# / \ \
# d1 d2 d3
# \ / \
# e1 e2
# |
# f1
super(GraphTraversalsTestCase, self).setUp()
self.graph_1 = {
'a1': ['b1'],
'a2': ['b2'],
'b1': ['c1', 'c2'],
'b2': [],
'c1': ['d1', 'd2'],
'c2': ['d3'],
'd1': ['e1'],
'd2': ['e1', 'e2'],
'd3': [],
'e1': [],
'e2': ['f1'],
'f1': [],
}
self.graph_1_parents = self.get_parent_map(self.graph_1)
@staticmethod
def get_parent_map(graph):
"""
...
"""
result = defaultdict(list)
for parent, children in graph.iteritems():
for child in children:
result[child].append(parent)
return result
def test_pre_order(self):
"""
...
"""
self.assertEqual(
list(traverse_pre_order(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_result=(lambda node: node + '_'),
predicate=(lambda node: node != 'd3'),
)),
['b1_', 'c1_', 'd1_', 'e1_', 'd2_', 'e2_', 'f1_', 'c2_']
)
def test_post_order(self):
"""
...
"""
self.assertEqual(
list(traverse_post_order(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_result=(lambda node: node + '_'),
predicate=(lambda node: node != 'd3'),
)),
['e1_', 'd1_', 'f1_', 'e2_', 'd2_', 'c1_', 'c2_', 'b1_']
)
def test_topological(self):
"""
...
"""
self.assertEqual(
list(traverse_topologically(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_parents=(lambda node: self.graph_1_parents[node]),
predicate=(lambda node: node != 'd3'),
)),
['b1', 'c1', 'd1', 'd2', 'e1', 'e2', 'f1', 'c2']
)
def test_topological_with_predicate(self):
"""
...
"""
self.assertEqual(
list(traverse_topologically(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_parents=(lambda node: self.graph_1_parents[node]),
predicate=(lambda node: node != 'd2')
)),
['b1', 'c1', 'd1', 'e1', 'c2', 'd3']
)
"""
Tests for transformer.py
"""
from mock import patch
from unittest import TestCase
from ..transformer import BlockStructureTransformers
from .test_utils import MockTransformer
class BlockStructureTransformersTestCase(TestCase):
"""
Test cases for BlockStructureTransformers.
"""
class TestTransformer1(MockTransformer):
pass
class TestTransformer2(MockTransformer):
pass
class UnregisteredTestTransformer3(MockTransformer):
pass
@patch('openedx.core.lib.block_cache.transformer.BlockStructureTransformers.get_available_plugins')
def test_are_all_registered(self, mock_available_transforms):
mock_available_transforms.return_value = {
transformer.name(): transformer
for transformer in [self.TestTransformer1, self.TestTransformer2]
}
for transformers, expected_are_all_registered in [
([], True),
([self.TestTransformer1()], True),
([self.TestTransformer1(), self.TestTransformer2()], True),
([self.UnregisteredTestTransformer3()], False),
([self.TestTransformer1(), self.UnregisteredTestTransformer3()], False),
]:
self.assertEquals(BlockStructureTransformers.are_all_registered(transformers), expected_are_all_registered)
"""
Common utilities for tests in block_cache module
"""
from ..transformer import BlockStructureTransformer
class MockXBlock(object):
def __init__(self, location, field_map=None, children=None, modulestore=None):
self.location = location
self.field_map = field_map or {}
self.children = children or []
self.modulestore = modulestore
def __getattr__(self, attr):
try:
return self.field_map[attr]
except KeyError:
raise AttributeError
def get_children(self):
return [self.modulestore.get_item(child) for child in self.children]
class MockModulestore(object):
def set_blocks(self, blocks):
self.blocks = blocks
def get_item(self, block_key, depth=None):
return self.blocks.get(block_key)
class MockCache(object):
def __init__(self):
self.map = {}
def set(self, key, val):
self.map[key] = val
def get(self, key, default):
return self.map.get(key, default)
def set_many(self, map):
for key, val in map.iteritems():
self.set(key, val)
def get_many(self, keys):
return {key: self.map[key] for key in keys if key in self.map}
def delete(self, key):
del self.map[key]
class MockModulestoreFactory(object):
@classmethod
def create(cls, children_map):
modulestore = MockModulestore()
modulestore.set_blocks({
block_key: MockXBlock(block_key, children=children, modulestore=modulestore)
for block_key, children in enumerate(children_map)
})
return modulestore
class MockUserInfo(object):
def has_staff_access(self):
return False
class MockTransformer(BlockStructureTransformer):
VERSION = 1
def transform(self, user_info, block_structure):
pass
# 0
# / \
# 1 2
# / \
# 3 4
SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
class BlockStructureTestMixin(object):
def verify_block_structure(self, block_structure, children_map):
for block_key, children in enumerate(children_map):
self.assertTrue(
block_structure.has_block(block_key)
)
self.assertEquals(
set(block_structure.get_children(block_key)),
set(children),
)
"""
...
"""
from abc import abstractmethod
from openedx.core.lib.api.plugins import PluginManager
class BlockStructureTransformer(object):
"""
...
"""
# All Transformers are expected to update and maintain a VERSION class attribute
VERSION = 0
@classmethod
def name(cls):
return cls.__name__
@classmethod
def collect(self, block_structure):
"""
Collects any information that's necessary to execute this transformer's
transform method.
"""
pass
@abstractmethod
def transform(self, user_info, block_structure):
"""
Mutates block_structure based on the given user_info.
"""
pass
class BlockStructureTransformers(PluginManager):
"""
Manager for all of the block structure transformers that have been made available.
All block structure transformers should implement `BlockStructureTransformer`.
"""
NAMESPACE = 'openedx.block_structure_transformer'
@classmethod
def get_registered_transformers(cls):
return set(cls.get_available_plugins().itervalues())
@classmethod
def are_all_registered(cls, transformers):
registered_transformers = cls.get_registered_transformers()
return all(
any(transformer.name() == reg_trans.name() for reg_trans in registered_transformers)
for transformer in transformers
)
"""
...
"""
from abc import abstractproperty
class UserInfo(object):
"""
...
"""
@abstractproperty
def has_staff_access(self):
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment