Commit eb88d9da by Nimisha Asthagiri

Block Cache Framework

parent d3f39b2f
"""
...
"""
from django.core.cache import get_cache
from openedx.core.lib.block_cache.block_cache import get_blocks, clear_block_cache
from xmodule.modulestore.django import modulestore
from .transformers import (
start_date,
user_partitions,
visibility,
)
from .user_info import CourseUserInfo
LMS_COURSE_TRANSFORMERS = [
visibility.VisibilityTransformer(),
start_date.StartDateTransformer(),
user_partitions.UserPartitionTransformer(),
]
_cache = None
def _get_cache():
global _cache
if not _cache:
_cache = get_cache('lms.course_blocks')
return _cache
def get_course_blocks(
user,
root_usage_key,
transformers=LMS_COURSE_TRANSFORMERS,
):
if transformers is None:
transformers = LMS_COURSE_TRANSFORMERS
return get_blocks(
_get_cache(), modulestore(), CourseUserInfo(root_usage_key.course_key, user), root_usage_key, transformers,
)
def clear_course_from_cache(course_key):
course_usage_key = modulestore().make_course_usage_key(course_key)
return clear_block_cache(_get_cache(), course_usage_key)
"""
...
"""
# TODO 8874: Write docstrings
# TODO 8874: Think about encoding of cache (compression? pickle vs. JSON?). David Baumgold wants us to consider using JSON.
"""
...
"""
from block_structure import BlockStructureFactory
from transformer import BlockStructureTransformers
def get_blocks(cache, modulestore, user_info, root_block_key, transformers):
if not BlockStructureTransformers.are_all_registered(transformers):
raise Exception("One or more requested transformers are not registered.")
# Load the cached block structure.
root_block_structure = BlockStructureFactory.create_from_cache(root_block_key, cache)
if not root_block_structure:
# Create the block structure from the modulestore
root_block_structure = BlockStructureFactory.create_from_modulestore(root_block_key, modulestore)
# Collect data from each registered transformer
for transformer in BlockStructureTransformers.get_registered_transformers():
root_block_structure.add_transformer(transformer)
transformer.collect(root_block_structure)
# Collect all fields that were requested by the transformers
root_block_structure.collect_requested_xblock_fields()
# Cache this information
BlockStructureFactory.serialize_to_cache(root_block_structure, cache)
# Execute requested transforms on block structure
for transformer in transformers:
transformer.transform(user_info, root_block_structure)
# Prune block structure
root_block_structure.prune()
return root_block_structure
def clear_block_cache(cache, root_block_key):
BlockStructureFactory.remove_from_cache(root_block_key, cache)
"""
...
"""
from collections import deque
def _traverse_generic(start_node, get_parents, get_children, get_result=None, predicate=None):
"""
Helper function to avoid duplicating functionality between
traverse_depth_first and traverse_topologically.
If get_parents is None, do a depth first traversal.
Else, do a topological traversal.
The topological traversal has a worse time complexity than depth-first does,
as it needs to check whether each node's parents have been visited.
Arguments:
start_node - the starting node for the traversal
get_parents - function that returns a list of parent nodes for the given node
get_children - function that returns a list of children nodes for the given node
get_result - function that computes and returns the resulting value to be yielded for the given node
predicate - function that returns whether or not to yield the given node
"""
# If get_result or predicate aren't provided, just make them to no-ops.
get_result = get_result or (lambda node_: node_)
predicate = predicate or (lambda __: True)
# For our stack, we use the deque type, which is O(1) for pop and append.
stack = deque([start_node])
yield_results = {}
# While there are more nodes on the stack...
while stack:
# Take a node off the top of the stack.
curr_node = stack.pop()
# If we're doing a topological traversal, then make sure all the node's
# parents have been visited. If they haven't, then skip the node for
# now; we'll encounter it again later through another one of its
# parents.
if get_parents and curr_node != start_node:
parents = get_parents(curr_node)
all_parents_visited = all(parent in yield_results for parent in parents)
any_parent_yielded = any(yield_results[parent] for parent in parents) if all_parents_visited else False
if not all_parents_visited or not any_parent_yielded:
continue
# Add its unvisited children to the stack in reverse order so that
# they are popped off in their original order.
# It's important that we visit the children even if the parent isn't yielded
# in case a child has multiple parents and this is its last parent.
unvisited_children = get_children(curr_node)
# If we're not doing a topological traversal, check whether the child has been visited.
if not get_parents:
unvisited_children = list(
child
for child in unvisited_children
if child not in yield_results
)
unvisited_children.reverse()
stack.extend(unvisited_children)
# Return the result if it satisfies the predicate.
# Keep track of the result so we know whether to yield its children.
should_yield_node = predicate(curr_node)
yield_results[curr_node] = should_yield_node
if should_yield_node:
yield get_result(curr_node)
def traverse_topologically(start_node, get_parents, get_children, get_result=None, predicate=None):
return _traverse_generic(
start_node,
get_parents=get_parents,
get_children=get_children,
get_result=get_result,
predicate=predicate
)
def traverse_pre_order(start_node, get_children, get_result=None, predicate=None):
return _traverse_generic(
start_node,
get_parents=None,
get_children=get_children,
get_result=get_result,
predicate=predicate
)
class BlockAndChildIndexStackItem(object):
"""
Class for items in the stack.
"""
def __init__(self, block):
self.block = block
self.children = None
self.child_index = 0
def next_child(self, get_children):
"""
Returns the next child of the block for this item in the stack.
"""
if self.children is None:
self.children = get_children(self.block)
child = None
if self.child_index < len(self.children):
child = self.children[self.child_index]
self.child_index += 1
return child
def traverse_post_order(start_node, get_children, get_result=None, predicate=None):
# If get_result or predicate aren't provided, just make them to no-ops.
get_result = get_result or (lambda node_: node_)
predicate = predicate or (lambda __: True)
# For our stack, we use the deque type, which is O(1) for pop and append.
stack = deque([BlockAndChildIndexStackItem(start_node)])
visited = set()
while stack:
# peek at the next item in the stack
current_stack_item = stack[len(stack)-1]
# verify the block wasn't already visited and the block satisfies the predicate
if current_stack_item.block in visited or not predicate(current_stack_item.block):
stack.pop()
continue
next_child = current_stack_item.next_child(get_children)
if next_child:
stack.append(BlockAndChildIndexStackItem(next_child))
else:
yield get_result(current_stack_item.block)
visited.add(current_stack_item.block)
stack.pop()
"""
Tests for block_cache.py
"""
from mock import patch
from unittest import TestCase
from .test_utils import (
MockModulestoreFactory, MockCache, MockUserInfo, MockTransformer, SIMPLE_CHILDREN_MAP, BlockStructureTestMixin
)
from ..block_cache import get_blocks
class TestBlockCache(TestCase, BlockStructureTestMixin):
class TestTransformer1(MockTransformer):
@classmethod
def block_key(cls):
return 't1.key1'
@classmethod
def block_val(cls, block_key):
return 't1.val1.' + unicode(block_key)
@classmethod
def collect(self, block_structure):
list(
block_structure.topological_traversal(
get_result=lambda block_key: block_structure.set_transformer_block_data(
block_key, self, self.block_key(), self.block_val(block_key)
)))
def transform(self, user_info, block_structure):
def assert_collected_value(block_key):
assert (
block_structure.get_transformer_block_data(
block_key,
self,
self.block_key()
) == self.block_val(block_key)
)
list(
block_structure.topological_traversal(
get_result=lambda block_key: assert_collected_value(block_key)
))
@patch('openedx.core.lib.block_cache.transformer.BlockStructureTransformers.get_available_plugins')
def test_get_blocks(self, mock_available_transforms):
children_map = SIMPLE_CHILDREN_MAP
cache = MockCache()
user_info = MockUserInfo()
modulestore = MockModulestoreFactory.create(children_map)
transformers = [self.TestTransformer1()]
mock_available_transforms.return_value = {}
with self.assertRaisesRegexp(Exception, "requested transformers are not registered"):
get_blocks(cache, modulestore, user_info, root_block_key=0, transformers=transformers)
mock_available_transforms.return_value = {transformer.name(): transformer for transformer in transformers}
block_structure = get_blocks(cache, modulestore, user_info, root_block_key=0, transformers=transformers)
self.verify_block_structure(block_structure, children_map)
"""
Tests for block_structure.py
"""
from collections import namedtuple
import ddt
from mock import patch
from unittest import TestCase
from ..block_structure import (
BlockStructure, BlockStructureCollectedData, BlockStructureBlockData, BlockStructureFactory
)
from ..transformer import BlockStructureTransformer, BlockStructureTransformers
from .test_utils import (
MockCache, MockXBlock, MockModulestoreFactory, MockTransformer, SIMPLE_CHILDREN_MAP, BlockStructureTestMixin
)
@ddt.ddt
class TestBlockStructure(TestCase):
"""
Tests for BlockStructure
"""
def get_parents_map(self, children_map):
parent_map = [[] for node in children_map]
for parent, children in enumerate(children_map):
for child in children:
parent_map[child].append(parent)
return parent_map
@ddt.data(
[],
# 0
# / \
# 1 2
# / \
# 3 4
SIMPLE_CHILDREN_MAP,
# 0
# /
# 1
# /
# 2
# /
# 3
[[1], [2], [3], []],
# 0
# / \
# 1 2
# \ /
# 3
[[1, 2], [3], [3], []],
)
def test_relations(self, children_map):
# create block structure
block_structure = BlockStructure(root_block_key=0)
# add_relation
for parent, children in enumerate(children_map):
for child in children:
block_structure.add_relation(parent, child)
# get_children
for parent, children in enumerate(children_map):
self.assertSetEqual(set(block_structure.get_children(parent)), set(children))
# get_parents
for child, parents in enumerate(self.get_parents_map(children_map)):
self.assertSetEqual(set(block_structure.get_parents(child)), set(parents))
# has_block
for node in range(len(children_map)):
self.assertTrue(block_structure.has_block(node))
self.assertFalse(block_structure.has_block(len(children_map) + 1))
class TestBlockStructureData(TestCase):
"""
Tests for BlockStructureBlockData and BlockStructureCollectedData
"""
def test_non_versioned_transformer(self):
class TestNonVersionedTransformer(BlockStructureTransformer):
def transform(self, user_info, block_structure):
pass
block_structure = BlockStructureCollectedData(root_block_key=0)
with self.assertRaisesRegexp(Exception, "VERSION attribute is not set"):
block_structure.add_transformer(TestNonVersionedTransformer())
def test_transformer_data(self):
# transformer test cases
TransformerInfo = namedtuple("TransformerInfo", "transformer structure_wide_data block_specific_data")
transformers_info = [
TransformerInfo(
transformer=MockTransformer(),
structure_wide_data=[("t1.global1", "t1.g.val1"), ("t1.global2", "t1.g.val2"),],
block_specific_data={
"B1": [("t1.key1", "t1.b1.val1"), ("t1.key2", "t1.b1.val2")],
"B2": [("t1.key1", "t1.b2.val1"), ("t1.key2", "t1.b2.val2")],
"B3": [("t1.key1", True), ("t1.key2", False)],
"B4": [("t1.key1", None), ("t1.key2", False)],
},
),
TransformerInfo(
transformer=MockTransformer(),
structure_wide_data=[("t2.global1", "t2.g.val1"), ("t2.global2", "t2.g.val2"),],
block_specific_data={
"B1": [("t2.key1", "t2.b1.val1"), ("t2.key2", "t2.b1.val2")],
"B2": [("t2.key1", "t2.b2.val1"), ("t2.key2", "t2.b2.val2")],
},
),
]
# create block structure
block_structure = BlockStructureCollectedData(root_block_key=0)
# set transformer data
for t_info in transformers_info:
block_structure.add_transformer(t_info.transformer)
for key, val in t_info.structure_wide_data:
block_structure.set_transformer_data(t_info.transformer, key, val)
for block, block_data in t_info.block_specific_data.iteritems():
for key, val in block_data:
block_structure.set_transformer_block_data(block, t_info.transformer, key, val)
# verify transformer data
for t_info in transformers_info:
self.assertEquals(
block_structure.get_transformer_data_version(t_info.transformer),
MockTransformer.VERSION
)
for key, val in t_info.structure_wide_data:
self.assertEquals(
block_structure.get_transformer_data(t_info.transformer, key),
val,
)
for block, block_data in t_info.block_specific_data.iteritems():
for key, val in block_data:
self.assertEquals(
block_structure.get_transformer_block_data(block, t_info.transformer, key),
val,
)
def test_xblock_data(self):
# block test cases
blocks = [
MockXBlock("A", {}),
MockXBlock("B", {"field1": "B.val1"}),
MockXBlock("C", {"field1": "C.val1", "field2": "C.val2"}),
MockXBlock("D", {"field1": True, "field2": False}),
MockXBlock("E", {"field1": None, "field2": False}),
]
# add each block
block_structure = BlockStructureCollectedData(root_block_key=0)
for block in blocks:
block_structure.add_xblock(block)
# request fields
fields = ["field1", "field2", "field3"]
block_structure.request_xblock_fields(*fields)
# verify fields have not been collected yet
for block in blocks:
for field in fields:
self.assertIsNone(block_structure.get_xblock_field(block.location, field))
# collect fields
block_structure.collect_requested_xblock_fields()
# verify values of collected fields
for block in blocks:
for field in fields:
self.assertEquals(
block_structure.get_xblock_field(block.location, field),
block.field_map.get(field),
)
def test_remove_block(self):
block_structure = BlockStructureBlockData(root_block_key=0)
for parent, children in enumerate(SIMPLE_CHILDREN_MAP):
for child in children:
block_structure.add_relation(parent, child)
self.assertTrue(block_structure.has_block(1))
self.assertTrue(1 in block_structure.get_children(0))
block_structure.remove_block(1)
self.assertFalse(block_structure.has_block(1))
self.assertFalse(1 in block_structure.get_children(0))
self.assertTrue(block_structure.has_block(3))
self.assertTrue(block_structure.has_block(4))
block_structure.prune()
self.assertFalse(block_structure.has_block(3))
self.assertFalse(block_structure.has_block(4))
class TestBlockStructureFactory(TestCase, BlockStructureTestMixin):
"""
Tests for BlockStructureFactory
"""
def test_factory_methods(self):
children_map = SIMPLE_CHILDREN_MAP
modulestore = MockModulestoreFactory.create(children_map)
cache = MockCache()
# test create from modulestore
block_structure = BlockStructureFactory.create_from_modulestore(root_block_key=0, modulestore=modulestore)
self.verify_block_structure(block_structure, children_map)
# test not in cache
self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
# test transformers outdated
BlockStructureFactory.serialize_to_cache(block_structure, cache)
with patch('openedx.core.lib.block_cache.block_structure.logger.info') as mock_logger:
self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
self.assertTrue(mock_logger.called)
# update transformers
for transformer in BlockStructureTransformers.get_registered_transformers():
block_structure.add_transformer(transformer)
block_structure.set_transformer_block_data(
usage_key=0, transformer=transformer, key='test', value='test.val'
)
BlockStructureFactory.serialize_to_cache(block_structure, cache)
# test re-create from cache
from_cache_block_structure = BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache)
self.assertIsNotNone(from_cache_block_structure)
self.verify_block_structure(from_cache_block_structure, children_map)
# test remove from cache
BlockStructureFactory.remove_from_cache(root_block_key=0, cache=cache)
self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
"""
...
"""
# TODO 8874: Test graph_traversals more comprehensively.
from collections import defaultdict
from unittest import TestCase
from ..graph_traversals import (
traverse_pre_order, traverse_post_order, traverse_topologically
)
class GraphTraversalsTestCase(TestCase):
"""
...
"""
def setUp(self):
"""
...
"""
# b1
# / \
# c1 c2
# / \ \
# d1 d2 d3
# \ / \
# e1 e2
# |
# f1
super(GraphTraversalsTestCase, self).setUp()
self.graph_1 = {
'a1': ['b1'],
'a2': ['b2'],
'b1': ['c1', 'c2'],
'b2': [],
'c1': ['d1', 'd2'],
'c2': ['d3'],
'd1': ['e1'],
'd2': ['e1', 'e2'],
'd3': [],
'e1': [],
'e2': ['f1'],
'f1': [],
}
self.graph_1_parents = self.get_parent_map(self.graph_1)
@staticmethod
def get_parent_map(graph):
"""
...
"""
result = defaultdict(list)
for parent, children in graph.iteritems():
for child in children:
result[child].append(parent)
return result
def test_pre_order(self):
"""
...
"""
self.assertEqual(
list(traverse_pre_order(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_result=(lambda node: node + '_'),
predicate=(lambda node: node != 'd3'),
)),
['b1_', 'c1_', 'd1_', 'e1_', 'd2_', 'e2_', 'f1_', 'c2_']
)
def test_post_order(self):
"""
...
"""
self.assertEqual(
list(traverse_post_order(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_result=(lambda node: node + '_'),
predicate=(lambda node: node != 'd3'),
)),
['e1_', 'd1_', 'f1_', 'e2_', 'd2_', 'c1_', 'c2_', 'b1_']
)
def test_topological(self):
"""
...
"""
self.assertEqual(
list(traverse_topologically(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_parents=(lambda node: self.graph_1_parents[node]),
predicate=(lambda node: node != 'd3'),
)),
['b1', 'c1', 'd1', 'd2', 'e1', 'e2', 'f1', 'c2']
)
def test_topological_with_predicate(self):
"""
...
"""
self.assertEqual(
list(traverse_topologically(
start_node='b1',
get_children=(lambda node: self.graph_1[node]),
get_parents=(lambda node: self.graph_1_parents[node]),
predicate=(lambda node: node != 'd2')
)),
['b1', 'c1', 'd1', 'e1', 'c2', 'd3']
)
"""
Tests for transformer.py
"""
from mock import patch
from unittest import TestCase
from ..transformer import BlockStructureTransformers
from .test_utils import MockTransformer
class BlockStructureTransformersTestCase(TestCase):
"""
Test cases for BlockStructureTransformers.
"""
class TestTransformer1(MockTransformer):
pass
class TestTransformer2(MockTransformer):
pass
class UnregisteredTestTransformer3(MockTransformer):
pass
@patch('openedx.core.lib.block_cache.transformer.BlockStructureTransformers.get_available_plugins')
def test_are_all_registered(self, mock_available_transforms):
mock_available_transforms.return_value = {
transformer.name(): transformer
for transformer in [self.TestTransformer1, self.TestTransformer2]
}
for transformers, expected_are_all_registered in [
([], True),
([self.TestTransformer1()], True),
([self.TestTransformer1(), self.TestTransformer2()], True),
([self.UnregisteredTestTransformer3()], False),
([self.TestTransformer1(), self.UnregisteredTestTransformer3()], False),
]:
self.assertEquals(BlockStructureTransformers.are_all_registered(transformers), expected_are_all_registered)
"""
Common utilities for tests in block_cache module
"""
from ..transformer import BlockStructureTransformer
class MockXBlock(object):
def __init__(self, location, field_map=None, children=None, modulestore=None):
self.location = location
self.field_map = field_map or {}
self.children = children or []
self.modulestore = modulestore
def __getattr__(self, attr):
try:
return self.field_map[attr]
except KeyError:
raise AttributeError
def get_children(self):
return [self.modulestore.get_item(child) for child in self.children]
class MockModulestore(object):
def set_blocks(self, blocks):
self.blocks = blocks
def get_item(self, block_key, depth=None):
return self.blocks.get(block_key)
class MockCache(object):
def __init__(self):
self.map = {}
def set(self, key, val):
self.map[key] = val
def get(self, key, default):
return self.map.get(key, default)
def set_many(self, map):
for key, val in map.iteritems():
self.set(key, val)
def get_many(self, keys):
return {key: self.map[key] for key in keys if key in self.map}
def delete(self, key):
del self.map[key]
class MockModulestoreFactory(object):
@classmethod
def create(cls, children_map):
modulestore = MockModulestore()
modulestore.set_blocks({
block_key: MockXBlock(block_key, children=children, modulestore=modulestore)
for block_key, children in enumerate(children_map)
})
return modulestore
class MockUserInfo(object):
def has_staff_access(self):
return False
class MockTransformer(BlockStructureTransformer):
VERSION = 1
def transform(self, user_info, block_structure):
pass
# 0
# / \
# 1 2
# / \
# 3 4
SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
class BlockStructureTestMixin(object):
def verify_block_structure(self, block_structure, children_map):
for block_key, children in enumerate(children_map):
self.assertTrue(
block_structure.has_block(block_key)
)
self.assertEquals(
set(block_structure.get_children(block_key)),
set(children),
)
"""
...
"""
from abc import abstractmethod
from openedx.core.lib.api.plugins import PluginManager
class BlockStructureTransformer(object):
"""
...
"""
# All Transformers are expected to update and maintain a VERSION class attribute
VERSION = 0
@classmethod
def name(cls):
return cls.__name__
@classmethod
def collect(self, block_structure):
"""
Collects any information that's necessary to execute this transformer's
transform method.
"""
pass
@abstractmethod
def transform(self, user_info, block_structure):
"""
Mutates block_structure based on the given user_info.
"""
pass
class BlockStructureTransformers(PluginManager):
"""
Manager for all of the block structure transformers that have been made available.
All block structure transformers should implement `BlockStructureTransformer`.
"""
NAMESPACE = 'openedx.block_structure_transformer'
@classmethod
def get_registered_transformers(cls):
return set(cls.get_available_plugins().itervalues())
@classmethod
def are_all_registered(cls, transformers):
registered_transformers = cls.get_registered_transformers()
return all(
any(transformer.name() == reg_trans.name() for reg_trans in registered_transformers)
for transformer in transformers
)
"""
...
"""
from abc import abstractproperty
class UserInfo(object):
"""
...
"""
@abstractproperty
def has_staff_access(self):
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment