Block Cache Framework

lms/djangoapps/course_blocks/api.py

+"""
+...
+"""
+from django.core.cache import get_cache
+
+from openedx.core.lib.block_cache.block_cache import get_blocks, clear_block_cache
+from xmodule.modulestore.django import modulestore
+
+from .transformers import (
+    start_date,
+    user_partitions,
+    visibility,
+)
+from .user_info import CourseUserInfo
+
+
+LMS_COURSE_TRANSFORMERS = [
+    visibility.VisibilityTransformer(),
+    start_date.StartDateTransformer(),
+    user_partitions.UserPartitionTransformer(),
+]
+
+
+_cache = None
+
+
+def _get_cache():
+    global _cache
+    if not _cache:
+        _cache = get_cache('lms.course_blocks')
+    return _cache
+
+
+def get_course_blocks(
+        user,
+        root_usage_key,
+        transformers=LMS_COURSE_TRANSFORMERS,
+):
+    if transformers is None:
+        transformers = LMS_COURSE_TRANSFORMERS
+
+    return get_blocks(
+        _get_cache(), modulestore(), CourseUserInfo(root_usage_key.course_key, user), root_usage_key, transformers,
+    )
+
+
+def clear_course_from_cache(course_key):
+    course_usage_key = modulestore().make_course_usage_key(course_key)
+    return clear_block_cache(_get_cache(), course_usage_key)

openedx/core/lib/block_cache/__init__.py

+"""
+...
+"""
+# TODO 8874: Write docstrings
+# TODO 8874: Think about encoding of cache (compression? pickle vs. JSON?). David Baumgold wants us to consider using JSON.

openedx/core/lib/block_cache/block_cache.py

+"""
+...
+"""
+from block_structure import BlockStructureFactory
+from transformer import BlockStructureTransformers
+
+
+def get_blocks(cache, modulestore, user_info, root_block_key, transformers):
+    if not BlockStructureTransformers.are_all_registered(transformers):
+        raise Exception("One or more requested transformers are not registered.")
+
+    # Load the cached block structure.
+    root_block_structure = BlockStructureFactory.create_from_cache(root_block_key, cache)
+
+    if not root_block_structure:
+
+        # Create the block structure from the modulestore
+        root_block_structure = BlockStructureFactory.create_from_modulestore(root_block_key, modulestore)
+
+        # Collect data from each registered transformer
+        for transformer in BlockStructureTransformers.get_registered_transformers():
+            root_block_structure.add_transformer(transformer)
+            transformer.collect(root_block_structure)
+
+        # Collect all fields that were requested by the transformers
+        root_block_structure.collect_requested_xblock_fields()
+
+        # Cache this information
+        BlockStructureFactory.serialize_to_cache(root_block_structure, cache)
+
+    # Execute requested transforms on block structure
+    for transformer in transformers:
+        transformer.transform(user_info, root_block_structure)
+
+    # Prune block structure
+    root_block_structure.prune()
+
+    return root_block_structure
+
+
+def clear_block_cache(cache, root_block_key):
+    BlockStructureFactory.remove_from_cache(root_block_key, cache)

openedx/core/lib/block_cache/block_structure.py

+"""
+...
+"""
+from collections import defaultdict
+from graph_traversals import traverse_topologically, traverse_post_order
+from logging import getLogger
+
+from transformer import BlockStructureTransformers
+
+
+logger = getLogger(__name__)  # pylint: disable=C0103
+
+TRANSFORMER_VERSION_KEY = '_version'
+
+
+class BlockStructure(object):
+    """
+    A class to encapsulate a structure of blocks, a directed acyclic graph of blocks.
+    """
+    class BlockRelations(object):
+        def __init__(self):
+            self.parents = []
+            self.children = []
+
+    def __init__(self, root_block_key):
+        self.root_block_key = root_block_key
+        self._block_relations = defaultdict(self.BlockRelations)
+        self._add_block(self._block_relations, root_block_key)
+
+    def add_relation(self, parent_key, child_key):
+        self._add_relation(self._block_relations, parent_key, child_key)
+
+    def get_parents(self, usage_key):
+        return self._block_relations.get(usage_key).parents if self.has_block(usage_key) else []
+
+    def get_children(self, usage_key):
+        return self._block_relations[usage_key].children if self.has_block(usage_key) else []
+
+    def has_block(self, usage_key):
+        return usage_key in self._block_relations
+
+    def get_block_keys(self):
+        return self._block_relations.iterkeys()
+
+    def topological_traversal(self, get_result=None, predicate=None):
+        return traverse_topologically(
+            start_node=self.root_block_key,
+            get_parents=self.get_parents,
+            get_children=self.get_children,
+            get_result=get_result,
+            predicate=predicate,
+        )
+
+    def prune(self):
+        # create a new block relations map with only those blocks that are still linked
+        pruned_block_relations = defaultdict(self.BlockRelations)
+        old_block_relations = self._block_relations
+
+        # def do_for_each_block(block_key):
+        #     if block_key in old_block_relations:
+        #         self._add_block(pruned_block_relations, block_key)
+        #
+        #         for parent in old_block_relations[block_key].parents:
+        #             if parent in pruned_block_relations:
+        #                 self._add_relation(pruned_block_relations, parent, block_key)
+
+        def do_for_each_block(block_key):
+            if block_key in old_block_relations:
+                self._add_block(pruned_block_relations, block_key)
+
+                for child in old_block_relations[block_key].children:
+                    if child in pruned_block_relations:
+                        self._add_relation(pruned_block_relations, block_key, child)
+
+        list(traverse_post_order(
+            start_node=self.root_block_key,
+            get_children=self.get_children,
+            get_result=do_for_each_block
+        ))
+
+        self._block_relations = pruned_block_relations
+
+    @classmethod
+    def _add_relation(cls, block_relations, parent_key, child_key):
+        block_relations[child_key].parents.append(parent_key)
+        block_relations[parent_key].children.append(child_key)
+
+    @classmethod
+    def _add_block(cls, block_relations, block_key):
+        _ = block_relations[block_key]
+
+
+class BlockStructureBlockData(BlockStructure):
+    """
+    A sub-class of BlockStructure that encapsulates data captured about the blocks.
+    """
+    class BlockData(object):
+        def __init__(self):
+            # dictionary mapping xblock field names to their values.
+            self._xblock_fields = {}
+
+            # dictionary mapping transformers' IDs to their collected data.
+            self._transformer_data = defaultdict(dict)
+
+    def __init__(self, root_block_key):
+        super(BlockStructureBlockData, self).__init__(root_block_key)
+
+        # dictionary mapping usage keys to BlockData
+        self._block_data_map = defaultdict(self.BlockData)
+
+        # dictionary mapping transformer IDs to block-structure-wide transformer data
+        self._transformer_data = defaultdict(dict)
+
+    def get_xblock_field(self, usage_key, field_name, default=None):
+        block_data = self._block_data_map.get(usage_key)
+        return block_data._xblock_fields.get(field_name, default) if block_data else default
+
+    def get_transformer_data(self, transformer, key, default=None):
+        return self._transformer_data.get(transformer.name(), {}).get(key, default)
+
+    def get_transformer_data_version(self, transformer):
+        return self.get_transformer_data(transformer, TRANSFORMER_VERSION_KEY, 0)
+
+    def get_transformer_block_data(self, usage_key, transformer, key, default=None):
+        block_data = self._block_data_map.get(usage_key)
+        return block_data._transformer_data.get(
+            transformer.name(), {}
+        ).get(key, default) if block_data else default
+
+    def remove_block(self, usage_key):
+        # Remove block from its children.
+        for child in self._block_relations[usage_key].children:
+            self._block_relations[child].parents.remove(usage_key)
+
+        # Remove block from its parents.
+        for parent_key in self._block_relations[usage_key].parents:
+            self._block_relations[parent_key].children.remove(usage_key)
+
+        # Remove block.
+        if usage_key in self._block_relations:
+            del self._block_relations[usage_key]
+        if usage_key in self._block_data_map:
+            del self._block_data_map[usage_key]
+
+    def remove_block_if(self, removal_condition):
+        def predicate(block_key):
+            if removal_condition(block_key):
+                self.remove_block(block_key)
+                return False
+            return True
+        list(self.topological_traversal(predicate=predicate))
+
+
+class BlockStructureCollectedData(BlockStructureBlockData):
+    """
+    A sub-class of BlockStructure that encapsulates information about the blocks during the collect phase.
+    """
+    def __init__(self, root_block_key):
+        super(BlockStructureCollectedData, self).__init__(root_block_key)
+
+        self._xblock_map = {}  # dict[UsageKey: XBlock]
+        self._requested_xblock_fields = set()
+
+    def request_xblock_fields(self, *field_names):
+        self._requested_xblock_fields.update(set(field_names))
+
+    def collect_requested_xblock_fields(self):
+        if not self._requested_xblock_fields:
+            return
+
+        for xblock in self._xblock_map.itervalues():
+            for field_name in self._requested_xblock_fields:
+                self._set_xblock_field(xblock, field_name)
+
+    def _set_xblock_field(self, xblock, field_name):
+        if hasattr(xblock, field_name):
+            self._block_data_map[xblock.location]._xblock_fields[field_name] = getattr(xblock, field_name)
+
+    def add_xblock(self, xblock):
+        self._xblock_map[xblock.location] = xblock
+
+    def get_xblock(self, usage_key):
+        return self._xblock_map[usage_key]
+
+    def add_transformer(self, transformer):
+        if transformer.VERSION == 0:
+            raise Exception('VERSION attribute is not set on transformer {0}.', transformer.name())
+        self.set_transformer_data(transformer, TRANSFORMER_VERSION_KEY, transformer.VERSION)
+
+    def set_transformer_data(self, transformer, key, value):
+        self._transformer_data[transformer.name()][key] = value
+
+    def set_transformer_block_data(self, usage_key, transformer, key, value):
+        self._block_data_map[usage_key]._transformer_data[transformer.name()][key] = value
+
+
+class BlockStructureFactory(object):
+    @classmethod
+    def create_from_modulestore(cls, root_block_key, modulestore):
+        block_structure = BlockStructureCollectedData(root_block_key)
+
+        def build_block_structure(xblock):
+            """
+            Helper function to recursively walk block structure
+            """
+            block_structure.add_xblock(xblock)
+            for child in xblock.get_children():
+                block_structure.add_relation(xblock.location, child.location)
+                build_block_structure(child)
+
+        root_xblock = modulestore.get_item(root_block_key, depth=None)
+        build_block_structure(root_xblock)
+        return block_structure
+
+    @classmethod
+    def serialize_to_cache(cls, block_structure, cache):
+        cache.set(
+            cls._encode_root_cache_key(block_structure.root_block_key),
+            (block_structure._block_relations, block_structure._transformer_data)
+        )
+        cache.set_many({
+            unicode(usage_key): block_data
+            for usage_key, block_data
+            in block_structure._block_data_map.iteritems()
+        })
+
+    @classmethod
+    def create_from_cache(cls, root_block_key, cache):
+        """
+        Returns:
+            BlockStructure, if the block structure is in the cache, and
+            NoneType otherwise.
+        """
+        block_relations, transformer_data = cache.get(cls._encode_root_cache_key(root_block_key), (None, None))
+        if block_relations:
+            block_structure = BlockStructureBlockData(root_block_key)
+            block_structure._block_relations = block_relations
+            block_structure._transformer_data = transformer_data
+
+            if all(
+                transformer.VERSION == block_structure.get_transformer_data_version(transformer)
+                for transformer in BlockStructureTransformers.get_registered_transformers()
+            ):
+                block_structure._block_data_map = cache.get_many(block_relations.iterkeys())
+                return block_structure
+            else:
+                logger.info("Collected data for transformer is outdated.")
+
+        return None
+
+    @classmethod
+    def remove_from_cache(cls, root_block_key, cache):
+        cache.delete(cls._encode_root_cache_key(root_block_key))
+        # TODO also remove all block data?
+
+    @classmethod
+    def _encode_root_cache_key(cls, root_block_key):
+        return "root.key." + unicode(root_block_key)

openedx/core/lib/block_cache/graph_traversals.py

+"""
+...
+"""
+from collections import deque
+
+
+def _traverse_generic(start_node, get_parents, get_children, get_result=None, predicate=None):
+    """
+    Helper function to avoid duplicating functionality between
+    traverse_depth_first and traverse_topologically.
+
+    If get_parents is None, do a depth first traversal.
+    Else, do a topological traversal.
+
+    The topological traversal has a worse time complexity than depth-first does,
+    as it needs to check whether each node's parents have been visited.
+
+    Arguments:
+        start_node - the starting node for the traversal
+        get_parents - function that returns a list of parent nodes for the given node
+        get_children - function that returns a list of children nodes for the given node
+        get_result - function that computes and returns the resulting value to be yielded for the given node
+        predicate - function that returns whether or not to yield the given node
+    """
+    # If get_result or predicate aren't provided, just make them to no-ops.
+    get_result = get_result or (lambda node_: node_)
+    predicate = predicate or (lambda __: True)
+
+    # For our stack, we use the deque type, which is O(1) for pop and append.
+    stack = deque([start_node])
+    yield_results = {}
+
+    # While there are more nodes on the stack...
+    while stack:
+
+        # Take a node off the top of the stack.
+        curr_node = stack.pop()
+
+        # If we're doing a topological traversal, then make sure all the node's
+        # parents have been visited. If they haven't, then skip the node for
+        # now; we'll encounter it again later through another one of its
+        # parents.
+        if get_parents and curr_node != start_node:
+            parents = get_parents(curr_node)
+            all_parents_visited = all(parent in yield_results for parent in parents)
+            any_parent_yielded = any(yield_results[parent] for parent in parents) if all_parents_visited else False
+            if not all_parents_visited or not any_parent_yielded:
+                continue
+
+        # Add its unvisited children to the stack in reverse order so that
+        # they are popped off in their original order.
+        # It's important that we visit the children even if the parent isn't yielded
+        # in case a child has multiple parents and this is its last parent.
+        unvisited_children = get_children(curr_node)
+
+        # If we're not doing a topological traversal, check whether the child has been visited.
+        if not get_parents:
+            unvisited_children = list(
+                child
+                for child in unvisited_children
+                if child not in yield_results
+            )
+        unvisited_children.reverse()
+        stack.extend(unvisited_children)
+
+        # Return the result if it satisfies the predicate.
+        # Keep track of the result so we know whether to yield its children.
+        should_yield_node = predicate(curr_node)
+        yield_results[curr_node] = should_yield_node
+        if should_yield_node:
+            yield get_result(curr_node)
+
+
+def traverse_topologically(start_node, get_parents, get_children, get_result=None, predicate=None):
+    return _traverse_generic(
+        start_node,
+        get_parents=get_parents,
+        get_children=get_children,
+        get_result=get_result,
+        predicate=predicate
+    )
+
+
+def traverse_pre_order(start_node, get_children, get_result=None, predicate=None):
+    return _traverse_generic(
+        start_node,
+        get_parents=None,
+        get_children=get_children,
+        get_result=get_result,
+        predicate=predicate
+    )
+
+
+class BlockAndChildIndexStackItem(object):
+    """
+    Class for items in the stack.
+    """
+    def __init__(self, block):
+        self.block = block
+        self.children = None
+        self.child_index = 0
+
+    def next_child(self, get_children):
+        """
+        Returns the next child of the block for this item in the stack.
+        """
+        if self.children is None:
+            self.children = get_children(self.block)
+
+        child = None
+        if self.child_index < len(self.children):
+            child = self.children[self.child_index]
+            self.child_index += 1
+        return child
+
+
+def traverse_post_order(start_node, get_children, get_result=None, predicate=None):
+    # If get_result or predicate aren't provided, just make them to no-ops.
+    get_result = get_result or (lambda node_: node_)
+    predicate = predicate or (lambda __: True)
+
+    # For our stack, we use the deque type, which is O(1) for pop and append.
+    stack = deque([BlockAndChildIndexStackItem(start_node)])
+    visited = set()
+
+    while stack:
+        # peek at the next item in the stack
+        current_stack_item = stack[len(stack)-1]
+
+        # verify the block wasn't already visited and the block satisfies the predicate
+        if current_stack_item.block in visited or not predicate(current_stack_item.block):
+            stack.pop()
+            continue
+
+        next_child = current_stack_item.next_child(get_children)
+        if next_child:
+            stack.append(BlockAndChildIndexStackItem(next_child))
+        else:
+            yield get_result(current_stack_item.block)
+            visited.add(current_stack_item.block)
+            stack.pop()

openedx/core/lib/block_cache/tests/test_block_cache.py

+"""
+Tests for block_cache.py
+"""
+
+from mock import patch
+from unittest import TestCase
+from .test_utils import (
+    MockModulestoreFactory, MockCache, MockUserInfo, MockTransformer, SIMPLE_CHILDREN_MAP, BlockStructureTestMixin
+)
+from ..block_cache import get_blocks
+
+class TestBlockCache(TestCase, BlockStructureTestMixin):
+
+    class TestTransformer1(MockTransformer):
+        @classmethod
+        def block_key(cls):
+            return 't1.key1'
+
+        @classmethod
+        def block_val(cls, block_key):
+            return 't1.val1.' + unicode(block_key)
+
+        @classmethod
+        def collect(self, block_structure):
+            list(
+                block_structure.topological_traversal(
+                    get_result=lambda block_key: block_structure.set_transformer_block_data(
+                        block_key, self, self.block_key(), self.block_val(block_key)
+            )))
+
+        def transform(self, user_info, block_structure):
+            def assert_collected_value(block_key):
+                assert (
+                    block_structure.get_transformer_block_data(
+                        block_key,
+                        self,
+                        self.block_key()
+                    ) == self.block_val(block_key)
+                )
+
+            list(
+                block_structure.topological_traversal(
+                    get_result=lambda block_key: assert_collected_value(block_key)
+            ))
+
+    @patch('openedx.core.lib.block_cache.transformer.BlockStructureTransformers.get_available_plugins')
+    def test_get_blocks(self, mock_available_transforms):
+        children_map = SIMPLE_CHILDREN_MAP
+        cache = MockCache()
+        user_info = MockUserInfo()
+        modulestore = MockModulestoreFactory.create(children_map)
+        transformers = [self.TestTransformer1()]
+
+        mock_available_transforms.return_value = {}
+        with self.assertRaisesRegexp(Exception, "requested transformers are not registered"):
+            get_blocks(cache, modulestore, user_info, root_block_key=0, transformers=transformers)
+
+        mock_available_transforms.return_value = {transformer.name(): transformer for transformer in transformers}
+        block_structure = get_blocks(cache, modulestore, user_info, root_block_key=0, transformers=transformers)
+        self.verify_block_structure(block_structure, children_map)

openedx/core/lib/block_cache/tests/test_block_structure.py

+"""
+Tests for block_structure.py
+"""
+from collections import namedtuple
+import ddt
+from mock import patch
+from unittest import TestCase
+
+from ..block_structure import (
+    BlockStructure, BlockStructureCollectedData, BlockStructureBlockData, BlockStructureFactory
+)
+from ..transformer import BlockStructureTransformer, BlockStructureTransformers
+from .test_utils import (
+    MockCache, MockXBlock, MockModulestoreFactory, MockTransformer, SIMPLE_CHILDREN_MAP, BlockStructureTestMixin
+)
+
+
+@ddt.ddt
+class TestBlockStructure(TestCase):
+    """
+    Tests for BlockStructure
+    """
+    def get_parents_map(self, children_map):
+        parent_map = [[] for node in children_map]
+        for parent, children in enumerate(children_map):
+            for child in children:
+                parent_map[child].append(parent)
+        return parent_map
+
+    @ddt.data(
+        [],
+        #     0
+        #    / \
+        #   1  2
+        #  / \
+        # 3   4
+        SIMPLE_CHILDREN_MAP,
+        #       0
+        #      /
+        #     1
+        #    /
+        #   2
+        #  /
+        # 3
+        [[1], [2], [3], []],
+        #     0
+        #    / \
+        #   1  2
+        #   \ /
+        #    3
+        [[1, 2], [3], [3], []],
+    )
+    def test_relations(self, children_map):
+        # create block structure
+        block_structure = BlockStructure(root_block_key=0)
+
+        # add_relation
+        for parent, children in enumerate(children_map):
+            for child in children:
+               block_structure.add_relation(parent, child)
+
+        # get_children
+        for parent, children in enumerate(children_map):
+           self.assertSetEqual(set(block_structure.get_children(parent)), set(children))
+
+        # get_parents
+        for child, parents in enumerate(self.get_parents_map(children_map)):
+           self.assertSetEqual(set(block_structure.get_parents(child)), set(parents))
+
+        # has_block
+        for node in range(len(children_map)):
+            self.assertTrue(block_structure.has_block(node))
+        self.assertFalse(block_structure.has_block(len(children_map) + 1))
+
+
+class TestBlockStructureData(TestCase):
+    """
+    Tests for BlockStructureBlockData and BlockStructureCollectedData
+    """
+    def test_non_versioned_transformer(self):
+        class TestNonVersionedTransformer(BlockStructureTransformer):
+            def transform(self, user_info, block_structure):
+                pass
+
+        block_structure = BlockStructureCollectedData(root_block_key=0)
+
+        with self.assertRaisesRegexp(Exception, "VERSION attribute is not set"):
+            block_structure.add_transformer(TestNonVersionedTransformer())
+
+    def test_transformer_data(self):
+        # transformer test cases
+        TransformerInfo = namedtuple("TransformerInfo", "transformer structure_wide_data block_specific_data")
+        transformers_info = [
+            TransformerInfo(
+                transformer=MockTransformer(),
+                structure_wide_data=[("t1.global1", "t1.g.val1"), ("t1.global2", "t1.g.val2"),],
+                block_specific_data={
+                    "B1": [("t1.key1", "t1.b1.val1"), ("t1.key2", "t1.b1.val2")],
+                    "B2": [("t1.key1", "t1.b2.val1"), ("t1.key2", "t1.b2.val2")],
+                    "B3": [("t1.key1", True), ("t1.key2", False)],
+                    "B4": [("t1.key1", None), ("t1.key2", False)],
+                },
+            ),
+            TransformerInfo(
+                transformer=MockTransformer(),
+                structure_wide_data=[("t2.global1", "t2.g.val1"), ("t2.global2", "t2.g.val2"),],
+                block_specific_data={
+                    "B1": [("t2.key1", "t2.b1.val1"), ("t2.key2", "t2.b1.val2")],
+                    "B2": [("t2.key1", "t2.b2.val1"), ("t2.key2", "t2.b2.val2")],
+                },
+            ),
+        ]
+
+        # create block structure
+        block_structure = BlockStructureCollectedData(root_block_key=0)
+
+        # set transformer data
+        for t_info in transformers_info:
+            block_structure.add_transformer(t_info.transformer)
+            for key, val in t_info.structure_wide_data:
+                block_structure.set_transformer_data(t_info.transformer, key, val)
+            for block, block_data in t_info.block_specific_data.iteritems():
+                for key, val in block_data:
+                    block_structure.set_transformer_block_data(block, t_info.transformer, key, val)
+
+        # verify transformer data
+        for t_info in transformers_info:
+            self.assertEquals(
+                block_structure.get_transformer_data_version(t_info.transformer),
+                MockTransformer.VERSION
+            )
+            for key, val in t_info.structure_wide_data:
+                self.assertEquals(
+                    block_structure.get_transformer_data(t_info.transformer, key),
+                    val,
+                )
+            for block, block_data in t_info.block_specific_data.iteritems():
+                for key, val in block_data:
+                    self.assertEquals(
+                        block_structure.get_transformer_block_data(block, t_info.transformer, key),
+                        val,
+                    )
+
+    def test_xblock_data(self):
+        # block test cases
+        blocks = [
+            MockXBlock("A", {}),
+            MockXBlock("B", {"field1": "B.val1"}),
+            MockXBlock("C", {"field1": "C.val1", "field2": "C.val2"}),
+            MockXBlock("D", {"field1": True, "field2": False}),
+            MockXBlock("E", {"field1": None, "field2": False}),
+        ]
+
+        # add each block
+        block_structure = BlockStructureCollectedData(root_block_key=0)
+        for block in blocks:
+            block_structure.add_xblock(block)
+
+        # request fields
+        fields = ["field1", "field2", "field3"]
+        block_structure.request_xblock_fields(*fields)
+
+        # verify fields have not been collected yet
+        for block in blocks:
+            for field in fields:
+                self.assertIsNone(block_structure.get_xblock_field(block.location, field))
+
+        # collect fields
+        block_structure.collect_requested_xblock_fields()
+
+        # verify values of collected fields
+        for block in blocks:
+            for field in fields:
+                self.assertEquals(
+                    block_structure.get_xblock_field(block.location, field),
+                    block.field_map.get(field),
+                )
+
+    def test_remove_block(self):
+        block_structure = BlockStructureBlockData(root_block_key=0)
+        for parent, children in enumerate(SIMPLE_CHILDREN_MAP):
+            for child in children:
+               block_structure.add_relation(parent, child)
+
+        self.assertTrue(block_structure.has_block(1))
+        self.assertTrue(1 in block_structure.get_children(0))
+
+        block_structure.remove_block(1)
+
+        self.assertFalse(block_structure.has_block(1))
+        self.assertFalse(1 in block_structure.get_children(0))
+
+        self.assertTrue(block_structure.has_block(3))
+        self.assertTrue(block_structure.has_block(4))
+
+        block_structure.prune()
+
+        self.assertFalse(block_structure.has_block(3))
+        self.assertFalse(block_structure.has_block(4))
+
+
+class TestBlockStructureFactory(TestCase, BlockStructureTestMixin):
+    """
+    Tests for BlockStructureFactory
+    """
+    def test_factory_methods(self):
+        children_map = SIMPLE_CHILDREN_MAP
+        modulestore = MockModulestoreFactory.create(children_map)
+        cache = MockCache()
+
+        # test create from modulestore
+        block_structure = BlockStructureFactory.create_from_modulestore(root_block_key=0, modulestore=modulestore)
+        self.verify_block_structure(block_structure, children_map)
+
+        # test not in cache
+        self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
+
+        # test transformers outdated
+        BlockStructureFactory.serialize_to_cache(block_structure, cache)
+        with patch('openedx.core.lib.block_cache.block_structure.logger.info') as mock_logger:
+            self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))
+            self.assertTrue(mock_logger.called)
+
+        # update transformers
+        for transformer in BlockStructureTransformers.get_registered_transformers():
+            block_structure.add_transformer(transformer)
+            block_structure.set_transformer_block_data(
+                usage_key=0, transformer=transformer, key='test', value='test.val'
+            )
+        BlockStructureFactory.serialize_to_cache(block_structure, cache)
+
+        # test re-create from cache
+        from_cache_block_structure = BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache)
+        self.assertIsNotNone(from_cache_block_structure)
+        self.verify_block_structure(from_cache_block_structure, children_map)
+
+        # test remove from cache
+        BlockStructureFactory.remove_from_cache(root_block_key=0, cache=cache)
+        self.assertIsNone(BlockStructureFactory.create_from_cache(root_block_key=0, cache=cache))

openedx/core/lib/block_cache/tests/test_graph_traversals.py

+"""
+...
+"""
+
+# TODO 8874: Test graph_traversals more comprehensively.
+
+from collections import defaultdict
+from unittest import TestCase
+
+from ..graph_traversals import (
+    traverse_pre_order, traverse_post_order, traverse_topologically
+)
+
+
+class GraphTraversalsTestCase(TestCase):
+    """
+    ...
+    """
+
+    def setUp(self):
+        """
+        ...
+        """
+        #       b1
+        #      /  \
+        #     c1  c2
+        #    /  \   \
+        #  d1  d2   d3
+        #  \  / \
+        #   e1  e2
+        #       |
+        #       f1
+        super(GraphTraversalsTestCase, self).setUp()
+        self.graph_1 = {
+            'a1': ['b1'],
+            'a2': ['b2'],
+            'b1': ['c1', 'c2'],
+            'b2': [],
+            'c1': ['d1', 'd2'],
+            'c2': ['d3'],
+            'd1': ['e1'],
+            'd2': ['e1', 'e2'],
+            'd3': [],
+            'e1': [],
+            'e2': ['f1'],
+            'f1': [],
+        }
+        self.graph_1_parents = self.get_parent_map(self.graph_1)
+
+    @staticmethod
+    def get_parent_map(graph):
+        """
+        ...
+        """
+        result = defaultdict(list)
+        for parent, children in graph.iteritems():
+            for child in children:
+                result[child].append(parent)
+        return result
+
+    def test_pre_order(self):
+        """
+        ...
+        """
+        self.assertEqual(
+            list(traverse_pre_order(
+                start_node='b1',
+                get_children=(lambda node: self.graph_1[node]),
+                get_result=(lambda node: node + '_'),
+                predicate=(lambda node: node != 'd3'),
+            )),
+            ['b1_', 'c1_', 'd1_', 'e1_', 'd2_', 'e2_', 'f1_', 'c2_']
+        )
+
+    def test_post_order(self):
+        """
+        ...
+        """
+        self.assertEqual(
+            list(traverse_post_order(
+                start_node='b1',
+                get_children=(lambda node: self.graph_1[node]),
+                get_result=(lambda node: node + '_'),
+                predicate=(lambda node: node != 'd3'),
+            )),
+            ['e1_', 'd1_', 'f1_', 'e2_', 'd2_', 'c1_', 'c2_', 'b1_']
+        )
+
+    def test_topological(self):
+        """
+        ...
+        """
+        self.assertEqual(
+            list(traverse_topologically(
+                start_node='b1',
+                get_children=(lambda node: self.graph_1[node]),
+                get_parents=(lambda node: self.graph_1_parents[node]),
+                predicate=(lambda node: node != 'd3'),
+            )),
+            ['b1', 'c1', 'd1', 'd2', 'e1', 'e2', 'f1', 'c2']
+        )
+
+    def test_topological_with_predicate(self):
+        """
+        ...
+        """
+        self.assertEqual(
+            list(traverse_topologically(
+                start_node='b1',
+                get_children=(lambda node: self.graph_1[node]),
+                get_parents=(lambda node: self.graph_1_parents[node]),
+                predicate=(lambda node: node != 'd2')
+            )),
+            ['b1', 'c1', 'd1', 'e1', 'c2', 'd3']
+        )

openedx/core/lib/block_cache/tests/test_transformer.py

+"""
+Tests for transformer.py
+"""
+
+from mock import patch
+from unittest import TestCase
+
+from ..transformer import BlockStructureTransformers
+from .test_utils import MockTransformer
+
+
+class BlockStructureTransformersTestCase(TestCase):
+    """
+    Test cases for BlockStructureTransformers.
+    """
+    class TestTransformer1(MockTransformer):
+        pass
+
+    class TestTransformer2(MockTransformer):
+        pass
+
+    class UnregisteredTestTransformer3(MockTransformer):
+        pass
+
+    @patch('openedx.core.lib.block_cache.transformer.BlockStructureTransformers.get_available_plugins')
+    def test_are_all_registered(self, mock_available_transforms):
+
+        mock_available_transforms.return_value = {
+            transformer.name(): transformer
+            for transformer in [self.TestTransformer1, self.TestTransformer2]
+        }
+
+        for transformers, expected_are_all_registered in [
+            ([], True),
+            ([self.TestTransformer1()], True),
+            ([self.TestTransformer1(), self.TestTransformer2()], True),
+            ([self.UnregisteredTestTransformer3()], False),
+            ([self.TestTransformer1(), self.UnregisteredTestTransformer3()], False),
+        ]:
+            self.assertEquals(BlockStructureTransformers.are_all_registered(transformers), expected_are_all_registered)

openedx/core/lib/block_cache/tests/test_utils.py

+"""
+Common utilities for tests in block_cache module
+"""
+
+from ..transformer import BlockStructureTransformer
+
+
+class MockXBlock(object):
+    def __init__(self, location, field_map=None, children=None, modulestore=None):
+        self.location = location
+        self.field_map = field_map or {}
+
+        self.children = children or []
+        self.modulestore = modulestore
+
+    def __getattr__(self, attr):
+        try:
+            return self.field_map[attr]
+        except KeyError:
+            raise AttributeError
+
+    def get_children(self):
+        return [self.modulestore.get_item(child) for child in self.children]
+
+
+class MockModulestore(object):
+    def set_blocks(self, blocks):
+        self.blocks = blocks
+
+    def get_item(self, block_key, depth=None):
+         return self.blocks.get(block_key)
+
+
+class MockCache(object):
+    def __init__(self):
+        self.map = {}
+
+    def set(self, key, val):
+        self.map[key] = val
+
+    def get(self, key, default):
+        return self.map.get(key, default)
+
+    def set_many(self, map):
+        for key, val in map.iteritems():
+            self.set(key, val)
+
+    def get_many(self, keys):
+        return {key: self.map[key] for key in keys if key in self.map}
+
+    def delete(self, key):
+        del self.map[key]
+
+
+class MockModulestoreFactory(object):
+    @classmethod
+    def create(cls, children_map):
+        modulestore = MockModulestore()
+        modulestore.set_blocks({
+            block_key: MockXBlock(block_key, children=children, modulestore=modulestore)
+            for block_key, children in enumerate(children_map)
+        })
+        return modulestore
+
+
+class MockUserInfo(object):
+    def has_staff_access(self):
+        return False
+
+
+class MockTransformer(BlockStructureTransformer):
+    VERSION = 1
+    def transform(self, user_info, block_structure):
+        pass
+
+
+#     0
+#    / \
+#   1  2
+#  / \
+# 3   4
+SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
+
+
+class BlockStructureTestMixin(object):
+    def verify_block_structure(self, block_structure, children_map):
+        for block_key, children in enumerate(children_map):
+            self.assertTrue(
+                block_structure.has_block(block_key)
+            )
+            self.assertEquals(
+                set(block_structure.get_children(block_key)),
+                set(children),
+            )

openedx/core/lib/block_cache/transformer.py

+"""
+...
+"""
+from abc import abstractmethod
+from openedx.core.lib.api.plugins import PluginManager
+
+
+class BlockStructureTransformer(object):
+    """
+    ...
+    """
+    # All Transformers are expected to update and maintain a VERSION class attribute
+    VERSION = 0
+
+    @classmethod
+    def name(cls):
+        return cls.__name__
+
+    @classmethod
+    def collect(self, block_structure):
+        """
+        Collects any information that's necessary to execute this transformer's
+        transform method.
+        """
+        pass
+
+    @abstractmethod
+    def transform(self, user_info, block_structure):
+        """
+        Mutates block_structure based on the given user_info.
+        """
+        pass
+
+
+class BlockStructureTransformers(PluginManager):
+    """
+    Manager for all of the block structure transformers that have been made available.
+
+    All block structure transformers should implement `BlockStructureTransformer`.
+    """
+    NAMESPACE = 'openedx.block_structure_transformer'
+
+    @classmethod
+    def get_registered_transformers(cls):
+        return set(cls.get_available_plugins().itervalues())
+
+    @classmethod
+    def are_all_registered(cls, transformers):
+        registered_transformers = cls.get_registered_transformers()
+        return all(
+            any(transformer.name() == reg_trans.name() for reg_trans in registered_transformers)
+            for transformer in transformers
+        )

openedx/core/lib/block_cache/user_info.py

+"""
+...
+"""
+from abc import abstractproperty
+
+class UserInfo(object):
+    """
+    ...
+    """
+    @abstractproperty
+    def has_staff_access(self):
+        pass