Merge pull request #10365 from edx/mobile/block-cache-framework

Block Cache framework (MA-1551)

lms/envs/test.py

@@ -219,6 +219,10 @@ CACHES = {
     'course_structure_cache': {
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
     },
+    'block_cache': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'edx_location_block_cache',
+    },
 }
 
 # Dummy secret key for dev

openedx/core/lib/block_cache/__init__.py

+"""
+The block_cache django app provides an extensible framework for caching
+data of block structures from the modulestore.
+
+Dual-Phase. The framework is meant to be used in 2 phases.
+
+  * Collect Phase (for expensive and full-tree traversals) - In the
+    first phase, the "collect" phase, any and all data from the
+    modulestore should be collected and cached for later access to
+    the block structure.  Instantiating any and all xBlocks in the block
+    structure is also done at this phase, since that is also (currently)
+    a costly operation.
+
+    Any full tree traversals should also be done during this phase. For
+    example, if data for a block depends on its parents, the traversal
+    should happen during the collection phase and any required data
+    for the block should be percolated down the tree and stored as
+    aggregate values on the descendants.  This allows for faster and
+    direct access to blocks in the Transform phase.
+
+  * Transform Phase (for fast access to blocks) - In the second
+    phase, the "transform" phase, only the previously collected and
+    cached data should be accessed. There should be no access to the
+    modulestore or instantiation of xBlocks in this phase.
+
+
+To make this framework extensible, the Transformer and
+Extensibility design patterns are used. This django app only
+provides the underlying framework for Block Structure Transformers
+and a Transformer Registry.  Clients are expected to provide actual
+implementations of Transformers or add them to the extensible Registry.
+
+Transformers. As inspired by
+http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
+a Block Structure Transformer takes in a block structure (or tree) and
+manipulates the structure and the data of its blocks according to its
+own requirements. Its output can then be used for further
+transformations by other transformers down the pipeline.
+
+Note: For performance and space optimization, our implementation
+differs from the paper in that our transformers mutate the block
+structure in-place rather than returning a modified copy of it.
+
+Block Structure. The BlockStructure and its family of classes
+provided with this framework are the base data types for accessing
+and manipulating block structures. BlockStructures are constructed
+using the BlockStructureFactory and then used as the currency across
+Transformers.
+
+Registry. Transformers are registered using the platform's
+PluginManager (e.g., Stevedore). This is currently done by updating
+setup.py.  Only registered transformers are called during the Collect
+Phase.  And only registered transformers can be used during the
+Transform phase.
+
+Note: A partial subset (as an ordered list) of the registered
+transformers can be requested during the Transform phase, allowing
+the client to manipulate exactly which transformers to call.
+"""

openedx/core/lib/block_cache/block_cache.py

+"""
+Top-level module for the Block Cache framework with higher order
+functions for getting and clearing cached blocks.
+"""
+from .block_structure_factory import BlockStructureFactory
+from .exceptions import TransformerException
+from .transformer_registry import TransformerRegistry
+
+
+def get_blocks(cache, modulestore, usage_info, root_block_usage_key, transformers):
+    """
+    Top-level function in the Block Cache framework that manages
+    the cache (populating it and updating it when needed), calls the
+    transformers as appropriate (collect and transform methods), and
+    accessing the modulestore when needed (at cache miss).
+
+    Arguments:
+        cache (django.core.cache.backends.base.BaseCache) - The
+            cache to use for storing/retrieving the block structure's
+            collected data.
+
+        modulestore (ModuleStoreRead) - The modulestore that
+            contains the data for the xBlock objects corresponding to
+            the block structure.
+
+        usage_info (any negotiated type) - A usage-specific object
+            that is forwarded to all requested Transformers in order
+            to apply a usage-specific transform. For example, an
+            instance of usage_info would contain a user object for
+            which the transform should be applied.
+
+        root_block_usage_key (UsageKey) - The usage_key for the root
+            of the block structure that is being accessed.
+
+        transformers ([BlockStructureTransformer]) - The list of
+            transformers whose transform methods are to be called.
+            This list should be a subset of the list of registered
+            transformers in the Transformer Registry.
+    """
+
+    # Verify that all requested transformers are registered in the
+    # Transformer Registry.
+    unregistered_transformers = TransformerRegistry.find_unregistered(transformers)
+    if unregistered_transformers:
+        raise TransformerException(
+            "The following requested transformers are not registered: {}".format(unregistered_transformers)
+        )
+
+    # Load the cached block structure.
+    root_block_structure = BlockStructureFactory.create_from_cache(root_block_usage_key, cache, transformers)
+
+    # On cache miss, execute the collect phase and update the cache.
+    if not root_block_structure:
+
+        # Create the block structure from the modulestore.
+        root_block_structure = BlockStructureFactory.create_from_modulestore(root_block_usage_key, modulestore)
+
+        # Collect data from each registered transformer.
+        for transformer in TransformerRegistry.get_registered_transformers():
+            root_block_structure._add_transformer(transformer)  # pylint: disable=protected-access
+            transformer.collect(root_block_structure)
+
+        # Collect all fields that were requested by the transformers.
+        root_block_structure._collect_requested_xblock_fields()  # pylint: disable=protected-access
+
+        # Cache this information.
+        BlockStructureFactory.serialize_to_cache(root_block_structure, cache)
+
+    # Execute requested transforms on block structure.
+    for transformer in transformers:
+        transformer.transform(usage_info, root_block_structure)
+
+    # Prune the block structure to remove any unreachable blocks.
+    root_block_structure._prune_unreachable()  # pylint: disable=protected-access
+
+    return root_block_structure
+
+
+def clear_block_cache(cache, root_block_usage_key):
+    """
+    Removes the block structure associated with the given root block
+    key.
+    """
+    BlockStructureFactory.remove_from_cache(root_block_usage_key, cache)

openedx/core/lib/block_cache/block_structure_factory.py

+"""
+Module for factory class for BlockStructure objects.
+"""
+# pylint: disable=protected-access
+from logging import getLogger
+
+from openedx.core.lib.cache_utils import zpickle, zunpickle
+
+from .block_structure import BlockStructureBlockData, BlockStructureModulestoreData
+
+
+logger = getLogger(__name__)  # pylint: disable=C0103
+
+
+class BlockStructureFactory(object):
+    """
+    Factory class for BlockStructure objects.
+    """
+    @classmethod
+    def create_from_modulestore(cls, root_block_usage_key, modulestore):
+        """
+        Creates and returns a block structure from the modulestore
+        starting at the given root_block_usage_key.
+
+        Arguments:
+            root_block_usage_key (UsageKey) - The usage_key for the root
+                of the block structure that is to be created.
+
+            modulestore (ModuleStoreRead) - The modulestore that
+                contains the data for the xBlocks within the block
+                structure starting at root_block_usage_key.
+
+        Returns:
+            BlockStructureModulestoreData - The created block structure
+                with instantiated xBlocks from the given modulestore
+                starting at root_block_usage_key.
+        """
+        # Create block structure.
+        block_structure = BlockStructureModulestoreData(root_block_usage_key)
+
+        # Create internal set of blocks visited to use when recursing.
+        blocks_visited = set()
+
+        def build_block_structure(xblock):
+            """
+            Recursively update the block structure with the given xBlock
+            and its descendants.
+            """
+            # Check if the xblock was already visited (can happen in
+            # DAGs).
+            if xblock.location in blocks_visited:
+                return
+
+            # Add the xBlock.
+            blocks_visited.add(xblock.location)
+            block_structure._add_xblock(xblock.location, xblock)
+
+            # Add relations with its children and recurse.
+            for child in xblock.get_children():
+                block_structure._add_relation(xblock.location, child.location)
+                build_block_structure(child)
+
+        root_xblock = modulestore.get_item(root_block_usage_key, depth=None)
+        build_block_structure(root_xblock)
+        return block_structure
+
+    @classmethod
+    def serialize_to_cache(cls, block_structure, cache):
+        """
+        Store a compressed and pickled serialization of the given
+        block structure into the given cache.
+
+        The key in the cache is 'root.key.<root_block_usage_key>'.
+        The data stored in the cache includes the structure's
+        block relations, transformer data, and block data.
+
+        Arguments:
+            block_structure (BlockStructure) - The block structure
+                that is to be serialized to the given cache.
+
+            cache (django.core.cache.backends.base.BaseCache) - The
+                cache into which cacheable data of the block structure
+                is to be serialized.
+        """
+        data_to_cache = (
+            block_structure._block_relations,
+            block_structure._transformer_data,
+            block_structure._block_data_map
+        )
+        zp_data_to_cache = zpickle(data_to_cache)
+        cache.set(
+            cls._encode_root_cache_key(block_structure.root_block_usage_key),
+            zp_data_to_cache
+        )
+        logger.debug(
+            "Wrote BlockStructure %s to cache, size: %s",
+            block_structure.root_block_usage_key,
+            len(zp_data_to_cache),
+        )
+
+    @classmethod
+    def create_from_cache(cls, root_block_usage_key, cache, transformers):
+        """
+        Deserializes and returns the block structure starting at
+        root_block_usage_key from the given cache, if it's found in the cache.
+
+        The given root_block_usage_key must equate the root_block_usage_key
+        previously passed to serialize_to_cache.
+
+        Arguments:
+            root_block_usage_key (UsageKey) - The usage_key for the root
+                of the block structure that is to be deserialized from
+                the given cache.
+
+            cache (django.core.cache.backends.base.BaseCache) - The
+                cache from which the block structure is to be
+                deserialized.
+
+            transformers ([BlockStructureTransformer]) - A list of
+                transformers for which the block structure will be
+                transformed.
+
+        Returns:
+            BlockStructure - The deserialized block structure starting
+            at root_block_usage_key, if found in the cache.
+
+            NoneType - If the root_block_usage_key is not found in the cache
+            or if the cached data is outdated for one or more of the
+            given transformers.
+        """
+
+        # Find root_block_usage_key in the cache.
+        zp_data_from_cache = cache.get(cls._encode_root_cache_key(root_block_usage_key))
+        if not zp_data_from_cache:
+            logger.debug(
+                "BlockStructure %r not found in the cache.",
+                root_block_usage_key,
+            )
+            return None
+        else:
+            logger.debug(
+                "Read BlockStructure %r from cache, size: %s",
+                root_block_usage_key,
+                len(zp_data_from_cache),
+            )
+
+        # Deserialize and construct the block structure.
+        block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache)
+        block_structure = BlockStructureBlockData(root_block_usage_key)
+        block_structure._block_relations = block_relations
+        block_structure._transformer_data = transformer_data
+        block_structure._block_data_map = block_data_map
+
+        # Verify that the cached data for all the given transformers are
+        # for their latest versions.
+        outdated_transformers = {}
+        for transformer in transformers:
+            cached_transformer_version = block_structure._get_transformer_data_version(transformer)
+            if transformer.VERSION != cached_transformer_version:
+                outdated_transformers[transformer.name()] = "version: {}, cached: {}".format(
+                    transformer.VERSION,
+                    cached_transformer_version,
+                )
+        if outdated_transformers:
+            logger.info(
+                "Collected data for the following transformers are outdated:\n%s.",
+                '\n'.join([t_name + ": " + t_value for t_name, t_value in outdated_transformers.iteritems()]),
+            )
+            return None
+
+        return block_structure
+
+    @classmethod
+    def remove_from_cache(cls, root_block_usage_key, cache):
+        """
+        Removes the block structure for the given root_block_usage_key
+        from the given cache.
+
+        Arguments:
+            root_block_usage_key (UsageKey) - The usage_key for the root
+                of the block structure that is to be removed from
+                the given cache.
+
+            cache (django.core.cache.backends.base.BaseCache) - The
+                cache from which the block structure is to be
+                removed.
+        """
+        cache.delete(cls._encode_root_cache_key(root_block_usage_key))
+        # TODO also remove all block data?
+
+    @classmethod
+    def _encode_root_cache_key(cls, root_block_usage_key):
+        """
+        Returns the cache key to use for storing the block structure
+        for the given root_block_usage_key.
+        """
+        return "root.key." + unicode(root_block_usage_key)

openedx/core/lib/block_cache/exceptions.py

+"""
+Application-specific exceptions raised by the block cache framework.
+"""
+
+
+class TransformerException(Exception):
+    """
+    Exception class for Transformer related errors.
+    """
+    pass

openedx/core/lib/block_cache/tests/test_block_cache.py

+"""
+Tests for block_cache.py
+"""
+
+from django.core.cache import get_cache
+from mock import patch
+from unittest import TestCase
+
+from ..block_cache import get_blocks
+from ..exceptions import TransformerException
+from .test_utils import (
+    MockModulestoreFactory, MockCache, MockTransformer, ChildrenMapTestMixin
+)
+
+
+@patch('openedx.core.lib.block_cache.transformer_registry.TransformerRegistry.get_available_plugins')
+class TestBlockCache(TestCase, ChildrenMapTestMixin):
+    """
+    Test class for block cache functionality.
+    """
+
+    class TestTransformer1(MockTransformer):
+        """
+        Test Transformer class.
+        """
+        @classmethod
+        def block_key(cls):
+            """
+            Returns the dictionary key for transformer block data.
+            """
+            return 't1.key1'
+
+        @classmethod
+        def block_val(cls, block_key):
+            """
+            Returns the dictionary value for transformer block data for
+            the block identified by the given block key.
+            """
+            return 't1.val1.' + unicode(block_key)
+
+        @classmethod
+        def collect(cls, block_structure):
+            """
+            Sets transformer block data for each block in the structure
+            as it is visited using topological traversal.
+            """
+            for block_key in block_structure.topological_traversal():
+                block_structure.set_transformer_block_field(
+                    block_key, cls, cls.block_key(), cls.block_val(block_key)
+                )
+
+        def transform(self, usage_info, block_structure):
+            """
+            Verifies the transformer block data set for each block
+            in the structure.
+            """
+            def assert_collected_value(block_key):
+                """
+                Verifies the transformer block data for the given
+                block equates the value stored in the collect method.
+                """
+                assert (
+                    block_structure.get_transformer_block_field(
+                        block_key,
+                        self,
+                        self.block_key()
+                    ) == self.block_val(block_key)
+                )
+
+            for block_key in block_structure.topological_traversal():
+                assert_collected_value(block_key)
+
+    def setUp(self):
+        super(TestBlockCache, self).setUp()
+        self.children_map = self.SIMPLE_CHILDREN_MAP
+        self.usage_info = None
+        self.mock_cache = MockCache()
+        self.modulestore = MockModulestoreFactory.create(self.children_map)
+        self.transformers = [self.TestTransformer1()]
+
+    def test_get_blocks(self, mock_available_transforms):
+        mock_available_transforms.return_value = {transformer.name(): transformer for transformer in self.transformers}
+        block_structure = get_blocks(
+            self.mock_cache, self.modulestore, self.usage_info, root_block_usage_key=0, transformers=self.transformers
+        )
+        self.assert_block_structure(block_structure, self.children_map)
+
+    def test_unregistered_transformers(self, mock_available_transforms):
+        mock_available_transforms.return_value = {}
+        with self.assertRaisesRegexp(TransformerException, "requested transformers are not registered"):
+            get_blocks(
+                self.mock_cache,
+                self.modulestore,
+                self.usage_info,
+                root_block_usage_key=0,
+                transformers=self.transformers,
+            )
+
+    def test_block_caching(self, mock_available_transforms):
+        mock_available_transforms.return_value = {transformer.name(): transformer for transformer in self.transformers}
+
+        cache = get_cache('block_cache')
+
+        for iteration in range(2):
+            self.modulestore.get_items_call_count = 0
+            block_structure = get_blocks(
+                cache, self.modulestore, self.usage_info, root_block_usage_key=0, transformers=self.transformers
+            )
+            self.assert_block_structure(block_structure, self.children_map)
+            if iteration == 0:
+                self.assertGreater(self.modulestore.get_items_call_count, 0)
+            else:
+                self.assertEquals(self.modulestore.get_items_call_count, 0)

openedx/core/lib/block_cache/tests/test_block_structure.py

+"""
+Tests for block_structure.py
+"""
+# pylint: disable=protected-access
+from collections import namedtuple
+from copy import deepcopy
+import ddt
+import itertools
+from unittest import TestCase
+
+from openedx.core.lib.graph_traversals import traverse_post_order
+
+from ..block_structure import BlockStructure, BlockStructureModulestoreData, BlockStructureBlockData
+from ..exceptions import TransformerException
+from .test_utils import MockXBlock, MockTransformer, ChildrenMapTestMixin
+
+
+@ddt.ddt
+class TestBlockStructure(TestCase, ChildrenMapTestMixin):
+    """
+    Tests for BlockStructure
+    """
+    @ddt.data(
+        [],
+        ChildrenMapTestMixin.SIMPLE_CHILDREN_MAP,
+        ChildrenMapTestMixin.LINEAR_CHILDREN_MAP,
+        ChildrenMapTestMixin.DAG_CHILDREN_MAP,
+    )
+    def test_relations(self, children_map):
+        block_structure = self.create_block_structure(BlockStructure, children_map)
+
+        # get_children
+        for parent, children in enumerate(children_map):
+            self.assertSetEqual(set(block_structure.get_children(parent)), set(children))
+
+        # get_parents
+        for child, parents in enumerate(self.get_parents_map(children_map)):
+            self.assertSetEqual(set(block_structure.get_parents(child)), set(parents))
+
+        # has_block
+        for node in range(len(children_map)):
+            self.assertTrue(block_structure.has_block(node))
+        self.assertFalse(block_structure.has_block(len(children_map) + 1))
+
+
+@ddt.ddt
+class TestBlockStructureData(TestCase, ChildrenMapTestMixin):
+    """
+    Tests for BlockStructureBlockData and BlockStructureModulestoreData
+    """
+    def test_non_versioned_transformer(self):
+        class TestNonVersionedTransformer(MockTransformer):
+            """
+            Test transformer with default version number (0).
+            """
+            VERSION = 0
+
+        block_structure = BlockStructureModulestoreData(root_block_usage_key=0)
+
+        with self.assertRaisesRegexp(TransformerException, "VERSION attribute is not set"):
+            block_structure._add_transformer(TestNonVersionedTransformer())
+
+    def test_transformer_data(self):
+        # transformer test cases
+        TransformerInfo = namedtuple("TransformerInfo", "transformer structure_wide_data block_specific_data")  # pylint: disable=invalid-name
+        transformers_info = [
+            TransformerInfo(
+                transformer=MockTransformer(),
+                structure_wide_data=[("t1.global1", "t1.g.val1"), ("t1.global2", "t1.g.val2")],
+                block_specific_data={
+                    "B1": [("t1.key1", "t1.b1.val1"), ("t1.key2", "t1.b1.val2")],
+                    "B2": [("t1.key1", "t1.b2.val1"), ("t1.key2", "t1.b2.val2")],
+                    "B3": [("t1.key1", True), ("t1.key2", False)],
+                    "B4": [("t1.key1", None), ("t1.key2", False)],
+                },
+            ),
+            TransformerInfo(
+                transformer=MockTransformer(),
+                structure_wide_data=[("t2.global1", "t2.g.val1"), ("t2.global2", "t2.g.val2")],
+                block_specific_data={
+                    "B1": [("t2.key1", "t2.b1.val1"), ("t2.key2", "t2.b1.val2")],
+                    "B2": [("t2.key1", "t2.b2.val1"), ("t2.key2", "t2.b2.val2")],
+                },
+            ),
+        ]
+
+        # create block structure
+        block_structure = BlockStructureModulestoreData(root_block_usage_key=0)
+
+        # set transformer data
+        for t_info in transformers_info:
+            block_structure._add_transformer(t_info.transformer)
+            for key, val in t_info.structure_wide_data:
+                block_structure.set_transformer_data(t_info.transformer, key, val)
+            for block, block_data in t_info.block_specific_data.iteritems():
+                for key, val in block_data:
+                    block_structure.set_transformer_block_field(block, t_info.transformer, key, val)
+
+        # verify transformer data
+        for t_info in transformers_info:
+            self.assertEquals(
+                block_structure._get_transformer_data_version(t_info.transformer),
+                MockTransformer.VERSION
+            )
+            for key, val in t_info.structure_wide_data:
+                self.assertEquals(
+                    block_structure.get_transformer_data(t_info.transformer, key),
+                    val,
+                )
+            for block, block_data in t_info.block_specific_data.iteritems():
+                for key, val in block_data:
+                    self.assertEquals(
+                        block_structure.get_transformer_block_field(block, t_info.transformer, key),
+                        val,
+                    )
+
+    def test_xblock_data(self):
+        # block test cases
+        blocks = [
+            MockXBlock("A", {}),
+            MockXBlock("B", {"field1": "B.val1"}),
+            MockXBlock("C", {"field1": "C.val1", "field2": "C.val2"}),
+            MockXBlock("D", {"field1": True, "field2": False}),
+            MockXBlock("E", {"field1": None, "field2": False}),
+        ]
+
+        # add each block
+        block_structure = BlockStructureModulestoreData(root_block_usage_key=0)
+        for block in blocks:
+            block_structure._add_xblock(block.location, block)
+
+        # request fields
+        fields = ["field1", "field2", "field3"]
+        block_structure.request_xblock_fields(*fields)
+
+        # verify fields have not been collected yet
+        for block in blocks:
+            for field in fields:
+                self.assertIsNone(block_structure.get_xblock_field(block.location, field))
+
+        # collect fields
+        block_structure._collect_requested_xblock_fields()
+
+        # verify values of collected fields
+        for block in blocks:
+            for field in fields:
+                self.assertEquals(
+                    block_structure.get_xblock_field(block.location, field),
+                    block.field_map.get(field),
+                )
+
+    @ddt.data(
+        *itertools.product(
+            [True, False],
+            range(7),
+            [
+                ChildrenMapTestMixin.SIMPLE_CHILDREN_MAP,
+                ChildrenMapTestMixin.LINEAR_CHILDREN_MAP,
+                ChildrenMapTestMixin.DAG_CHILDREN_MAP,
+            ],
+        )
+    )
+    @ddt.unpack
+    def test_remove_block(self, keep_descendants, block_to_remove, children_map):
+        ### skip test if invalid
+        if (block_to_remove >= len(children_map)) or (keep_descendants and block_to_remove == 0):
+            return
+
+        ### create structure
+        block_structure = self.create_block_structure(BlockStructureBlockData, children_map)
+        parents_map = self.get_parents_map(children_map)
+
+        ### verify blocks pre-exist
+        self.assert_block_structure(block_structure, children_map)
+
+        ### remove block
+        block_structure.remove_block(block_to_remove, keep_descendants)
+        missing_blocks = [block_to_remove]
+
+        ### compute and verify updated children_map
+        removed_children_map = deepcopy(children_map)
+        removed_children_map[block_to_remove] = []
+        for parent in parents_map[block_to_remove]:
+            removed_children_map[parent].remove(block_to_remove)
+
+        if keep_descendants:
+            # update the graph connecting the old parents to the old children
+            for child in children_map[block_to_remove]:
+                for parent in parents_map[block_to_remove]:
+                    removed_children_map[parent].append(child)
+
+        self.assert_block_structure(block_structure, removed_children_map, missing_blocks)
+
+        ### prune the structure
+        block_structure._prune_unreachable()
+
+        ### compute and verify updated children_map
+        pruned_children_map = deepcopy(removed_children_map)
+
+        if not keep_descendants:
+            pruned_parents_map = self.get_parents_map(pruned_children_map)
+            # update all descendants
+            for child in children_map[block_to_remove]:
+                # if the child has another parent, continue
+                if pruned_parents_map[child]:
+                    continue
+                for block in traverse_post_order(child, get_children=lambda block: pruned_children_map[block]):
+                    # add descendant to missing blocks and empty its
+                    # children
+                    missing_blocks.append(block)
+                    pruned_children_map[block] = []
+
+        self.assert_block_structure(block_structure, pruned_children_map, missing_blocks)
+
+    def test_remove_block_if(self):
+        block_structure = self.create_block_structure(BlockStructureBlockData, ChildrenMapTestMixin.LINEAR_CHILDREN_MAP)
+        block_structure.remove_block_if(lambda block: block == 2)
+        self.assert_block_structure(block_structure, [[1], [], [], []], missing_blocks=[2])

openedx/core/lib/block_cache/tests/test_block_structure_factory.py

+"""
+Tests for block_structure_factory.py
+"""
+# pylint: disable=protected-access
+from mock import patch
+from unittest import TestCase
+
+from ..block_structure_factory import BlockStructureFactory
+from .test_utils import (
+    MockCache, MockModulestoreFactory, MockTransformer, ChildrenMapTestMixin
+)
+
+
+class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin):
+    """
+    Tests for BlockStructureFactory
+    """
+    def setUp(self):
+        super(TestBlockStructureFactory, self).setUp()
+        self.children_map = self.SIMPLE_CHILDREN_MAP
+        self.modulestore = MockModulestoreFactory.create(self.children_map)
+
+        self.block_structure = BlockStructureFactory.create_from_modulestore(
+            root_block_usage_key=0, modulestore=self.modulestore
+        )
+
+        self.transformers = [MockTransformer]
+        mock_registry = patch(
+            'openedx.core.lib.block_cache.transformer_registry.TransformerRegistry.get_available_plugins'
+        )
+        mock_registry.return_value = {transformer.name(): transformer for transformer in self.transformers}
+        self.addCleanup(mock_registry.stop)
+        mock_registry.start()
+
+    def add_transformers(self):
+        """
+        Add each registered transformer to the block structure.
+        Mimic collection by setting test transformer block data.
+        """
+        for transformer in self.transformers:
+            self.block_structure._add_transformer(transformer)
+            self.block_structure.set_transformer_block_field(
+                usage_key=0, transformer=transformer, key='test', value='{} val'.format(transformer.name())
+            )
+
+    def test_create_from_modulestore(self):
+        self.assert_block_structure(self.block_structure, self.children_map)
+
+    def test_not_in_cache(self):
+        cache = MockCache()
+
+        self.assertIsNone(
+            BlockStructureFactory.create_from_cache(
+                root_block_usage_key=0,
+                cache=cache,
+                transformers=self.transformers,
+            )
+        )
+
+    def test_uncollected_transformers(self):
+        cache = MockCache()
+
+        # serialize the structure to cache, but without collecting any transformer data
+        BlockStructureFactory.serialize_to_cache(self.block_structure, cache)
+
+        with patch('openedx.core.lib.block_cache.block_structure_factory.logger.info') as mock_logger:
+            # cached data does not have collected information for all registered transformers
+            self.assertIsNone(
+                BlockStructureFactory.create_from_cache(
+                    root_block_usage_key=0,
+                    cache=cache,
+                    transformers=self.transformers,
+                )
+            )
+            self.assertTrue(mock_logger.called)
+
+    def test_cache(self):
+        cache = MockCache()
+
+        # collect transformer data
+        self.add_transformers()
+
+        # serialize to cache
+        BlockStructureFactory.serialize_to_cache(self.block_structure, cache)
+
+        # test re-create from cache
+        self.modulestore.get_items_call_count = 0
+        from_cache_block_structure = BlockStructureFactory.create_from_cache(
+            root_block_usage_key=0,
+            cache=cache,
+            transformers=self.transformers,
+        )
+        self.assertIsNotNone(from_cache_block_structure)
+        self.assert_block_structure(from_cache_block_structure, self.children_map)
+        self.assertEquals(self.modulestore.get_items_call_count, 0)
+
+    def test_remove_from_cache(self):
+        cache = MockCache()
+
+        # collect transformer data
+        self.add_transformers()
+
+        # serialize to cache
+        BlockStructureFactory.serialize_to_cache(self.block_structure, cache)
+
+        # remove from cache
+        BlockStructureFactory.remove_from_cache(root_block_usage_key=0, cache=cache)
+        self.assertIsNone(
+            BlockStructureFactory.create_from_cache(
+                root_block_usage_key=0,
+                cache=cache,
+                transformers=self.transformers
+            )
+        )

openedx/core/lib/block_cache/tests/test_transformer_registry.py

+"""
+Tests for transformer_registry.py
+"""
+
+import ddt
+from mock import patch
+from unittest import TestCase
+
+from ..transformer_registry import TransformerRegistry
+from .test_utils import MockTransformer
+
+
+class TestTransformer1(MockTransformer):
+    """
+    1st test instance of the MockTransformer that is registered.
+    """
+    pass
+
+
+class TestTransformer2(MockTransformer):
+    """
+    2nd test instance of the MockTransformer that is registered.
+    """
+    pass
+
+
+class UnregisteredTestTransformer3(MockTransformer):
+    """
+    3rd test instance of the MockTransformer that is not registered.
+    """
+    pass
+
+
+@ddt.ddt
+class TransformerRegistryTestCase(TestCase):
+    """
+    Test cases for TransformerRegistry.
+    """
+    @ddt.data(
+        # None case
+        ([], []),
+
+        # 1 registered
+        ([TestTransformer1()], []),
+
+        # 2 registered
+        ([TestTransformer1(), TestTransformer2()], []),
+
+        # 1 unregistered
+        ([UnregisteredTestTransformer3()], [UnregisteredTestTransformer3.name()]),
+
+        # 1 registered and 1 unregistered
+        ([TestTransformer1(), UnregisteredTestTransformer3()], [UnregisteredTestTransformer3.name()]),
+    )
+    @ddt.unpack
+    def test_find_unregistered(self, transformers, expected_unregistered):
+
+        with (
+            patch('openedx.core.lib.block_cache.transformer_registry.TransformerRegistry.get_available_plugins')
+        ) as mock_registry:
+            mock_registry.return_value = {
+                transformer.name(): transformer
+                for transformer in [TestTransformer1, TestTransformer2]
+            }
+
+            self.assertSetEqual(
+                TransformerRegistry.find_unregistered(transformers), set(expected_unregistered)
+            )

openedx/core/lib/block_cache/tests/test_utils.py

+"""
+Common utilities for tests in block_cache module
+"""
+# pylint: disable=protected-access
+from ..transformer import BlockStructureTransformer
+
+
+class MockXBlock(object):
+    """
+    A mock XBlock to be used in unit tests, thereby decoupling the
+    implementation of the block cache framework from the xBlock
+    implementation.  This class provides only the minimum xBlock
+    capabilities needed by the block cache framework.
+    """
+    def __init__(self, location, field_map=None, children=None, modulestore=None):
+        self.location = location
+        self.field_map = field_map or {}
+
+        self.children = children or []
+        self.modulestore = modulestore
+
+    def __getattr__(self, attr):
+        try:
+            return self.field_map[attr]
+        except KeyError:
+            raise AttributeError
+
+    def get_children(self):
+        """
+        Returns the children of the mock XBlock.
+        """
+        return [self.modulestore.get_item(child) for child in self.children]
+
+
+class MockModulestore(object):
+    """
+    A mock Modulestore to be used in unit tests, providing only the
+    minimum methods needed by the block cache framework.
+    """
+    def __init__(self):
+        self.get_items_call_count = 0
+        self.blocks = None
+
+    def set_blocks(self, blocks):
+        """
+        Updates the mock modulestore with a dictionary of blocks.
+
+        Arguments:
+            blocks ({block key, MockXBlock}) - A map of block_key
+            to its mock xBlock.
+        """
+        self.blocks = blocks
+
+    def get_item(self, block_key, depth=None):  # pylint: disable=unused-argument
+        """
+        Returns the mock XBlock (MockXBlock) associated with the
+        given block_key.
+        """
+        self.get_items_call_count += 1
+        return self.blocks.get(block_key)
+
+
+class MockCache(object):
+    """
+    A mock Cache object, providing only the minimum features needed
+    by the block cache framework.
+    """
+    def __init__(self):
+        # An in-memory map of cache keys to cache values.
+        self.map = {}
+
+    def set(self, key, val):
+        """
+        Associates the given key with the given value in the cache.
+        """
+        self.map[key] = val
+
+    def get(self, key, default=None):
+        """
+        Returns the value associated with the given key in the cache;
+        returns default if not found.
+        """
+        return self.map.get(key, default)
+
+    def set_many(self, map_):
+        """
+        For each dictionary entry in the given map, updates the cache
+        with that entry.
+        """
+        for key, val in map_.iteritems():
+            self.set(key, val)
+
+    def get_many(self, keys):
+        """
+        Returns a dictionary of entries for each key found in the cache.
+        """
+        return {key: self.map[key] for key in keys if key in self.map}
+
+    def delete(self, key):
+        """
+        Deletes the given key from the cache.
+        """
+        del self.map[key]
+
+
+class MockModulestoreFactory(object):
+    """
+    A factory for creating MockModulestore objects.
+    """
+    @classmethod
+    def create(cls, children_map):
+        """
+        Creates and returns a MockModulestore from the given
+        children_map.
+
+        Arguments:
+            children_map ({block_key: [block_key]}) - A dictionary
+                mapping a block key to a list of block keys of the
+                block's corresponding children.
+        """
+        modulestore = MockModulestore()
+        modulestore.set_blocks({
+            block_key: MockXBlock(block_key, children=children, modulestore=modulestore)
+            for block_key, children in enumerate(children_map)
+        })
+        return modulestore
+
+
+class MockTransformer(BlockStructureTransformer):
+    """
+    A mock BlockStructureTransformer class.
+    """
+    VERSION = 1
+
+    @classmethod
+    def name(cls):
+        # Use the class' name for Mock transformers.
+        return cls.__name__
+
+    def transform(self, usage_info, block_structure):
+        pass
+
+
+class ChildrenMapTestMixin(object):
+    """
+    A Test Mixin with utility methods for testing with block structures
+    created and manipulated using children_map and parents_map.
+    """
+
+    #     0
+    #    / \
+    #   1  2
+    #  / \
+    # 3   4
+    SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
+
+    #       0
+    #      /
+    #     1
+    #    /
+    #   2
+    #  /
+    # 3
+    LINEAR_CHILDREN_MAP = [[1], [2], [3], []]
+
+    #     0
+    #    / \
+    #   1  2
+    #   \ / \
+    #    3  4
+    #   / \
+    #  5  6
+    DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []]
+
+    def create_block_structure(self, block_structure_cls, children_map):
+        """
+        Factory method for creating and returning a block structure
+        for the given children_map.
+        """
+        # create empty block structure
+        block_structure = block_structure_cls(root_block_usage_key=0)
+
+        # _add_relation
+        for parent, children in enumerate(children_map):
+            for child in children:
+                block_structure._add_relation(parent, child)
+        return block_structure
+
+    def get_parents_map(self, children_map):
+        """
+        Converts and returns the given children_map to a parents_map.
+        """
+        parent_map = [[] for _ in children_map]
+        for parent, children in enumerate(children_map):
+            for child in children:
+                parent_map[child].append(parent)
+        return parent_map
+
+    def assert_block_structure(self, block_structure, children_map, missing_blocks=None):
+        """
+        Verifies that the relations in the given block structure
+        equate the relations described in the children_map. Use the
+        missing_blocks parameter to pass in any blocks that were removed
+        from the block structure but still have a positional entry in
+        the children_map.
+        """
+        if not missing_blocks:
+            missing_blocks = []
+
+        for block_key, children in enumerate(children_map):
+            # Verify presence
+            self.assertEquals(
+                block_structure.has_block(block_key),
+                block_key not in missing_blocks,
+                'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format(
+                    unicode(block_key)
+                ),
+            )
+
+            # Verify children
+            if block_key not in missing_blocks:
+                self.assertEquals(
+                    set(block_structure.get_children(block_key)),
+                    set(children),
+                )
+
+        # Verify parents
+        parents_map = self.get_parents_map(children_map)
+        for block_key, parents in enumerate(parents_map):
+            if block_key not in missing_blocks:
+                self.assertEquals(
+                    set(block_structure.get_parents(block_key)),
+                    set(parents),
+                )

openedx/core/lib/block_cache/transformer.py

+"""
+This module provides the abstract base class for all Block Structure
+Transformers.
+"""
+from abc import abstractmethod
+
+
+class BlockStructureTransformer(object):
+    """
+    Abstract base class for all block structure transformers.
+    """
+
+    # All Transformers are expected to maintain a VERSION class
+    # attribute.  While the value for the base class is set to 0,
+    # the value for each concrete transformer should be 1 or higher.
+    #
+    # A transformer's version attribute is used by the block_cache
+    # framework in order to determine whether any collected data for a
+    # transformer is outdated.  When a transformer's data is collected
+    # and cached, it's version number at the time of collection is
+    # stored along with the data.  That version number is then checked
+    # at the time of accessing the collected data (during the transform
+    # phase).
+    #
+    # The version number of a Transformer should be incremented each
+    # time the implementation of its collect method is updated such that
+    # its collected data is changed.
+    #
+    VERSION = 0
+
+    @classmethod
+    def name(cls):
+        """
+        Unique identifier for the transformer's class. It is used to
+        identify the transformer's cached data. So it should be unique
+        and not conflict with other transformers. Consider using the
+        same name that is used in the Transformer Registry. For example,
+        for Stevedore, it is specified in the setup.py file.
+
+        Once the transformer is in use and its data is cached, do not
+        modify this name value without consideration of backward
+        compatibility with previously collected data.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def collect(cls, block_structure):
+        """
+        Collects and stores any xBlock and modulestore data into the
+        block_structure that's necessary for later execution of the
+        transformer's transform method. Transformers should store such
+        data in the block_structure using the following methods:
+            set_transformer_data
+            set_transformer_block_field
+            request_xblock_fields
+
+        Transformers can call block_structure.request_xblock_fields for
+        any common xBlock fields that should be collected by the
+        framework.
+
+        Any full block tree traversals should be implemented in this
+        collect phase, leaving the transform phase for fast and direct
+        access to a sub-block. If a block's transform output is
+        dependent on its ancestors' data, the ancestor's data should be
+        percolated down to the descendants. So when a (non-root) block
+        is directly accessed in the transform, all of its relevant data
+        is readily available (without needing to access its ancestors).
+
+        Traversals of the block_structure can be implemented using the
+        following methods:
+            topological_traversal
+            post_order_traversal
+
+        Arguments:
+            block_structure (BlockStructureModulestoreData) - A mutable
+                block structure that is to be modified with collected
+                data to be cached for the transformer.
+        """
+        pass
+
+    @abstractmethod
+    def transform(self, usage_info, block_structure):
+        """
+        Transforms the given block_structure for the given usage_info,
+        assuming the block_structure contains cached data from a prior
+        call to the collect method of the latest version of the
+        Transformer.
+
+        No access to the modulestore nor instantiation of xBlocks should
+        be performed during the execution of this method. However,
+        accesses to user-specific data (outside of the modulestore and
+        not via xBlocks) is permitted in order to apply the transform
+        for the given usage_info.
+
+        Note: The root of the given block_structure is not necessarily
+        the same as the root of the block_structure passed to the prior
+        collect method. The collect method is given the top-most root
+        of the structure, while the transform method may be called upon
+        any sub-structure or even a single block within the originally
+        collected structure.
+
+        A Transformer may choose to remove entire sub-structures during
+        the transform method and may do so using the remove_block and
+        remove_block_if methods.
+
+        Amongst the many methods available for a block_structure, the
+        following methods are commonly used during transforms:
+            get_xblock_field
+            get_transformer_data
+            get_transformer_block_field
+            remove_block
+            remove_block_if
+            topological_traversal
+            post_order_traversal
+
+        Arguments:
+            usage_info (any negotiated type) - A usage-specific object
+                that is passed to the block_cache and forwarded to all
+                requested Transformers in order to apply a
+                usage-specific transform. For example, an instance of
+                usage_info would contain a user object for which the
+                transform should be applied.
+
+            block_structure (BlockStructureBlockData) - A mutable
+                block structure, with already collected data for the
+                transformer, that is to be transformed in place.
+        """
+        pass

openedx/core/lib/block_cache/transformer_registry.py

+"""
+Block Structure Transformer Registry implemented using the platform's
+PluginManager.
+"""
+from openedx.core.lib.api.plugins import PluginManager
+
+
+class TransformerRegistry(PluginManager):
+    """
+    Registry for all of the block structure transformers that have been
+    made available.
+
+    All block structure transformers should implement
+    `BlockStructureTransformer`.
+    """
+    NAMESPACE = 'openedx.block_structure_transformer'
+
+    @classmethod
+    def get_registered_transformers(cls):
+        """
+        Returns a set of all registered transformers.
+
+        Returns:
+            {BlockStructureTransformer} - All transformers that are
+                registered with the platform's PluginManager.
+        """
+        return set(cls.get_available_plugins().itervalues())
+
+    @classmethod
+    def find_unregistered(cls, transformers):
+        """
+        Find and returns the names of all the transformers from the
+        given list that aren't registered with the platform's
+        PluginManager.
+
+        Arguments:
+            transformers ([BlockStructureTransformer] - List of
+                transformers to check in the registry.
+
+        Returns:
+            [string] - The names of a subset of the given
+                transformers that weren't found in the registry.
+        """
+        registered_transformer_names = set(reg_trans.name() for reg_trans in cls.get_registered_transformers())
+        requested_transformer_names = set(transformer.name() for transformer in transformers)
+        return requested_transformer_names - registered_transformer_names

openedx/core/lib/cache_utils.py

 """
 Utilities related to caching.
 """
-
+import cPickle as pickle
 import functools
+import zlib
 from xblock.core import XBlock
 
 
@@ -47,3 +48,13 @@ def hashvalue(arg):
         return unicode(arg.location)
     else:
         return unicode(arg)
+
+
+def zpickle(data):
+    """Given any data structure, returns a zlib compressed pickled serialization."""
+    return zlib.compress(pickle.dumps(data, pickle.HIGHEST_PROTOCOL))
+
+
+def zunpickle(zdata):
+    """Given a zlib compressed pickled serialization, returns the deserialized data."""
+    return pickle.loads(zlib.decompress(zdata))