Consolidate Block Structure folders

TNL-6518

lms/djangoapps/course_api/blocks/api.py

@@ -4,7 +4,7 @@ API function for retrieving course blocks data
 
 from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACCESS_TRANSFORMERS
 from lms.djangoapps.course_blocks.transformers.hidden_content import HiddenContentTransformer
-from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
+from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
 
 from .transformers.blocks_api import BlocksAPITransformer
 from .transformers.milestones import MilestonesTransformer

lms/djangoapps/course_api/blocks/tests/test_serializers.py

@@ -3,7 +3,7 @@ Tests for Course Blocks serializers
 """
 from mock import MagicMock
 
-from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
+from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
 from student.tests.factories import UserFactory
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase

lms/djangoapps/course_api/blocks/transformers/block_counts.py

 """
 Block Counts Transformer
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
+from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
 
 
 class BlockCountsTransformer(BlockStructureTransformer):

lms/djangoapps/course_api/blocks/transformers/block_depth.py

 """
 Block Depth Transformer
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
+from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
 
 
 class BlockDepthTransformer(BlockStructureTransformer):

lms/djangoapps/course_api/blocks/transformers/blocks_api.py

 """
 Blocks API Transformer
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
+from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
 from .block_counts import BlockCountsTransformer
 from .block_depth import BlockDepthTransformer
 from .navigation import BlockNavigationTransformer

lms/djangoapps/course_api/blocks/transformers/milestones.py

@@ -4,7 +4,10 @@ Milestones Transformer
 
 from django.conf import settings
 
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 from util import milestones_helpers
 
 

lms/djangoapps/course_api/blocks/transformers/navigation.py

 """
 TODO
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
+from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
 from .block_depth import BlockDepthTransformer
 
 

lms/djangoapps/course_api/blocks/transformers/student_view.py

 """
 Student View Transformer
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
+from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
 
 
 class StudentViewTransformer(BlockStructureTransformer):

lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py

@@ -3,7 +3,7 @@ Tests for BlockCountsTransformer.
 """
 
 # pylint: disable=protected-access
-from openedx.core.lib.block_structure.factory import BlockStructureFactory
+from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import SampleCourseFactory
 

lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py

@@ -7,8 +7,8 @@ Tests for BlockDepthTransformer.
 import ddt
 from unittest import TestCase
 
-from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
-from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
+from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin
+from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData
 from ..block_depth import BlockDepthTransformer
 
 

lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py

@@ -7,9 +7,9 @@ from unittest import TestCase
 
 from lms.djangoapps.course_api.blocks.transformers.block_depth import BlockDepthTransformer
 from lms.djangoapps.course_api.blocks.transformers.navigation import BlockNavigationTransformer
-from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
-from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
-from openedx.core.lib.block_structure.factory import BlockStructureFactory
+from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin
+from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData
+from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import SampleCourseFactory
 from xmodule.modulestore import ModuleStoreEnum

lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py

@@ -4,7 +4,7 @@ Tests for StudentViewTransformer.
 
 # pylint: disable=protected-access
 
-from openedx.core.lib.block_structure.factory import BlockStructureFactory
+from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import ToyCourseFactory
 

lms/djangoapps/course_blocks/__init__.py

 """
 The Course Blocks app, built upon the Block Cache framework in
-openedx.core.lib.block_structure, is a higher layer django app in LMS that
+openedx.core.djangoapps.content.block_structure, is a higher layer django app in LMS that
 provides additional context of Courses and Users (via usage_info.py) with
 implementations for Block Structure Transformers that are related to
 block structure course access.

lms/djangoapps/course_blocks/api.py

@@ -3,7 +3,7 @@ API entry point to the course_blocks app with top-level
 get_course_blocks function.
 """
 from openedx.core.djangoapps.content.block_structure.api import get_block_structure_manager
-from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
+from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
 
 from .transformers import (
     library_content,

lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py

@@ -8,7 +8,7 @@ from xmodule.modulestore.django import modulestore
 
 import openedx.core.djangoapps.content.block_structure.api as api
 import openedx.core.djangoapps.content.block_structure.tasks as tasks
-import openedx.core.lib.block_structure.store as store
+import openedx.core.djangoapps.content.block_structure.store as store
 from openedx.core.lib.command_utils import (
     get_mutually_exclusive_required_option,
     validate_dependent_option,

lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py

@@ -59,7 +59,7 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
         self.command.handle(all_courses=True)
         self._assert_courses_in_block_cache(*self.course_keys)
         with patch(
-            'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_modulestore'
+            'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore'
         ) as mock_update_from_store:
             self.command.handle(all_courses=True, force_update=force_update)
             self.assertEqual(mock_update_from_store.call_count, self.num_courses if force_update else 0)

lms/djangoapps/course_blocks/transformers/hidden_content.py

@@ -4,7 +4,10 @@ Visibility Transformer implementation.
 from datetime import datetime
 from pytz import utc
 
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 from xmodule.seq_module import SequenceModule
 from .utils import collect_merged_boolean_field, collect_merged_date_field
 

lms/djangoapps/course_blocks/transformers/library_content.py

@@ -3,7 +3,10 @@ Content Library Transformer.
 """
 import json
 from courseware.models import StudentModule
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 from xmodule.library_content_module import LibraryContentModule
 from xmodule.modulestore.django import modulestore
 from eventtracking import tracker

lms/djangoapps/course_blocks/transformers/split_test.py

 """
 Split Test Block Transformer
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 
 
 class SplitTestTransformer(FilteringTransformerMixin, BlockStructureTransformer):

lms/djangoapps/course_blocks/transformers/start_date.py

 """
 Start Date Transformer implementation.
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 from lms.djangoapps.courseware.access_utils import check_start_date
 from xmodule.course_metadata_utils import DEFAULT_START_DATE
 

lms/djangoapps/course_blocks/transformers/tests/helpers.py

@@ -4,8 +4,8 @@ Test helpers for testing course block transformers.
 from mock import patch
 from course_modes.models import CourseMode
 from lms.djangoapps.courseware.access import has_access
-from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
-from openedx.core.lib.block_structure.tests.helpers import clear_registered_transformers_cache
+from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
+from openedx.core.djangoapps.content.block_structure.tests.helpers import clear_registered_transformers_cache
 from student.tests.factories import CourseEnrollmentFactory, UserFactory
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.django import modulestore
@@ -23,7 +23,8 @@ class TransformerRegistryTestMixin(object):
     def setUp(self):
         super(TransformerRegistryTestMixin, self).setUp()
         self.patcher = patch(
-            'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
+            'openedx.core.djangoapps.content.block_structure.transformer_registry.'
+            'TransformerRegistry.get_registered_transformers'
         )
         mock_registry = self.patcher.start()
         mock_registry.return_value = {self.TRANSFORMER_CLASS_TO_TEST}

lms/djangoapps/course_blocks/transformers/tests/test_library_content.py

@@ -5,7 +5,7 @@ Tests for ContentLibraryTransformer.
 from student.tests.factories import CourseEnrollmentFactory
 
 from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache
-from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
+from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
 
 from ...api import get_course_blocks
 from ..library_content import ContentLibraryTransformer

lms/djangoapps/course_blocks/transformers/user_partitions.py

 """
 User Partitions Transformer
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 
 from .split_test import SplitTestTransformer
 from .utils import get_field_on_block

lms/djangoapps/course_blocks/transformers/visibility.py

 """
 Visibility Transformer implementation.
 """
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
+from openedx.core.djangoapps.content.block_structure.transformer import (
+    BlockStructureTransformer,
+    FilteringTransformerMixin,
+)
 from .utils import collect_merged_boolean_field
 
 

lms/djangoapps/grades/tests/test_grades.py

@@ -12,7 +12,7 @@ from courseware.model_data import set_score
 from courseware.tests.helpers import LoginEnrollmentTestCase
 
 from lms.djangoapps.course_blocks.api import get_course_blocks
-from openedx.core.lib.block_structure.factory import BlockStructureFactory
+from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
 from openedx.core.djangolib.testing.utils import get_mock_request
 from student.tests.factories import UserFactory
 from student.models import CourseEnrollment

lms/djangoapps/grades/tests/test_scores.py

@@ -11,7 +11,7 @@ from lms.djangoapps.grades.models import BlockRecord
 import lms.djangoapps.grades.scores as scores
 from lms.djangoapps.grades.transformer import GradesTransformer
 from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
-from openedx.core.lib.block_structure.block_structure import BlockData
+from openedx.core.djangoapps.content.block_structure.block_structure import BlockData
 from xmodule.graders import ProblemScore
 
 

lms/djangoapps/grades/tests/test_tasks.py

@@ -13,7 +13,7 @@ from mock import patch, MagicMock
 import pytz
 from util.date_utils import to_timestamp
 
-from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
+from openedx.core.djangoapps.content.block_structure.exceptions import BlockStructureNotFound
 from student.models import anonymous_id_for_user
 from student.tests.factories import UserFactory
 from track.event_transaction_utils import (
@@ -136,7 +136,7 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
         self.set_up_course()
         self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
         with patch(
-            'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_store',
+            'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_store',
             side_effect=BlockStructureNotFound(self.course.location),
         ) as mock_block_structure_create:
             self._apply_recalculate_subsection_grade()

lms/djangoapps/grades/transformer.py

@@ -8,7 +8,7 @@ from logging import getLogger
 import json
 
 from lms.djangoapps.course_blocks.transformers.utils import collect_unioned_set_field, get_field_on_block
-from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
+from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
 
 
 log = getLogger(__name__)

openedx/core/djangoapps/content/block_structure/__init__.py

 """
-This code exists in openedx/core/djangoapp because it needs access to django signaling mechanisms
+The block_structure django app provides an extensible framework for caching
+data of block structures from the modulestore.
 
-Most of the underlying functionality is implemented in openedx/core/lib/block_structure/
+Dual-Phase. The framework is meant to be used in 2 phases.
+
+  * Collect Phase (for expensive and full-tree traversals) - In the
+    first phase, the "collect" phase, any and all data from the
+    modulestore should be collected and cached for later access to
+    the block structure.  Instantiating any and all xBlocks in the block
+    structure is also done at this phase, since that is also (currently)
+    a costly operation.
+
+    Any full tree traversals should also be done during this phase. For
+    example, if data for a block depends on its parents, the traversal
+    should happen during the collection phase and any required data
+    for the block should be percolated down the tree and stored as
+    aggregate values on the descendants.  This allows for faster and
+    direct access to blocks in the Transform phase.
+
+  * Transform Phase (for fast access to blocks) - In the second
+    phase, the "transform" phase, only the previously collected and
+    cached data should be accessed. There should be no access to the
+    modulestore or instantiation of xBlocks in this phase.
+
+
+To make this framework extensible, the Transformer and
+Extensibility design patterns are used. This django app only
+provides the underlying framework for Block Structure Transformers
+and a Transformer Registry.  Clients are expected to provide actual
+implementations of Transformers or add them to the extensible Registry.
+
+Transformers. As inspired by
+http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
+a Block Structure Transformer takes in a block structure (or tree) and
+manipulates the structure and the data of its blocks according to its
+own requirements. Its output can then be used for further
+transformations by other transformers down the pipeline.
+
+Note: For performance and space optimization, our implementation
+differs from the paper in that our transformers mutate the block
+structure in-place rather than returning a modified copy of it.
+
+Block Structure. The BlockStructure and its family of classes
+provided with this framework are the base data types for accessing
+and manipulating block structures. BlockStructures are constructed
+using the BlockStructureFactory and then used as the currency across
+Transformers.
+
+Registry. Transformers are registered using the platform's
+PluginManager (e.g., Stevedore). This is currently done by updating
+setup.py.  Only registered transformers are called during the Collect
+Phase.  And only registered transformers can be used during the
+Transform phase.  Exceptions to this rule are any nested transformers
+that are contained within higher-order transformers - as long as the
+higher-order transformers are registered and appropriately call the
+contained transformers within them.
+
+Note: A partial subset (as an ordered list) of the registered
+transformers can be requested during the Transform phase, allowing
+the client to manipulate exactly which transformers to call.
 """

openedx/core/djangoapps/content/block_structure/api.py

@@ -2,9 +2,10 @@
 Higher order functions built on the BlockStructureManager to interact with a django cache.
 """
 from django.core.cache import cache
-from openedx.core.lib.block_structure.manager import BlockStructureManager
 from xmodule.modulestore.django import modulestore
 
+from .manager import BlockStructureManager
+
 
 def get_course_in_cache(course_key):
     """

openedx/core/lib/block_structure/manager.py → openedx/core/djangoapps/content/block_structure/manager.py

@@ -4,8 +4,7 @@ BlockStructures.
 """
 from contextlib import contextmanager
 
-from openedx.core.djangoapps.content.block_structure import config
-
+from . import config
 from .exceptions import UsageKeyNotInBlockStructure, TransformerDataIncompatible, BlockStructureNotFound
 from .factory import BlockStructureFactory
 from .store import BlockStructureStore

openedx/core/djangoapps/content/block_structure/models.py

@@ -10,10 +10,10 @@ from logging import getLogger
 
 from model_utils.models import TimeStampedModel
 from openedx.core.djangoapps.xmodule_django.models import UsageKeyField
-from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
 from openedx.core.storage import get_storage
 
-import openedx.core.djangoapps.content.block_structure.config as config
+from . import config
+from .exceptions import BlockStructureNotFound
 
 
 log = getLogger(__name__)

openedx/core/lib/block_structure/store.py → openedx/core/djangoapps/content/block_structure/store.py

@@ -4,14 +4,13 @@ Module for the Storage of BlockStructure objects.
 # pylint: disable=protected-access
 from logging import getLogger
 
-import openedx.core.djangoapps.content.block_structure.config as config
-from openedx.core.djangoapps.content.block_structure.models import BlockStructureModel
-
 from openedx.core.lib.cache_utils import zpickle, zunpickle
 
+from . import config
 from .block_structure import BlockStructureBlockData
 from .exceptions import BlockStructureNotFound
 from .factory import BlockStructureFactory
+from .models import BlockStructureModel
 from .transformer_registry import TransformerRegistry
 
 

openedx/core/djangoapps/content/block_structure/tests/helpers.py

 """
-Helpers for Course Blocks tests.
+Common utilities for tests in block_structure module
 """
+from contextlib import contextmanager
+from mock import patch
+from xmodule.modulestore.exceptions import ItemNotFoundError
+from uuid import uuid4
+
+from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
 from openedx.core.djangolib.testing.waffle_utils import override_switch
-from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
-from openedx.core.lib.block_structure.store import BlockStructureStore
 
 from ..api import get_cache
+from ..block_structure import BlockStructureBlockData
 from ..config import _bs_waffle_switch_name
+from ..exceptions import BlockStructureNotFound
+from ..store import BlockStructureStore
+from ..transformer import BlockStructureTransformer, FilteringTransformerMixin
+from ..transformer_registry import TransformerRegistry
 
 
 def is_course_in_block_structure_cache(course_key, store):
@@ -31,3 +40,301 @@ class override_config_setting(override_switch):  # pylint:disable=invalid-name
             _bs_waffle_switch_name(name),
             active
         )
+
+
+class MockXBlock(object):
+    """
+    A mock XBlock to be used in unit tests, thereby decoupling the
+    implementation of the block cache framework from the xBlock
+    implementation.  This class provides only the minimum xBlock
+    capabilities needed by the block cache framework.
+    """
+    def __init__(self, location, field_map=None, children=None, modulestore=None):
+        self.location = location
+        self.field_map = field_map or {}
+
+        self.children = children or []
+        self.modulestore = modulestore
+
+    def __getattr__(self, attr):
+        try:
+            return self.field_map[attr]
+        except KeyError:
+            raise AttributeError
+
+    def get_children(self):
+        """
+        Returns the children of the mock XBlock.
+        """
+        return [self.modulestore.get_item(child) for child in self.children]
+
+
+class MockModulestore(object):
+    """
+    A mock Modulestore to be used in unit tests, providing only the
+    minimum methods needed by the block cache framework.
+    """
+    def __init__(self):
+        self.get_items_call_count = 0
+        self.blocks = None
+
+    def set_blocks(self, blocks):
+        """
+        Updates the mock modulestore with a dictionary of blocks.
+
+        Arguments:
+            blocks ({block key, MockXBlock}) - A map of block_key
+            to its mock xBlock.
+        """
+        self.blocks = blocks
+
+    def get_item(self, block_key, depth=None, lazy=False):  # pylint: disable=unused-argument
+        """
+        Returns the mock XBlock (MockXBlock) associated with the
+        given block_key.
+
+        Raises ItemNotFoundError if the item is not found.
+        """
+        self.get_items_call_count += 1
+        item = self.blocks.get(block_key)
+        if not item:
+            raise ItemNotFoundError
+        return item
+
+    @contextmanager
+    def bulk_operations(self, ignore):  # pylint: disable=unused-argument
+        """
+        A context manager for notifying the store of bulk operations.
+        """
+        yield
+
+
+class MockCache(object):
+    """
+    A mock Cache object, providing only the minimum features needed
+    by the block cache framework.
+    """
+    def __init__(self):
+        # An in-memory map of cache keys to cache values.
+        self.map = {}
+        self.set_call_count = 0
+        self.timeout_from_last_call = 0
+
+    def set(self, key, val, timeout):
+        """
+        Associates the given key with the given value in the cache.
+        """
+        self.set_call_count += 1
+        self.map[key] = val
+        self.timeout_from_last_call = timeout
+
+    def get(self, key, default=None):
+        """
+        Returns the value associated with the given key in the cache;
+        returns default if not found.
+        """
+        return self.map.get(key, default)
+
+    def delete(self, key):
+        """
+        Deletes the given key from the cache.
+        """
+        del self.map[key]
+
+
+class MockModulestoreFactory(object):
+    """
+    A factory for creating MockModulestore objects.
+    """
+    @classmethod
+    def create(cls, children_map, block_key_factory):
+        """
+        Creates and returns a MockModulestore from the given
+        children_map.
+
+        Arguments:
+            children_map ({block_key: [block_key]}) - A dictionary
+                mapping a block key to a list of block keys of the
+                block's corresponding children.
+        """
+        modulestore = MockModulestore()
+        modulestore.set_blocks({
+            block_key_factory(block_key): MockXBlock(
+                block_key_factory(block_key),
+                children=[block_key_factory(child) for child in children],
+                modulestore=modulestore,
+            )
+            for block_key, children in enumerate(children_map)
+        })
+        return modulestore
+
+
+class MockTransformer(BlockStructureTransformer):
+    """
+    A mock BlockStructureTransformer class.
+    """
+    WRITE_VERSION = 1
+    READ_VERSION = 1
+
+    @classmethod
+    def name(cls):
+        # Use the class' name for Mock transformers.
+        return cls.__name__
+
+    def transform(self, usage_info, block_structure):
+        pass
+
+
+class MockFilteringTransformer(FilteringTransformerMixin, BlockStructureTransformer):
+    """
+    A mock FilteringTransformerMixin class.
+    """
+    WRITE_VERSION = 1
+    READ_VERSION = 1
+
+    @classmethod
+    def name(cls):
+        # Use the class' name for Mock transformers.
+        return cls.__name__
+
+    def transform_block_filters(self, usage_info, block_structure):
+        return [block_structure.create_universal_filter()]
+
+
+def clear_registered_transformers_cache():
+    """
+    Test helper to clear out any cached values of registered transformers.
+    """
+    TransformerRegistry.get_write_version_hash.cache.clear()
+
+
+@contextmanager
+def mock_registered_transformers(transformers):
+    """
+    Context manager for mocking the transformer registry to return the given transformers.
+    """
+    clear_registered_transformers_cache()
+    with patch(
+        'openedx.core.djangoapps.content.block_structure.transformer_registry.'
+        'TransformerRegistry.get_registered_transformers'
+    ) as mock_available_transforms:
+        mock_available_transforms.return_value = {transformer for transformer in transformers}
+        yield
+
+
+class ChildrenMapTestMixin(object):
+    """
+    A Test Mixin with utility methods for testing with block structures
+    created and manipulated using children_map and parents_map.
+    """
+
+    #     0
+    #    / \
+    #   1  2
+    #  / \
+    # 3   4
+    SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
+
+    #       0
+    #      /
+    #     1
+    #    /
+    #   2
+    #  /
+    # 3
+    LINEAR_CHILDREN_MAP = [[1], [2], [3], []]
+
+    #     0
+    #    / \
+    #   1  2
+    #   \ / \
+    #    3  4
+    #   / \
+    #  5  6
+    DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []]
+
+    def block_key_factory(self, block_id):
+        """
+        Returns a block key object for the given block_id.
+        Override this method if the block_key should be anything
+        different from the index integer values in the Children Maps.
+        """
+        return block_id
+
+    def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData):
+        """
+        Factory method for creating and returning a block structure
+        for the given children_map.
+        """
+        # create empty block structure
+        block_structure = block_structure_cls(root_block_usage_key=self.block_key_factory(0))
+
+        # _add_relation
+        for parent, children in enumerate(children_map):
+            for child in children:
+                block_structure._add_relation(self.block_key_factory(parent), self.block_key_factory(child))  # pylint: disable=protected-access
+        return block_structure
+
+    def get_parents_map(self, children_map):
+        """
+        Converts and returns the given children_map to a parents_map.
+        """
+        parent_map = [[] for _ in children_map]
+        for parent, children in enumerate(children_map):
+            for child in children:
+                parent_map[child].append(parent)
+        return parent_map
+
+    def assert_block_structure(self, block_structure, children_map, missing_blocks=None):
+        """
+        Verifies that the relations in the given block structure
+        equate the relations described in the children_map. Use the
+        missing_blocks parameter to pass in any blocks that were removed
+        from the block structure but still have a positional entry in
+        the children_map.
+        """
+        if not missing_blocks:
+            missing_blocks = []
+
+        for block_key, children in enumerate(children_map):
+            # Verify presence
+            self.assertEqual(
+                self.block_key_factory(block_key) in block_structure,
+                block_key not in missing_blocks,
+                'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format(
+                    unicode(block_key)
+                ),
+            )
+
+            # Verify children
+            if block_key not in missing_blocks:
+                self.assertEqual(
+                    set(block_structure.get_children(self.block_key_factory(block_key))),
+                    set(self.block_key_factory(child) for child in children),
+                )
+
+        # Verify parents
+        parents_map = self.get_parents_map(children_map)
+        for block_key, parents in enumerate(parents_map):
+            if block_key not in missing_blocks:
+                self.assertEqual(
+                    set(block_structure.get_parents(self.block_key_factory(block_key))),
+                    set(self.block_key_factory(parent) for parent in parents),
+                )
+
+
+class UsageKeyFactoryMixin(object):
+    """
+    Test Mixin that provides a block_key_factory to create OpaqueKey objects
+    for block_ids rather than simple integers. By default, the children maps in
+    ChildrenMapTestMixin use integers for block_ids.
+    """
+    def setUp(self):
+        super(UsageKeyFactoryMixin, self).setUp()
+        self.course_key = CourseLocator('org', 'course', unicode(uuid4()))
+
+    def block_key_factory(self, block_id):
+        """
+        Returns a block key object for the given block_id.
+        """
+        return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=unicode(block_id))

openedx/core/lib/block_structure/tests/test_manager.py → openedx/core/djangoapps/content/block_structure/tests/test_manager.py

@@ -5,10 +5,8 @@ import ddt
 from nose.plugins.attrib import attr
 from unittest import TestCase
 
-from openedx.core.djangoapps.content.block_structure.config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE
-from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting
-
 from ..block_structure import BlockStructureBlockData
+from ..config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE
 from ..exceptions import UsageKeyNotInBlockStructure, BlockStructureNotFound
 from ..manager import BlockStructureManager
 from ..transformers import BlockStructureTransformers
@@ -16,6 +14,7 @@ from .helpers import (
     MockModulestoreFactory, MockCache, MockTransformer,
     ChildrenMapTestMixin, UsageKeyFactoryMixin,
     mock_registered_transformers,
+    override_config_setting,
 )
 
 

openedx/core/djangoapps/content/block_structure/tests/test_models.py

@@ -10,9 +10,9 @@ from mock import patch, Mock
 from uuid import uuid4
 
 from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
-from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
 
 from ..config import PRUNE_OLD_VERSIONS
+from ..exceptions import BlockStructureNotFound
 from ..models import BlockStructureModel
 from .helpers import override_config_setting
 

openedx/core/djangoapps/content/block_structure/tests/test_signals.py

@@ -50,7 +50,7 @@ class CourseBlocksSignalTest(ModuleStoreTestCase):
         )
 
     @ddt.data(True, False)
-    @patch('openedx.core.lib.block_structure.manager.BlockStructureManager.clear')
+    @patch('openedx.core.djangoapps.content.block_structure.manager.BlockStructureManager.clear')
     def test_cache_invalidation(self, invalidate_cache_enabled, mock_bs_manager_clear):
         test_display_name = "Jedi 101"
 

openedx/core/lib/block_structure/tests/test_store.py → openedx/core/djangoapps/content/block_structure/tests/test_store.py

@@ -4,14 +4,13 @@ Tests for block_structure/cache.py
 import ddt
 from nose.plugins.attrib import attr
 
-from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE
-from openedx.core.djangoapps.content.block_structure.config.models import BlockStructureConfiguration
-from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting
 from openedx.core.djangolib.testing.utils import CacheIsolationTestCase
 
-from ..store import BlockStructureStore
+from ..config import STORAGE_BACKING_FOR_CACHE
+from ..config.models import BlockStructureConfiguration
 from ..exceptions import BlockStructureNotFound
-from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer
+from ..store import BlockStructureStore
+from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer, override_config_setting
 
 
 @attr(shard=2)

openedx/core/lib/block_structure/tests/test_transformers.py → openedx/core/djangoapps/content/block_structure/tests/test_transformers.py

@@ -57,7 +57,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
     def test_collect(self):
         with mock_registered_transformers(self.registered_transformers):
             with patch(
-                'openedx.core.lib.block_structure.tests.helpers.MockTransformer.collect'
+                'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.collect'
             ) as mock_collect_call:
                 BlockStructureTransformers.collect(block_structure=MagicMock())
                 self.assertTrue(mock_collect_call.called)
@@ -66,7 +66,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
         self.add_mock_transformer()
 
         with patch(
-            'openedx.core.lib.block_structure.tests.helpers.MockTransformer.transform'
+            'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.transform'
         ) as mock_transform_call:
             self.transformers.transform(block_structure=MagicMock())
             self.assertTrue(mock_transform_call.called)

openedx/core/lib/block_structure/__init__.py

-"""
-The block_structure django app provides an extensible framework for caching
-data of block structures from the modulestore.
-
-Dual-Phase. The framework is meant to be used in 2 phases.
-
-  * Collect Phase (for expensive and full-tree traversals) - In the
-    first phase, the "collect" phase, any and all data from the
-    modulestore should be collected and cached for later access to
-    the block structure.  Instantiating any and all xBlocks in the block
-    structure is also done at this phase, since that is also (currently)
-    a costly operation.
-
-    Any full tree traversals should also be done during this phase. For
-    example, if data for a block depends on its parents, the traversal
-    should happen during the collection phase and any required data
-    for the block should be percolated down the tree and stored as
-    aggregate values on the descendants.  This allows for faster and
-    direct access to blocks in the Transform phase.
-
-  * Transform Phase (for fast access to blocks) - In the second
-    phase, the "transform" phase, only the previously collected and
-    cached data should be accessed. There should be no access to the
-    modulestore or instantiation of xBlocks in this phase.
-
-
-To make this framework extensible, the Transformer and
-Extensibility design patterns are used. This django app only
-provides the underlying framework for Block Structure Transformers
-and a Transformer Registry.  Clients are expected to provide actual
-implementations of Transformers or add them to the extensible Registry.
-
-Transformers. As inspired by
-http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
-a Block Structure Transformer takes in a block structure (or tree) and
-manipulates the structure and the data of its blocks according to its
-own requirements. Its output can then be used for further
-transformations by other transformers down the pipeline.
-
-Note: For performance and space optimization, our implementation
-differs from the paper in that our transformers mutate the block
-structure in-place rather than returning a modified copy of it.
-
-Block Structure. The BlockStructure and its family of classes
-provided with this framework are the base data types for accessing
-and manipulating block structures. BlockStructures are constructed
-using the BlockStructureFactory and then used as the currency across
-Transformers.
-
-Registry. Transformers are registered using the platform's
-PluginManager (e.g., Stevedore). This is currently done by updating
-setup.py.  Only registered transformers are called during the Collect
-Phase.  And only registered transformers can be used during the
-Transform phase.  Exceptions to this rule are any nested transformers
-that are contained within higher-order transformers - as long as the
-higher-order transformers are registered and appropriately call the
-contained transformers within them.
-
-Note: A partial subset (as an ordered list) of the registered
-transformers can be requested during the Transform phase, allowing
-the client to manipulate exactly which transformers to call.
-"""

openedx/core/lib/block_structure/tests/helpers.py

-"""
-Common utilities for tests in block_structure module
-"""
-from contextlib import contextmanager
-from mock import patch
-from xmodule.modulestore.exceptions import ItemNotFoundError
-from uuid import uuid4
-
-from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
-
-from ..block_structure import BlockStructureBlockData
-from ..transformer import BlockStructureTransformer, FilteringTransformerMixin
-from ..transformer_registry import TransformerRegistry
-
-
-class MockXBlock(object):
-    """
-    A mock XBlock to be used in unit tests, thereby decoupling the
-    implementation of the block cache framework from the xBlock
-    implementation.  This class provides only the minimum xBlock
-    capabilities needed by the block cache framework.
-    """
-    def __init__(self, location, field_map=None, children=None, modulestore=None):
-        self.location = location
-        self.field_map = field_map or {}
-
-        self.children = children or []
-        self.modulestore = modulestore
-
-    def __getattr__(self, attr):
-        try:
-            return self.field_map[attr]
-        except KeyError:
-            raise AttributeError
-
-    def get_children(self):
-        """
-        Returns the children of the mock XBlock.
-        """
-        return [self.modulestore.get_item(child) for child in self.children]
-
-
-class MockModulestore(object):
-    """
-    A mock Modulestore to be used in unit tests, providing only the
-    minimum methods needed by the block cache framework.
-    """
-    def __init__(self):
-        self.get_items_call_count = 0
-        self.blocks = None
-
-    def set_blocks(self, blocks):
-        """
-        Updates the mock modulestore with a dictionary of blocks.
-
-        Arguments:
-            blocks ({block key, MockXBlock}) - A map of block_key
-            to its mock xBlock.
-        """
-        self.blocks = blocks
-
-    def get_item(self, block_key, depth=None, lazy=False):  # pylint: disable=unused-argument
-        """
-        Returns the mock XBlock (MockXBlock) associated with the
-        given block_key.
-
-        Raises ItemNotFoundError if the item is not found.
-        """
-        self.get_items_call_count += 1
-        item = self.blocks.get(block_key)
-        if not item:
-            raise ItemNotFoundError
-        return item
-
-    @contextmanager
-    def bulk_operations(self, ignore):  # pylint: disable=unused-argument
-        """
-        A context manager for notifying the store of bulk operations.
-        """
-        yield
-
-
-class MockCache(object):
-    """
-    A mock Cache object, providing only the minimum features needed
-    by the block cache framework.
-    """
-    def __init__(self):
-        # An in-memory map of cache keys to cache values.
-        self.map = {}
-        self.set_call_count = 0
-        self.timeout_from_last_call = 0
-
-    def set(self, key, val, timeout):
-        """
-        Associates the given key with the given value in the cache.
-        """
-        self.set_call_count += 1
-        self.map[key] = val
-        self.timeout_from_last_call = timeout
-
-    def get(self, key, default=None):
-        """
-        Returns the value associated with the given key in the cache;
-        returns default if not found.
-        """
-        return self.map.get(key, default)
-
-    def delete(self, key):
-        """
-        Deletes the given key from the cache.
-        """
-        del self.map[key]
-
-
-class MockModulestoreFactory(object):
-    """
-    A factory for creating MockModulestore objects.
-    """
-    @classmethod
-    def create(cls, children_map, block_key_factory):
-        """
-        Creates and returns a MockModulestore from the given
-        children_map.
-
-        Arguments:
-            children_map ({block_key: [block_key]}) - A dictionary
-                mapping a block key to a list of block keys of the
-                block's corresponding children.
-        """
-        modulestore = MockModulestore()
-        modulestore.set_blocks({
-            block_key_factory(block_key): MockXBlock(
-                block_key_factory(block_key),
-                children=[block_key_factory(child) for child in children],
-                modulestore=modulestore,
-            )
-            for block_key, children in enumerate(children_map)
-        })
-        return modulestore
-
-
-class MockTransformer(BlockStructureTransformer):
-    """
-    A mock BlockStructureTransformer class.
-    """
-    WRITE_VERSION = 1
-    READ_VERSION = 1
-
-    @classmethod
-    def name(cls):
-        # Use the class' name for Mock transformers.
-        return cls.__name__
-
-    def transform(self, usage_info, block_structure):
-        pass
-
-
-class MockFilteringTransformer(FilteringTransformerMixin, BlockStructureTransformer):
-    """
-    A mock FilteringTransformerMixin class.
-    """
-    WRITE_VERSION = 1
-    READ_VERSION = 1
-
-    @classmethod
-    def name(cls):
-        # Use the class' name for Mock transformers.
-        return cls.__name__
-
-    def transform_block_filters(self, usage_info, block_structure):
-        return [block_structure.create_universal_filter()]
-
-
-def clear_registered_transformers_cache():
-    """
-    Test helper to clear out any cached values of registered transformers.
-    """
-    TransformerRegistry.get_write_version_hash.cache.clear()
-
-
-@contextmanager
-def mock_registered_transformers(transformers):
-    """
-    Context manager for mocking the transformer registry to return the given transformers.
-    """
-    clear_registered_transformers_cache()
-    with patch(
-        'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
-    ) as mock_available_transforms:
-        mock_available_transforms.return_value = {transformer for transformer in transformers}
-        yield
-
-
-class ChildrenMapTestMixin(object):
-    """
-    A Test Mixin with utility methods for testing with block structures
-    created and manipulated using children_map and parents_map.
-    """
-
-    #     0
-    #    / \
-    #   1  2
-    #  / \
-    # 3   4
-    SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
-
-    #       0
-    #      /
-    #     1
-    #    /
-    #   2
-    #  /
-    # 3
-    LINEAR_CHILDREN_MAP = [[1], [2], [3], []]
-
-    #     0
-    #    / \
-    #   1  2
-    #   \ / \
-    #    3  4
-    #   / \
-    #  5  6
-    DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []]
-
-    def block_key_factory(self, block_id):
-        """
-        Returns a block key object for the given block_id.
-        Override this method if the block_key should be anything
-        different from the index integer values in the Children Maps.
-        """
-        return block_id
-
-    def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData):
-        """
-        Factory method for creating and returning a block structure
-        for the given children_map.
-        """
-        # create empty block structure
-        block_structure = block_structure_cls(root_block_usage_key=self.block_key_factory(0))
-
-        # _add_relation
-        for parent, children in enumerate(children_map):
-            for child in children:
-                block_structure._add_relation(self.block_key_factory(parent), self.block_key_factory(child))  # pylint: disable=protected-access
-        return block_structure
-
-    def get_parents_map(self, children_map):
-        """
-        Converts and returns the given children_map to a parents_map.
-        """
-        parent_map = [[] for _ in children_map]
-        for parent, children in enumerate(children_map):
-            for child in children:
-                parent_map[child].append(parent)
-        return parent_map
-
-    def assert_block_structure(self, block_structure, children_map, missing_blocks=None):
-        """
-        Verifies that the relations in the given block structure
-        equate the relations described in the children_map. Use the
-        missing_blocks parameter to pass in any blocks that were removed
-        from the block structure but still have a positional entry in
-        the children_map.
-        """
-        if not missing_blocks:
-            missing_blocks = []
-
-        for block_key, children in enumerate(children_map):
-            # Verify presence
-            self.assertEqual(
-                self.block_key_factory(block_key) in block_structure,
-                block_key not in missing_blocks,
-                'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format(
-                    unicode(block_key)
-                ),
-            )
-
-            # Verify children
-            if block_key not in missing_blocks:
-                self.assertEqual(
-                    set(block_structure.get_children(self.block_key_factory(block_key))),
-                    set(self.block_key_factory(child) for child in children),
-                )
-
-        # Verify parents
-        parents_map = self.get_parents_map(children_map)
-        for block_key, parents in enumerate(parents_map):
-            if block_key not in missing_blocks:
-                self.assertEqual(
-                    set(block_structure.get_parents(self.block_key_factory(block_key))),
-                    set(self.block_key_factory(parent) for parent in parents),
-                )
-
-
-class UsageKeyFactoryMixin(object):
-    """
-    Test Mixin that provides a block_key_factory to create OpaqueKey objects
-    for block_ids rather than simple integers. By default, the children maps in
-    ChildrenMapTestMixin use integers for block_ids.
-    """
-    def setUp(self):
-        super(UsageKeyFactoryMixin, self).setUp()
-        self.course_key = CourseLocator('org', 'course', unicode(uuid4()))
-
-    def block_key_factory(self, block_id):
-        """
-        Returns a block key object for the given block_id.
-        """
-        return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=unicode(block_id))