Commit 00d46622 by Nimisha Asthagiri

Consolidate Block Structure folders

TNL-6518
parent 11dce770
......@@ -4,7 +4,7 @@ API function for retrieving course blocks data
from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACCESS_TRANSFORMERS
from lms.djangoapps.course_blocks.transformers.hidden_content import HiddenContentTransformer
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from .transformers.blocks_api import BlocksAPITransformer
from .transformers.milestones import MilestonesTransformer
......
......@@ -3,7 +3,7 @@ Tests for Course Blocks serializers
"""
from mock import MagicMock
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
......
"""
Block Counts Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
class BlockCountsTransformer(BlockStructureTransformer):
......
"""
Block Depth Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
class BlockDepthTransformer(BlockStructureTransformer):
......
"""
Blocks API Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
from .block_counts import BlockCountsTransformer
from .block_depth import BlockDepthTransformer
from .navigation import BlockNavigationTransformer
......
......@@ -4,7 +4,10 @@ Milestones Transformer
from django.conf import settings
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from util import milestones_helpers
......
"""
TODO
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
from .block_depth import BlockDepthTransformer
......
"""
Student View Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
class StudentViewTransformer(BlockStructureTransformer):
......
......@@ -3,7 +3,7 @@ Tests for BlockCountsTransformer.
"""
# pylint: disable=protected-access
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory
......
......@@ -7,8 +7,8 @@ Tests for BlockDepthTransformer.
import ddt
from unittest import TestCase
from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData
from ..block_depth import BlockDepthTransformer
......
......@@ -7,9 +7,9 @@ from unittest import TestCase
from lms.djangoapps.course_api.blocks.transformers.block_depth import BlockDepthTransformer
from lms.djangoapps.course_api.blocks.transformers.navigation import BlockNavigationTransformer
from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory
from xmodule.modulestore import ModuleStoreEnum
......
......@@ -4,7 +4,7 @@ Tests for StudentViewTransformer.
# pylint: disable=protected-access
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import ToyCourseFactory
......
"""
The Course Blocks app, built upon the Block Cache framework in
openedx.core.lib.block_structure, is a higher layer django app in LMS that
openedx.core.djangoapps.content.block_structure, is a higher layer django app in LMS that
provides additional context of Courses and Users (via usage_info.py) with
implementations for Block Structure Transformers that are related to
block structure course access.
......
......@@ -3,7 +3,7 @@ API entry point to the course_blocks app with top-level
get_course_blocks function.
"""
from openedx.core.djangoapps.content.block_structure.api import get_block_structure_manager
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from .transformers import (
library_content,
......
......@@ -8,7 +8,7 @@ from xmodule.modulestore.django import modulestore
import openedx.core.djangoapps.content.block_structure.api as api
import openedx.core.djangoapps.content.block_structure.tasks as tasks
import openedx.core.lib.block_structure.store as store
import openedx.core.djangoapps.content.block_structure.store as store
from openedx.core.lib.command_utils import (
get_mutually_exclusive_required_option,
validate_dependent_option,
......
......@@ -59,7 +59,7 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
self.command.handle(all_courses=True)
self._assert_courses_in_block_cache(*self.course_keys)
with patch(
'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_modulestore'
'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore'
) as mock_update_from_store:
self.command.handle(all_courses=True, force_update=force_update)
self.assertEqual(mock_update_from_store.call_count, self.num_courses if force_update else 0)
......
......@@ -4,7 +4,10 @@ Visibility Transformer implementation.
from datetime import datetime
from pytz import utc
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from xmodule.seq_module import SequenceModule
from .utils import collect_merged_boolean_field, collect_merged_date_field
......
......@@ -3,7 +3,10 @@ Content Library Transformer.
"""
import json
from courseware.models import StudentModule
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from xmodule.library_content_module import LibraryContentModule
from xmodule.modulestore.django import modulestore
from eventtracking import tracker
......
"""
Split Test Block Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
class SplitTestTransformer(FilteringTransformerMixin, BlockStructureTransformer):
......
"""
Start Date Transformer implementation.
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from lms.djangoapps.courseware.access_utils import check_start_date
from xmodule.course_metadata_utils import DEFAULT_START_DATE
......
......@@ -4,8 +4,8 @@ Test helpers for testing course block transformers.
from mock import patch
from course_modes.models import CourseMode
from lms.djangoapps.courseware.access import has_access
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.lib.block_structure.tests.helpers import clear_registered_transformers_cache
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.tests.helpers import clear_registered_transformers_cache
from student.tests.factories import CourseEnrollmentFactory, UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
......@@ -23,7 +23,8 @@ class TransformerRegistryTestMixin(object):
def setUp(self):
super(TransformerRegistryTestMixin, self).setUp()
self.patcher = patch(
'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
'openedx.core.djangoapps.content.block_structure.transformer_registry.'
'TransformerRegistry.get_registered_transformers'
)
mock_registry = self.patcher.start()
mock_registry.return_value = {self.TRANSFORMER_CLASS_TO_TEST}
......
......@@ -5,7 +5,7 @@ Tests for ContentLibraryTransformer.
from student.tests.factories import CourseEnrollmentFactory
from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from ...api import get_course_blocks
from ..library_content import ContentLibraryTransformer
......
"""
User Partitions Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from .split_test import SplitTestTransformer
from .utils import get_field_on_block
......
"""
Visibility Transformer implementation.
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from .utils import collect_merged_boolean_field
......
......@@ -12,7 +12,7 @@ from courseware.model_data import set_score
from courseware.tests.helpers import LoginEnrollmentTestCase
from lms.djangoapps.course_blocks.api import get_course_blocks
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from openedx.core.djangolib.testing.utils import get_mock_request
from student.tests.factories import UserFactory
from student.models import CourseEnrollment
......
......@@ -11,7 +11,7 @@ from lms.djangoapps.grades.models import BlockRecord
import lms.djangoapps.grades.scores as scores
from lms.djangoapps.grades.transformer import GradesTransformer
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from openedx.core.lib.block_structure.block_structure import BlockData
from openedx.core.djangoapps.content.block_structure.block_structure import BlockData
from xmodule.graders import ProblemScore
......
......@@ -13,7 +13,7 @@ from mock import patch, MagicMock
import pytz
from util.date_utils import to_timestamp
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from openedx.core.djangoapps.content.block_structure.exceptions import BlockStructureNotFound
from student.models import anonymous_id_for_user
from student.tests.factories import UserFactory
from track.event_transaction_utils import (
......@@ -136,7 +136,7 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
self.set_up_course()
self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
with patch(
'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_store',
'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_store',
side_effect=BlockStructureNotFound(self.course.location),
) as mock_block_structure_create:
self._apply_recalculate_subsection_grade()
......
......@@ -8,7 +8,7 @@ from logging import getLogger
import json
from lms.djangoapps.course_blocks.transformers.utils import collect_unioned_set_field, get_field_on_block
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
log = getLogger(__name__)
......
"""
This code exists in openedx/core/djangoapp because it needs access to django signaling mechanisms
The block_structure django app provides an extensible framework for caching
data of block structures from the modulestore.
Most of the underlying functionality is implemented in openedx/core/lib/block_structure/
Dual-Phase. The framework is meant to be used in 2 phases.
* Collect Phase (for expensive and full-tree traversals) - In the
first phase, the "collect" phase, any and all data from the
modulestore should be collected and cached for later access to
the block structure. Instantiating any and all xBlocks in the block
structure is also done at this phase, since that is also (currently)
a costly operation.
Any full tree traversals should also be done during this phase. For
example, if data for a block depends on its parents, the traversal
should happen during the collection phase and any required data
for the block should be percolated down the tree and stored as
aggregate values on the descendants. This allows for faster and
direct access to blocks in the Transform phase.
* Transform Phase (for fast access to blocks) - In the second
phase, the "transform" phase, only the previously collected and
cached data should be accessed. There should be no access to the
modulestore or instantiation of xBlocks in this phase.
To make this framework extensible, the Transformer and
Extensibility design patterns are used. This django app only
provides the underlying framework for Block Structure Transformers
and a Transformer Registry. Clients are expected to provide actual
implementations of Transformers or add them to the extensible Registry.
Transformers. As inspired by
http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
a Block Structure Transformer takes in a block structure (or tree) and
manipulates the structure and the data of its blocks according to its
own requirements. Its output can then be used for further
transformations by other transformers down the pipeline.
Note: For performance and space optimization, our implementation
differs from the paper in that our transformers mutate the block
structure in-place rather than returning a modified copy of it.
Block Structure. The BlockStructure and its family of classes
provided with this framework are the base data types for accessing
and manipulating block structures. BlockStructures are constructed
using the BlockStructureFactory and then used as the currency across
Transformers.
Registry. Transformers are registered using the platform's
PluginManager (e.g., Stevedore). This is currently done by updating
setup.py. Only registered transformers are called during the Collect
Phase. And only registered transformers can be used during the
Transform phase. Exceptions to this rule are any nested transformers
that are contained within higher-order transformers - as long as the
higher-order transformers are registered and appropriately call the
contained transformers within them.
Note: A partial subset (as an ordered list) of the registered
transformers can be requested during the Transform phase, allowing
the client to manipulate exactly which transformers to call.
"""
......@@ -2,9 +2,10 @@
Higher order functions built on the BlockStructureManager to interact with a django cache.
"""
from django.core.cache import cache
from openedx.core.lib.block_structure.manager import BlockStructureManager
from xmodule.modulestore.django import modulestore
from .manager import BlockStructureManager
def get_course_in_cache(course_key):
"""
......
......@@ -4,8 +4,7 @@ BlockStructures.
"""
from contextlib import contextmanager
from openedx.core.djangoapps.content.block_structure import config
from . import config
from .exceptions import UsageKeyNotInBlockStructure, TransformerDataIncompatible, BlockStructureNotFound
from .factory import BlockStructureFactory
from .store import BlockStructureStore
......
......@@ -10,10 +10,10 @@ from logging import getLogger
from model_utils.models import TimeStampedModel
from openedx.core.djangoapps.xmodule_django.models import UsageKeyField
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from openedx.core.storage import get_storage
import openedx.core.djangoapps.content.block_structure.config as config
from . import config
from .exceptions import BlockStructureNotFound
log = getLogger(__name__)
......
......@@ -4,14 +4,13 @@ Module for the Storage of BlockStructure objects.
# pylint: disable=protected-access
from logging import getLogger
import openedx.core.djangoapps.content.block_structure.config as config
from openedx.core.djangoapps.content.block_structure.models import BlockStructureModel
from openedx.core.lib.cache_utils import zpickle, zunpickle
from . import config
from .block_structure import BlockStructureBlockData
from .exceptions import BlockStructureNotFound
from .factory import BlockStructureFactory
from .models import BlockStructureModel
from .transformer_registry import TransformerRegistry
......
......@@ -5,10 +5,8 @@ import ddt
from nose.plugins.attrib import attr
from unittest import TestCase
from openedx.core.djangoapps.content.block_structure.config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE
from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting
from ..block_structure import BlockStructureBlockData
from ..config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE
from ..exceptions import UsageKeyNotInBlockStructure, BlockStructureNotFound
from ..manager import BlockStructureManager
from ..transformers import BlockStructureTransformers
......@@ -16,6 +14,7 @@ from .helpers import (
MockModulestoreFactory, MockCache, MockTransformer,
ChildrenMapTestMixin, UsageKeyFactoryMixin,
mock_registered_transformers,
override_config_setting,
)
......
......@@ -10,9 +10,9 @@ from mock import patch, Mock
from uuid import uuid4
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from ..config import PRUNE_OLD_VERSIONS
from ..exceptions import BlockStructureNotFound
from ..models import BlockStructureModel
from .helpers import override_config_setting
......
......@@ -50,7 +50,7 @@ class CourseBlocksSignalTest(ModuleStoreTestCase):
)
@ddt.data(True, False)
@patch('openedx.core.lib.block_structure.manager.BlockStructureManager.clear')
@patch('openedx.core.djangoapps.content.block_structure.manager.BlockStructureManager.clear')
def test_cache_invalidation(self, invalidate_cache_enabled, mock_bs_manager_clear):
test_display_name = "Jedi 101"
......
......@@ -4,14 +4,13 @@ Tests for block_structure/cache.py
import ddt
from nose.plugins.attrib import attr
from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE
from openedx.core.djangoapps.content.block_structure.config.models import BlockStructureConfiguration
from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase
from ..store import BlockStructureStore
from ..config import STORAGE_BACKING_FOR_CACHE
from ..config.models import BlockStructureConfiguration
from ..exceptions import BlockStructureNotFound
from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer
from ..store import BlockStructureStore
from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer, override_config_setting
@attr(shard=2)
......
......@@ -57,7 +57,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
def test_collect(self):
with mock_registered_transformers(self.registered_transformers):
with patch(
'openedx.core.lib.block_structure.tests.helpers.MockTransformer.collect'
'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.collect'
) as mock_collect_call:
BlockStructureTransformers.collect(block_structure=MagicMock())
self.assertTrue(mock_collect_call.called)
......@@ -66,7 +66,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
self.add_mock_transformer()
with patch(
'openedx.core.lib.block_structure.tests.helpers.MockTransformer.transform'
'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.transform'
) as mock_transform_call:
self.transformers.transform(block_structure=MagicMock())
self.assertTrue(mock_transform_call.called)
......
"""
The block_structure django app provides an extensible framework for caching
data of block structures from the modulestore.
Dual-Phase. The framework is meant to be used in 2 phases.
* Collect Phase (for expensive and full-tree traversals) - In the
first phase, the "collect" phase, any and all data from the
modulestore should be collected and cached for later access to
the block structure. Instantiating any and all xBlocks in the block
structure is also done at this phase, since that is also (currently)
a costly operation.
Any full tree traversals should also be done during this phase. For
example, if data for a block depends on its parents, the traversal
should happen during the collection phase and any required data
for the block should be percolated down the tree and stored as
aggregate values on the descendants. This allows for faster and
direct access to blocks in the Transform phase.
* Transform Phase (for fast access to blocks) - In the second
phase, the "transform" phase, only the previously collected and
cached data should be accessed. There should be no access to the
modulestore or instantiation of xBlocks in this phase.
To make this framework extensible, the Transformer and
Extensibility design patterns are used. This django app only
provides the underlying framework for Block Structure Transformers
and a Transformer Registry. Clients are expected to provide actual
implementations of Transformers or add them to the extensible Registry.
Transformers. As inspired by
http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
a Block Structure Transformer takes in a block structure (or tree) and
manipulates the structure and the data of its blocks according to its
own requirements. Its output can then be used for further
transformations by other transformers down the pipeline.
Note: For performance and space optimization, our implementation
differs from the paper in that our transformers mutate the block
structure in-place rather than returning a modified copy of it.
Block Structure. The BlockStructure and its family of classes
provided with this framework are the base data types for accessing
and manipulating block structures. BlockStructures are constructed
using the BlockStructureFactory and then used as the currency across
Transformers.
Registry. Transformers are registered using the platform's
PluginManager (e.g., Stevedore). This is currently done by updating
setup.py. Only registered transformers are called during the Collect
Phase. And only registered transformers can be used during the
Transform phase. Exceptions to this rule are any nested transformers
that are contained within higher-order transformers - as long as the
higher-order transformers are registered and appropriately call the
contained transformers within them.
Note: A partial subset (as an ordered list) of the registered
transformers can be requested during the Transform phase, allowing
the client to manipulate exactly which transformers to call.
"""
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment