Skip to content

E
edx-platform
Commit 599173c6 by Nimisha Asthagiri
Options

Merge pull request #11198 from edx/block-cache/collect-phase 

Refactor Block Cache to separate Collect and Transform phases
parents 90f2fb86 05767b43
Showing with 1193 additions and 647 deletions
lms/djangoapps/course_api/blocks/api.py
......@@ -2,10 +2,12 @@
API function for retrieving course blocks data
"""
from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACCESS_TRANSFORMERS
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from .transformers.blocks_api import BlocksAPITransformer
from .transformers.proctored_exam import ProctoredExamTransformer
from .serializers import BlockSerializer, BlockDictSerializer
from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACCESS_TRANSFORMERS
def get_blocks(
......@@ -17,7 +19,7 @@ def get_blocks(
requested_fields=None,
block_counts=None,
student_view_data=None,
return_type='dict'
return_type='dict',
):
"""
Return a serialized representation of the course blocks.
......@@ -43,25 +45,21 @@ def get_blocks(
return_type (string): Possible values are 'dict' or 'list'. Indicates
the format for returning the blocks.
"""
# construct BlocksAPITransformer
blocks_api_transformer = BlocksAPITransformer(
block_counts,
student_view_data,
depth,
nav_depth
)
# list of transformers to apply, adding user-specific ones if user is provided
transformers = []
# create ordered list of transformers, adding BlocksAPITransformer at end.
transformers = BlockStructureTransformers()
if user is not None:
transformers += COURSE_BLOCK_ACCESS_TRANSFORMERS + [ProctoredExamTransformer()]
transformers += [blocks_api_transformer]
transformers += [
BlocksAPITransformer(
block_counts,
student_view_data,
depth,
nav_depth
)
]
blocks = get_course_blocks(
user,
usage_key,
transformers=transformers,
)
# transform
blocks = get_course_blocks(user, usage_key, transformers)
# serialize
serializer_context = {
......
lms/djangoapps/course_api/blocks/tests/test_api.py
......@@ -3,6 +3,8 @@ Tests for Blocks api.py
"""
from django.test.client import RequestFactory
from course_blocks.tests.helpers import EnableTransformerRegistryMixin
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
......@@ -11,7 +13,7 @@ from xmodule.modulestore.tests.factories import SampleCourseFactory
from ..api import get_blocks
class TestGetBlocks(SharedModuleStoreTestCase):
class TestGetBlocks(EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
"""
Tests for the get_blocks function
"""
......
lms/djangoapps/course_api/blocks/tests/test_forms.py
......@@ -6,6 +6,7 @@ from django.http import Http404, QueryDict
from urllib import urlencode
from rest_framework.exceptions import PermissionDenied
from course_blocks.tests.helpers import EnableTransformerRegistryMixin
from opaque_keys.edx.locator import CourseLocator
from openedx.core.djangoapps.util.test_forms import FormTestMixin
from student.models import CourseEnrollment
......@@ -17,7 +18,7 @@ from ..forms import BlockListGetForm
@ddt.ddt
class TestBlockListGetForm(FormTestMixin, SharedModuleStoreTestCase):
class TestBlockListGetForm(EnableTransformerRegistryMixin, FormTestMixin, SharedModuleStoreTestCase):
"""
Tests for BlockListGetForm
"""
......
lms/djangoapps/course_api/blocks/tests/test_serializers.py
......@@ -3,6 +3,8 @@ Tests for Course Blocks serializers
"""
from mock import MagicMock
from course_blocks.tests.helpers import EnableTransformerRegistryMixin
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from student.tests.factories import UserFactory
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import ToyCourseFactory
......@@ -10,10 +12,10 @@ from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACC
from ..transformers.blocks_api import BlocksAPITransformer
from ..serializers import BlockSerializer, BlockDictSerializer
from .test_utils import deserialize_usage_key
from .helpers import deserialize_usage_key
class TestBlockSerializerBase(SharedModuleStoreTestCase):
class TestBlockSerializerBase(EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
"""
Base class for testing BlockSerializer and BlockDictSerializer
"""
......@@ -33,8 +35,8 @@ class TestBlockSerializerBase(SharedModuleStoreTestCase):
)
self.block_structure = get_course_blocks(
self.user,
root_block_usage_key=self.course.location,
transformers=COURSE_BLOCK_ACCESS_TRANSFORMERS + [blocks_api_transformer],
self.course.location,
BlockStructureTransformers(COURSE_BLOCK_ACCESS_TRANSFORMERS + [blocks_api_transformer]),
)
self.serializer_context = {
'request': MagicMock(),
......
lms/djangoapps/course_api/blocks/tests/test_views.py
......@@ -7,16 +7,17 @@ from string import join
from urllib import urlencode
from urlparse import urlunparse
from course_blocks.tests.helpers import EnableTransformerRegistryMixin
from opaque_keys.edx.locator import CourseLocator
from student.models import CourseEnrollment
from student.tests.factories import AdminFactory, CourseEnrollmentFactory, UserFactory
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import ToyCourseFactory
from .test_utils import deserialize_usage_key
from .helpers import deserialize_usage_key
class TestBlocksView(SharedModuleStoreTestCase):
class TestBlocksView(EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
"""
Test class for BlocksView
"""
......
lms/djangoapps/course_api/blocks/transformers/block_counts.py
"""
Block Counts Transformer
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
class BlockCountsTransformer(BlockStructureTransformer):
......
lms/djangoapps/course_api/blocks/transformers/block_depth.py
"""
Block Depth Transformer
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
class BlockDepthTransformer(BlockStructureTransformer):
......
lms/djangoapps/course_api/blocks/transformers/blocks_api.py
"""
Blocks API Transformer
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from .block_counts import BlockCountsTransformer
from .block_depth import BlockDepthTransformer
from .navigation import BlockNavigationTransformer
......
lms/djangoapps/course_api/blocks/transformers/navigation.py
"""
TODO
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from .block_depth import BlockDepthTransformer
......
lms/djangoapps/course_api/blocks/transformers/proctored_exam.py
......@@ -6,7 +6,7 @@ from django.conf import settings
from edx_proctoring.api import get_attempt_status_summary
from edx_proctoring.models import ProctoredExamStudentAttemptStatus
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
class ProctoredExamTransformer(BlockStructureTransformer):
......
lms/djangoapps/course_api/blocks/transformers/student_view.py
"""
Student View Transformer
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
class StudentViewTransformer(BlockStructureTransformer):
......
lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py
......@@ -3,7 +3,7 @@ Tests for BlockCountsTransformer.
"""
# pylint: disable=protected-access
from openedx.core.lib.block_cache.block_structure_factory import BlockStructureFactory
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory
......
lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py
......@@ -7,8 +7,8 @@ Tests for BlockDepthTransformer.
import ddt
from unittest import TestCase
from openedx.core.lib.block_cache.tests.test_utils import ChildrenMapTestMixin
from openedx.core.lib.block_cache.block_structure import BlockStructureModulestoreData
from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
from ..block_depth import BlockDepthTransformer
......@@ -34,7 +34,7 @@ class BlockDepthTransformerTestCase(TestCase, ChildrenMapTestMixin):
)
@ddt.unpack
def test_block_depth(self, block_depth, children_map, transformed_children_map, missing_blocks):
block_structure = self.create_block_structure(BlockStructureModulestoreData, children_map)
block_structure = self.create_block_structure(children_map, BlockStructureModulestoreData)
BlockDepthTransformer(block_depth).transform(usage_info=None, block_structure=block_structure)
block_structure._prune_unreachable()
self.assert_block_structure(block_structure, transformed_children_map, missing_blocks)
lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py
......@@ -7,9 +7,9 @@ from unittest import TestCase
from lms.djangoapps.course_api.blocks.transformers.block_depth import BlockDepthTransformer
from lms.djangoapps.course_api.blocks.transformers.navigation import BlockNavigationTransformer
from openedx.core.lib.block_cache.tests.test_utils import ChildrenMapTestMixin
from openedx.core.lib.block_cache.block_structure import BlockStructureModulestoreData
from openedx.core.lib.block_cache.block_structure_factory import BlockStructureFactory
from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory
from xmodule.modulestore import ModuleStoreEnum
......@@ -46,7 +46,7 @@ class BlockNavigationTransformerTestCase(TestCase, ChildrenMapTestMixin):
@ddt.unpack
def test_navigation(self, depth, nav_depth, children_map, expected_nav_map):
block_structure = self.create_block_structure(BlockStructureModulestoreData, children_map)
block_structure = self.create_block_structure(children_map, BlockStructureModulestoreData)
BlockDepthTransformer(depth).transform(usage_info=None, block_structure=block_structure)
BlockNavigationTransformer(nav_depth).transform(usage_info=None, block_structure=block_structure)
block_structure._prune_unreachable()
......
lms/djangoapps/course_api/blocks/transformers/tests/test_proctored_exam.py
......@@ -12,7 +12,7 @@ from edx_proctoring.api import (
from edx_proctoring.models import ProctoredExamStudentAttemptStatus
from edx_proctoring.runtime import set_runtime_service
from edx_proctoring.tests.test_services import MockCreditService
from lms.djangoapps.course_blocks.transformers.tests.test_helpers import CourseStructureTestCase
from lms.djangoapps.course_blocks.transformers.tests.helpers import CourseStructureTestCase
from student.tests.factories import CourseEnrollmentFactory
from ..proctored_exam import ProctoredExamTransformer
......@@ -25,6 +25,8 @@ class ProctoredExamTransformerTestCase(CourseStructureTestCase):
"""
Test behavior of ProctoredExamTransformer
"""
TRANSFORMER_CLASS_TO_TEST = ProctoredExamTransformer
def setUp(self):
"""
Setup course structure and create user for split test transformer test.
......@@ -41,8 +43,6 @@ class ProctoredExamTransformerTestCase(CourseStructureTestCase):
# Enroll user in course.
CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id, is_active=True)
self.transformer = ProctoredExamTransformer()
def setup_proctored_exam(self, block, attempt_status, user_id):
"""
Test helper to configure the given block as a proctored exam.
......@@ -123,7 +123,7 @@ class ProctoredExamTransformerTestCase(CourseStructureTestCase):
block_structure = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer},
self.transformers,
)
self.assertEqual(
set(block_structure.get_block_keys()),
......@@ -163,7 +163,7 @@ class ProctoredExamTransformerTestCase(CourseStructureTestCase):
block_structure = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer},
self.transformers,
)
self.assertEqual(
set(block_structure.get_block_keys()),
......
lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py
......@@ -4,7 +4,7 @@ Tests for StudentViewTransformer.
# pylint: disable=protected-access
from openedx.core.lib.block_cache.block_structure_factory import BlockStructureFactory
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import ToyCourseFactory
......
lms/djangoapps/course_blocks/__init__.py
"""
The Course Blocks app, built upon the Block Cache framework in
openedx.core.lib.block_cache, is a higher layer django app in LMS that
openedx.core.lib.block_structure, is a higher layer django app in LMS that
provides additional context of Courses and Users (via usage_info.py) with
implementations for Block Structure Transformers that are related to
block structure course access.
......
lms/djangoapps/course_blocks/api.py
......@@ -3,8 +3,8 @@ API entry point to the course_blocks app with top-level
get_course_blocks and clear_course_from_cache functions.
"""
from django.core.cache import cache
from openedx.core.lib.block_cache.block_cache import get_blocks, clear_block_cache
from openedx.core.lib.block_structure.manager import BlockStructureManager
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from xmodule.modulestore.django import modulestore
from .transformers import (
......@@ -29,11 +29,11 @@ COURSE_BLOCK_ACCESS_TRANSFORMERS = [
def get_course_blocks(
user,
root_block_usage_key,
transformers=None
transformers=None,
):
"""
A higher order function implemented on top of the
block_cache.get_blocks function returning a transformed block
block_structure.get_blocks function returning a transformed block
structure for the given user starting at root_block_usage_key.
Note: The current implementation requires the root_block_usage_key
......@@ -50,7 +50,7 @@ def get_course_blocks(
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is being accessed.
transformers ([BlockStructureTransformer]) - The list of
transformers (BlockStructureTransformers) - A collection of
transformers whose transform methods are to be called.
If None, COURSE_BLOCK_ACCESS_TRANSFORMERS is used.
......@@ -63,8 +63,7 @@ def get_course_blocks(
exactly equivalent to the blocks that the given user has
access.
"""
store = modulestore()
if root_block_usage_key != store.make_course_usage_key(root_block_usage_key.course_key):
if root_block_usage_key != modulestore().make_course_usage_key(root_block_usage_key.course_key):
# Enforce this check for now until MA-1604 is implemented.
# Otherwise, callers will get incorrect block data after a
# new version of the course is published, since
......@@ -72,26 +71,60 @@ def get_course_blocks(
# structures starting at the root block of the course.
raise NotImplementedError
return get_blocks(
cache,
store,
CourseUsageInfo(root_block_usage_key.course_key, user),
root_block_usage_key,
COURSE_BLOCK_ACCESS_TRANSFORMERS if transformers is None else transformers,
)
if not transformers:
transformers = BlockStructureTransformers(COURSE_BLOCK_ACCESS_TRANSFORMERS)
transformers.usage_info = CourseUsageInfo(root_block_usage_key.course_key, user)
return _get_block_structure_manager(root_block_usage_key.course_key).get_transformed(transformers)
def get_course_in_cache(course_key):
"""
A higher order function implemented on top of the
block_structure.get_collected function that returns the block
structure in the cache for the given course_key.
Returns:
BlockStructureBlockData - The collected block structure,
starting at root_block_usage_key.
"""
return _get_block_structure_manager(course_key).get_collected()
def update_course_in_cache(course_key):
"""
A higher order function implemented on top of the
block_structure.updated_collected function that updates the block
structure in the cache for the given course_key.
"""
return _get_block_structure_manager(course_key).update_collected()
def clear_course_from_cache(course_key):
"""
A higher order function implemented on top of the
block_cache.clear_block_cache function that clears the block
structure from the cache for the block structure starting at the
root block of the course for the given course_key.
block_structure.clear_block_cache function that clears the block
structure from the cache for the given course_key.
Note: See Note in get_course_blocks. Even after MA-1604 is
implemented, this implementation should still be valid since the
entire block structure of the course is cached, even though
arbitrary access to an intermediate block will be supported.
"""
course_usage_key = modulestore().make_course_usage_key(course_key)
return clear_block_cache(cache, course_usage_key)
_get_block_structure_manager(course_key).clear()
def _get_block_structure_manager(course_key):
"""
Returns the manager for managing Block Structures for the given course.
"""
store = modulestore()
course_usage_key = store.make_course_usage_key(course_key)
return BlockStructureManager(course_usage_key, store, _get_cache())
def _get_cache():
"""
Returns the storage for caching Block Structures.
"""
return cache
lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py 0 → 100644
"""
Command to load course blocks.
"""
import logging
from django.core.management.base import BaseCommand, CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from xmodule.modulestore.django import modulestore
from ...api import get_course_in_cache
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Example usage:
$ ./manage.py lms generate_course_blocks --all --settings=devstack
$ ./manage.py lms generate_course_blocks 'edX/DemoX/Demo_Course' --settings=devstack
"""
args = '<course_id course_id ...>'
help = 'Generates and stores course blocks for one or more courses.'
def add_arguments(self, parser):
"""
Entry point for subclassed commands to add custom arguments.
"""
parser.add_argument(
'--all',
help='Generate course blocks for all or specified courses.',
action='store_true',
default=False,
)
parser.add_argument(
'--dags',
help='Find and log DAGs for all or specified courses.',
action='store_true',
default=False,
)
def handle(self, *args, **options):
if options.get('all'):
course_keys = [course.id for course in modulestore().get_course_summaries()]
else:
if len(args) < 1:
raise CommandError('At least one course or --all must be specified.')
try:
course_keys = [CourseKey.from_string(arg) for arg in args]
except InvalidKeyError:
raise CommandError('Invalid key specified.')
log.info('Generating course blocks for %d courses.', len(course_keys))
log.debug('Generating course blocks for the following courses: %s', course_keys)
for course_key in course_keys:
try:
block_structure = get_course_in_cache(course_key)
if options.get('dags'):
self._find_and_log_dags(block_structure, course_key)
except Exception as ex: # pylint: disable=broad-except
log.exception(
'An error occurred while generating course blocks for %s: %s',
unicode(course_key),
ex.message,
)
log.info('Finished generating course blocks.')
def _find_and_log_dags(self, block_structure, course_key):
"""
Finds all DAGs within the given block structure.
Arguments:
BlockStructureBlockData - The block structure in which to find DAGs.
"""
log.info('DAG check starting for course %s.', unicode(course_key))
for block_key in block_structure.get_block_keys():
parents = block_structure.get_parents(block_key)
if len(parents) > 1:
log.warning(
'DAG alert - %s has multiple parents: %s.',
unicode(block_key),
[unicode(parent) for parent in parents],
)
log.info('DAG check complete for course %s.', unicode(course_key))
lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py 0 → 100644
"""
Tests for generate_course_blocks management command.
"""
from django.core.management.base import CommandError
from mock import patch
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from .. import generate_course_blocks
from ....tests.helpers import is_course_in_block_structure_cache
class TestGenerateCourseBlocks(ModuleStoreTestCase):
"""
Tests generate course blocks management command.
"""
def setUp(self):
"""
Create courses in modulestore.
"""
super(TestGenerateCourseBlocks, self).setUp()
self.course_1 = CourseFactory.create()
self.course_2 = CourseFactory.create()
self.command = generate_course_blocks.Command()
def _assert_courses_not_in_block_cache(self, *courses):
"""
Assert courses don't exist in the course block cache.
"""
for course_key in courses:
self.assertFalse(is_course_in_block_structure_cache(course_key, self.store))
def _assert_courses_in_block_cache(self, *courses):
"""
Assert courses exist in course block cache.
"""
for course_key in courses:
self.assertTrue(is_course_in_block_structure_cache(course_key, self.store))
def test_generate_all(self):
self._assert_courses_not_in_block_cache(self.course_1.id, self.course_2.id)
self.command.handle(all=True)
self._assert_courses_in_block_cache(self.course_1.id, self.course_2.id)
def test_generate_one(self):
self._assert_courses_not_in_block_cache(self.course_1.id, self.course_2.id)
self.command.handle(unicode(self.course_1.id))
self._assert_courses_in_block_cache(self.course_1.id)
self._assert_courses_not_in_block_cache(self.course_2.id)
@patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log')
def test_generate_no_dags(self, mock_log):
self.command.handle(dags=True, all=True)
self.assertEquals(mock_log.warning.call_count, 0)
@patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log')
def test_generate_with_dags(self, mock_log):
with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred):
item1 = ItemFactory.create(parent=self.course_1)
item2 = ItemFactory.create(parent=item1)
item3 = ItemFactory.create(parent=item1)
item2.children.append(item3.location)
self.store.update_item(item2, ModuleStoreEnum.UserID.mgmt_command)
self.store.publish(self.course_1.location, ModuleStoreEnum.UserID.mgmt_command)
self.command.handle(dags=True, all=True)
self.assertEquals(mock_log.warning.call_count, 1)
@patch('lms.djangoapps.course_blocks.management.commands.generate_course_blocks.log')
def test_not_found_key(self, mock_log):
self.command.handle('fake/course/id', all=False)
self.assertTrue(mock_log.exception.called)
def test_invalid_key(self):
with self.assertRaises(CommandError):
self.command.handle('not/found', all=False)
def test_no_params(self):
with self.assertRaises(CommandError):
self.command.handle(all=False)
lms/djangoapps/course_blocks/signals.py
......@@ -6,16 +6,21 @@ from django.dispatch.dispatcher import receiver
from xmodule.modulestore.django import SignalHandler
from .api import clear_course_from_cache
from .tasks import update_course_in_cache
@receiver(SignalHandler.course_published)
def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Catches the signal that a course has been published in the module
store and invalidates the corresponding cache entry if one exists.
store and creates/updates the corresponding cache entry.
"""
clear_course_from_cache(course_key)
# The countdown=0 kwarg ensures the call occurs after the signal emitter
# has finished all operations.
update_course_in_cache.apply_async([unicode(course_key)], countdown=0)
@receiver(SignalHandler.course_deleted)
def _listen_for_course_delete(sender, course_key, **kwargs): # pylint: disable=unused-argument
......
lms/djangoapps/course_blocks/tasks.py 0 → 100644
"""
Asynchronous tasks related to the Course Blocks sub-application.
"""
import logging
from celery.task import task
from opaque_keys.edx.keys import CourseKey
from . import api
log = logging.getLogger('edx.celery.task')
@task()
def update_course_in_cache(course_key):
"""
Updates the course blocks (in the database) for the specified course.
"""
course_key = CourseKey.from_string(course_key)
api.update_course_in_cache(course_key)
lms/djangoapps/course_blocks/tests/helpers.py 0 → 100644
"""
Helpers for Course Blocks tests.
"""
from openedx.core.lib.block_structure.cache import BlockStructureCache
from openedx.core.lib.block_structure.transformer_registry import TransformerRegistry
from ..api import _get_cache
class EnableTransformerRegistryMixin(object):
"""
Mixin that enables the TransformerRegistry to USE_PLUGIN_MANAGER for
finding registered transformers. USE_PLUGIN_MANAGER is set to False
for LMS unit tests to speed up performance of the unit tests, so all
registered transformers in the platform do not need to be collected.
This Mixin is expected to be used by Tests for integration testing
with all registered transformers.
"""
def setUp(self, **kwargs):
super(EnableTransformerRegistryMixin, self).setUp(**kwargs)
TransformerRegistry.USE_PLUGIN_MANAGER = True
def tearDown(self):
super(EnableTransformerRegistryMixin, self).tearDown()
TransformerRegistry.USE_PLUGIN_MANAGER = False
def is_course_in_block_structure_cache(course_key, store):
"""
Returns whether the given course is in the Block Structure cache.
"""
course_usage_key = store.make_course_usage_key(course_key)
return BlockStructureCache(_get_cache()).get(course_usage_key) is not None
lms/djangoapps/course_blocks/tests/test_signals.py 0 → 100644
"""
Unit tests for the Course Blocks signals
"""
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from ..api import get_course_blocks, _get_block_structure_manager
from ..transformers.visibility import VisibilityTransformer
from .helpers import is_course_in_block_structure_cache, EnableTransformerRegistryMixin
class CourseBlocksSignalTest(EnableTransformerRegistryMixin, ModuleStoreTestCase):
"""
Tests for the Course Blocks signal
"""
def setUp(self):
super(CourseBlocksSignalTest, self).setUp(create_user=True)
self.course = CourseFactory.create()
self.course_usage_key = self.store.make_course_usage_key(self.course.id)
def test_course_publish(self):
# course is not visible to staff only
self.assertFalse(self.course.visible_to_staff_only)
orig_block_structure = get_course_blocks(self.user, self.course_usage_key)
self.assertFalse(
VisibilityTransformer.get_visible_to_staff_only(orig_block_structure, self.course_usage_key)
)
# course becomes visible to staff only
self.course.visible_to_staff_only = True
self.store.update_item(self.course, self.user.id)
updated_block_structure = get_course_blocks(self.user, self.course_usage_key)
self.assertTrue(
VisibilityTransformer.get_visible_to_staff_only(updated_block_structure, self.course_usage_key)
)
def test_course_delete(self):
get_course_blocks(self.user, self.course_usage_key)
bs_manager = _get_block_structure_manager(self.course.id)
self.assertIsNotNone(bs_manager.get_collected())
self.assertTrue(is_course_in_block_structure_cache(self.course.id, self.store))
self.store.delete_course(self.course.id, self.user.id)
with self.assertRaises(ItemNotFoundError):
bs_manager.get_collected()
self.assertFalse(is_course_in_block_structure_cache(self.course.id, self.store))
lms/djangoapps/course_blocks/transformers/library_content.py
......@@ -3,7 +3,7 @@ Content Library Transformer.
"""
import json
from courseware.models import StudentModule
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from xmodule.library_content_module import LibraryContentModule
from xmodule.modulestore.django import modulestore
from eventtracking import tracker
......
lms/djangoapps/course_blocks/transformers/split_test.py
"""
Split Test Block Transformer
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
class SplitTestTransformer(BlockStructureTransformer):
......
lms/djangoapps/course_blocks/transformers/start_date.py
"""
Start Date Transformer implementation.
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from lms.djangoapps.courseware.access_utils import check_start_date
from xmodule.course_metadata_utils import DEFAULT_START_DATE
......
lms/djangoapps/course_blocks/transformers/tests/test_helpers.py → lms/djangoapps/course_blocks/transformers/tests/helpers.py
"""
Test helpers for testing course block transformers.
"""
from mock import patch
from course_modes.models import CourseMode
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from student.tests.factories import CourseEnrollmentFactory, UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
......@@ -11,7 +14,25 @@ from lms.djangoapps.courseware.access import has_access
from ...api import get_course_blocks
class CourseStructureTestCase(ModuleStoreTestCase):
class TransformerRegistryTestMixin(object):
"""
Mixin that overrides the TransformerRegistry so that it returns
TRANSFORMER_CLASS_TO_TEST as a registered transformer.
"""
def setUp(self):
super(TransformerRegistryTestMixin, self).setUp()
self.patcher = patch(
'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
)
mock_registry = self.patcher.start()
mock_registry.return_value = {self.TRANSFORMER_CLASS_TO_TEST}
self.transformers = BlockStructureTransformers([self.TRANSFORMER_CLASS_TO_TEST()])
def tearDown(self):
self.patcher.stop()
class CourseStructureTestCase(TransformerRegistryTestMixin, ModuleStoreTestCase):
"""
Helper for test cases that need to build course structures.
"""
......@@ -157,6 +178,8 @@ class CourseStructureTestCase(ModuleStoreTestCase):
for block_hierarchy in course_hierarchy:
self.add_parents(block_hierarchy, block_map)
publish_course(block_map['course'])
return block_map
def get_block_key_set(self, blocks, *refs):
......@@ -170,7 +193,7 @@ class CourseStructureTestCase(ModuleStoreTestCase):
return set([xblock.location for xblock in xblocks])
class BlockParentsMapTestCase(ModuleStoreTestCase):
class BlockParentsMapTestCase(TransformerRegistryTestMixin, ModuleStoreTestCase):
"""
Test helper class for creating a test course of
a graph of vertical blocks based on a parents_map.
......@@ -203,7 +226,6 @@ class BlockParentsMapTestCase(ModuleStoreTestCase):
if i == 0:
continue # course already created
# create the block as a vertical
self.xblock_keys.append(
ItemFactory.create(
parent=self.get_block(parents_index[0]),
......@@ -252,66 +274,23 @@ class BlockParentsMapTestCase(ModuleStoreTestCase):
transformers result and the current implementation of
has_access.
transformers (BlockStructureTransformer): An optional list
of transformer that are to be executed. If not
transformers (BlockStructureTransformers): An optional collection
of transformers that are to be executed. If not
provided, the default value used by get_course_blocks
is used.
"""
def check_results(user, expected_accessible_blocks, blocks_with_differing_access):
"""
Verifies the results of transforming the blocks in the
course for the given user.
"""
self.client.login(username=user.username, password=self.password)
block_structure = get_course_blocks(user, self.course.location, transformers=transformers)
# Enumerate through all the blocks that were created in the
# course
for i, xblock_key in enumerate(self.xblock_keys):
# verify existence of the block
block_structure_result = block_structure.has_block(xblock_key)
has_access_result = bool(has_access(user, 'load', self.get_block(i), course_key=self.course.id))
# compare with expected value
self.assertEquals(
block_structure_result,
i in expected_accessible_blocks,
"block_structure return value {0} not equal to expected value for block {1} for user {2}".format(
block_structure_result, i, user.username
)
)
# compare with has_access result
if i in blocks_with_differing_access:
self.assertNotEqual(
block_structure_result,
has_access_result,
"block structure ({0}) & has_access ({1}) results are equal for block {2} for user {3}".format(
block_structure_result, has_access_result, i, user.username
)
)
else:
self.assertEquals(
block_structure_result,
has_access_result,
"block structure ({0}) & has_access ({1}) results not equal for block {2} for user {3}".format(
block_structure_result, has_access_result, i, user.username
)
)
self.client.logout()
publish_course(self.course)
# verify given test user has access to expected blocks
check_results(
self._check_results(
test_user,
expected_user_accessible_blocks,
blocks_with_differing_access
blocks_with_differing_access,
transformers,
)
# verify staff has access to all blocks
check_results(self.staff, set(range(len(self.parents_map))), {})
self._check_results(self.staff, set(range(len(self.parents_map))), {}, transformers)
def get_block(self, block_index):
"""
......@@ -320,12 +299,65 @@ class BlockParentsMapTestCase(ModuleStoreTestCase):
"""
return modulestore().get_item(self.xblock_keys[block_index])
def _check_results(self, user, expected_accessible_blocks, blocks_with_differing_access, transformers):
"""
Verifies the results of transforming the blocks in the
course for the given user.
"""
self.client.login(username=user.username, password=self.password)
block_structure = get_course_blocks(user, self.course.location, transformers)
for i, xblock_key in enumerate(self.xblock_keys):
# compute access results of the block
block_structure_result = block_structure.has_block(xblock_key)
has_access_result = bool(has_access(user, 'load', self.get_block(i), course_key=self.course.id))
# compare with expected value
self.assertEquals(
block_structure_result,
i in expected_accessible_blocks,
"block_structure return value {0} not equal to expected value for block {1} for user {2}".format(
block_structure_result, i, user.username
)
)
# compare with has_access_result
if i in blocks_with_differing_access:
self.assertNotEqual(
block_structure_result,
has_access_result,
"block structure ({0}) & has_access ({1}) results are equal for block {2} for user {3}".format(
block_structure_result, has_access_result, i, user.username
)
)
else:
self.assertEquals(
block_structure_result,
has_access_result,
"block structure ({0}) & has_access ({1}) results not equal for block {2} for user {3}".format(
block_structure_result, has_access_result, i, user.username
)
)
self.client.logout()
def update_block(block):
"""
Helper method to update the block in the modulestore
"""
return modulestore().update_item(block, 'test_user')
return modulestore().update_item(block, ModuleStoreEnum.UserID.test)
def publish_course(course):
"""
Helper method to publish the course (from draft to publish branch)
"""
store = modulestore()
with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course.id):
store.publish(course.location, ModuleStoreEnum.UserID.test)
def create_location(org, course, run, block_type, block_id):
......
lms/djangoapps/course_blocks/transformers/tests/test_library_content.py
......@@ -4,9 +4,11 @@ Tests for ContentLibraryTransformer.
import mock
from student.tests.factories import CourseEnrollmentFactory
from course_blocks.transformers.library_content import ContentLibraryTransformer
from course_blocks.api import get_course_blocks, clear_course_from_cache
from lms.djangoapps.course_blocks.transformers.tests.test_helpers import CourseStructureTestCase
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from ...api import get_course_blocks, clear_course_from_cache
from ..library_content import ContentLibraryTransformer
from .helpers import CourseStructureTestCase
class MockedModule(object):
......@@ -24,6 +26,7 @@ class ContentLibraryTransformerTestCase(CourseStructureTestCase):
"""
ContentLibraryTransformer Test
"""
TRANSFORMER_CLASS_TO_TEST = ContentLibraryTransformer
def setUp(self):
"""
......@@ -40,9 +43,6 @@ class ContentLibraryTransformerTestCase(CourseStructureTestCase):
# Enroll user in course.
CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id, is_active=True)
self.selected_module = MockedModule('{"selected": [["vertical", "vertical_vertical2"]]}')
self.transformer = ContentLibraryTransformer()
def get_course_hierarchy(self):
"""
Get a course hierarchy to test with.
......@@ -116,7 +116,7 @@ class ContentLibraryTransformerTestCase(CourseStructureTestCase):
raw_block_structure = get_course_blocks(
self.user,
self.course.location,
transformers={}
transformers=BlockStructureTransformers(),
)
self.assertEqual(len(list(raw_block_structure.get_block_keys())), len(self.blocks))
......@@ -124,7 +124,7 @@ class ContentLibraryTransformerTestCase(CourseStructureTestCase):
trans_block_structure = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer}
self.transformers,
)
# Should dynamically assign a block to student
......@@ -141,14 +141,14 @@ class ContentLibraryTransformerTestCase(CourseStructureTestCase):
# Check course structure again, with mocked selected modules for a user.
with mock.patch(
'course_blocks.transformers.library_content.ContentLibraryTransformer._get_student_module',
return_value=self.selected_module
'lms.djangoapps.course_blocks.transformers.library_content.ContentLibraryTransformer._get_student_module',
return_value=MockedModule('{"selected": [["vertical", "vertical_vertical2"]]}'),
):
clear_course_from_cache(self.course.id)
trans_block_structure = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer}
self.transformers,
)
self.assertEqual(
set(trans_block_structure.get_block_keys()),
......
lms/djangoapps/course_blocks/transformers/tests/test_split_test.py
......@@ -11,7 +11,7 @@ from xmodule.modulestore.tests.factories import check_mongo_calls, check_mongo_c
from ...api import get_course_blocks
from ..user_partitions import UserPartitionTransformer, _get_user_partition_groups
from .test_helpers import CourseStructureTestCase, create_location
from .helpers import CourseStructureTestCase, create_location
@ddt.ddt
......@@ -20,6 +20,7 @@ class SplitTestTransformerTestCase(CourseStructureTestCase):
SplitTestTransformer Test
"""
TEST_PARTITION_ID = 0
TRANSFORMER_CLASS_TO_TEST = UserPartitionTransformer
def setUp(self):
"""
......@@ -47,8 +48,6 @@ class SplitTestTransformerTestCase(CourseStructureTestCase):
# Enroll user in course.
CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id, is_active=True)
self.transformer = UserPartitionTransformer()
def get_course_hierarchy(self):
"""
Get a course hierarchy to test with.
......@@ -193,7 +192,7 @@ class SplitTestTransformerTestCase(CourseStructureTestCase):
block_structure1 = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer},
self.transformers,
)
self.assertEqual(
set(block_structure1.get_block_keys()),
......@@ -208,17 +207,16 @@ class SplitTestTransformerTestCase(CourseStructureTestCase):
self.assertEquals(len(user_groups), 1)
# calling twice should result in the same block set
with check_mongo_calls_range(min_finds=1):
block_structure1 = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer},
)
block_structure1 = get_course_blocks(
self.user,
self.course.location,
self.transformers,
)
with check_mongo_calls(0):
block_structure2 = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer},
self.transformers,
)
self.assertEqual(
set(block_structure1.get_block_keys()),
......
lms/djangoapps/course_blocks/transformers/tests/test_start_date.py
......@@ -8,7 +8,7 @@ from mock import patch
from courseware.tests.factories import BetaTesterFactory
from ..start_date import StartDateTransformer, DEFAULT_START_DATE
from .test_helpers import BlockParentsMapTestCase, update_block
from .helpers import BlockParentsMapTestCase, update_block
@ddt.ddt
......@@ -18,6 +18,7 @@ class StartDateTransformerTestCase(BlockParentsMapTestCase):
"""
STUDENT = 1
BETA_USER = 2
TRANSFORMER_CLASS_TO_TEST = StartDateTransformer
class StartDateType(object):
"""
......@@ -114,5 +115,5 @@ class StartDateTransformerTestCase(BlockParentsMapTestCase):
self.beta_user if user_type == self.BETA_USER else self.student,
expected_student_visible_blocks,
blocks_with_differing_student_access,
[StartDateTransformer()],
self.transformers,
)
lms/djangoapps/course_blocks/transformers/tests/test_user_partitions.py
......@@ -15,13 +15,15 @@ from xmodule.modulestore.tests.factories import CourseFactory
from ...api import get_course_blocks
from ..user_partitions import UserPartitionTransformer, _MergedGroupAccess
from .test_helpers import CourseStructureTestCase, update_block
from .helpers import CourseStructureTestCase, update_block
class UserPartitionTestMixin(object):
"""
Helper Mixin for testing user partitions.
"""
TRANSFORMER_CLASS_TO_TEST = UserPartitionTransformer
def setup_groups_partitions(self, num_user_partitions=1, num_groups=4):
"""
Sets up groups and user partitions for testing.
......@@ -90,8 +92,6 @@ class UserPartitionTransformerTestCase(UserPartitionTestMixin, CourseStructureTe
# Set up cohorts.
self.setup_cohorts(self.course)
self.transformer = UserPartitionTransformer()
def get_course_hierarchy(self):
"""
Returns a course hierarchy to test with.
......@@ -204,7 +204,7 @@ class UserPartitionTransformerTestCase(UserPartitionTestMixin, CourseStructureTe
trans_block_structure = get_course_blocks(
self.user,
self.course.location,
transformers={self.transformer}
self.transformers,
)
self.assertSetEqual(
set(trans_block_structure.get_block_keys()),
......
lms/djangoapps/course_blocks/transformers/tests/test_visibility.py
......@@ -4,7 +4,7 @@ Tests for VisibilityTransformer.
import ddt
from ..visibility import VisibilityTransformer
from .test_helpers import BlockParentsMapTestCase, update_block
from .helpers import BlockParentsMapTestCase, update_block
@ddt.ddt
......@@ -12,6 +12,8 @@ class VisibilityTransformerTestCase(BlockParentsMapTestCase):
"""
VisibilityTransformer Test
"""
TRANSFORMER_CLASS_TO_TEST = VisibilityTransformer
# Following test cases are based on BlockParentsMapTestCase.parents_map
@ddt.data(
({}, {0, 1, 2, 3, 4, 5, 6}, {}),
......@@ -39,5 +41,5 @@ class VisibilityTransformerTestCase(BlockParentsMapTestCase):
self.student,
expected_visible_blocks,
blocks_with_differing_access,
[VisibilityTransformer()],
self.transformers,
)
lms/djangoapps/course_blocks/transformers/user_partitions.py
"""
User Partitions Transformer
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from .split_test import SplitTestTransformer
from .utils import get_field_on_block
......@@ -66,12 +66,7 @@ class UserPartitionTransformer(BlockStructureTransformer):
def transform(self, usage_info, block_structure):
"""
Mutates block_structure and block_data based on the given
usage_info.
Arguments:
usage_info (object)
block_structure (BlockStructureCollectedData)
Mutates block_structure based on the given usage_info.
"""
SplitTestTransformer().transform(usage_info, block_structure)
......
lms/djangoapps/course_blocks/transformers/visibility.py
"""
Visibility Transformer implementation.
"""
from openedx.core.lib.block_cache.transformer import BlockStructureTransformer
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
class VisibilityTransformer(BlockStructureTransformer):
......
lms/envs/dev.py
......@@ -87,11 +87,6 @@ CACHES = {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_course_structure_mem_cache',
},
'lms.course_blocks': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'KEY_FUNCTION': 'util.memcache.safe_key',
'LOCATION': 'lms_course_blocks_cache',
},
}
......
lms/envs/test.py
......@@ -226,14 +226,6 @@ CACHES = {
'course_structure_cache': {
'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
},
'block_cache': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_location_block_cache',
},
'lms.course_blocks': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'edx_location_course_blocks',
},
}
# Dummy secret key for dev
......@@ -572,3 +564,8 @@ JWT_AUTH.update({
'JWT_ISSUER': 'https://test-provider/oauth2',
'JWT_AUDIENCE': 'test-key',
})
# Disable the use of the plugin manager in the transformer registry for
# better performant unit tests.
from openedx.core.lib.block_structure.transformer_registry import TransformerRegistry
TransformerRegistry.USE_PLUGIN_MANAGER = False
openedx/core/djangoapps/content/course_overviews/management/commands/generate_course_overview.py
......@@ -34,7 +34,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
if options['all']:
course_keys = [course.id for course in modulestore().get_courses()]
course_keys = [course.id for course in modulestore().get_course_summaries()]
else:
if len(args) < 1:
raise CommandError('At least one course or --all must be specified.')
......
openedx/core/lib/block_cache/block_cache.py deleted 100644 → 0
"""
Top-level module for the Block Cache framework with higher order
functions for getting and clearing cached blocks.
"""
from .block_structure_factory import BlockStructureFactory
from .exceptions import TransformerException
from .transformer_registry import TransformerRegistry
def get_blocks(cache, modulestore, usage_info, root_block_usage_key, transformers):
"""
Top-level function in the Block Cache framework that manages
the cache (populating it and updating it when needed), calls the
transformers as appropriate (collect and transform methods), and
accessing the modulestore when needed (at cache miss).
Arguments:
cache (django.core.cache.backends.base.BaseCache) - The
cache to use for storing/retrieving the block structure's
collected data.
modulestore (ModuleStoreRead) - The modulestore that
contains the data for the xBlock objects corresponding to
the block structure.
usage_info (any negotiated type) - A usage-specific object
that is forwarded to all requested Transformers in order
to apply a usage-specific transform. For example, an
instance of usage_info would contain a user object for
which the transform should be applied.
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is being accessed.
transformers ([BlockStructureTransformer]) - The list of
transformers whose transform methods are to be called.
This list should be a subset of the list of registered
transformers in the Transformer Registry.
Returns:
BlockStructureBlockData - A transformed block structure,
starting at root_block_usage_key, that has undergone the
transform methods in the given transformers with the
given usage_info.
"""
# Verify that all requested transformers are registered in the
# Transformer Registry.
unregistered_transformers = TransformerRegistry.find_unregistered(transformers)
if unregistered_transformers:
raise TransformerException(
"The following requested transformers are not registered: {}".format(unregistered_transformers)
)
# Load the cached block structure.
root_block_structure = BlockStructureFactory.create_from_cache(root_block_usage_key, cache, transformers)
# On cache miss, execute the collect phase and update the cache.
if not root_block_structure:
# Create the block structure from the modulestore.
root_block_structure = BlockStructureFactory.create_from_modulestore(root_block_usage_key, modulestore)
# Collect data from each registered transformer.
for transformer in TransformerRegistry.get_registered_transformers():
root_block_structure._add_transformer(transformer) # pylint: disable=protected-access
transformer.collect(root_block_structure)
# Collect all fields that were requested by the transformers.
root_block_structure._collect_requested_xblock_fields() # pylint: disable=protected-access
# Cache this information.
BlockStructureFactory.serialize_to_cache(root_block_structure, cache)
# Execute requested transforms on block structure.
for transformer in transformers:
transformer.transform(usage_info, root_block_structure)
# Prune the block structure to remove any unreachable blocks.
root_block_structure._prune_unreachable() # pylint: disable=protected-access
return root_block_structure
def clear_block_cache(cache, root_block_usage_key):
"""
Removes the block structure associated with the given root block
key.
"""
BlockStructureFactory.remove_from_cache(root_block_usage_key, cache)
openedx/core/lib/block_cache/tests/test_block_cache.py deleted 100644 → 0
"""
Tests for block_cache.py
"""
from django.core.cache import get_cache
from django.conf import settings
from mock import patch
from unittest import TestCase, skipUnless
from ..block_cache import get_blocks
from ..exceptions import TransformerException
from .test_utils import (
MockModulestoreFactory, MockCache, MockTransformer, ChildrenMapTestMixin
)
@patch('openedx.core.lib.block_cache.transformer_registry.TransformerRegistry.get_available_plugins')
class TestBlockCache(TestCase, ChildrenMapTestMixin):
"""
Test class for block cache functionality.
"""
class TestTransformer1(MockTransformer):
"""
Test Transformer class.
"""
@classmethod
def block_key(cls):
"""
Returns the dictionary key for transformer block data.
"""
return 't1.key1'
@classmethod
def block_val(cls, block_key):
"""
Returns the dictionary value for transformer block data for
the block identified by the given block key.
"""
return 't1.val1.' + unicode(block_key)
@classmethod
def collect(cls, block_structure):
"""
Sets transformer block data for each block in the structure
as it is visited using topological traversal.
"""
for block_key in block_structure.topological_traversal():
block_structure.set_transformer_block_field(
block_key, cls, cls.block_key(), cls.block_val(block_key)
)
def transform(self, usage_info, block_structure):
"""
Verifies the transformer block data set for each block
in the structure.
"""
def assert_collected_value(block_key):
"""
Verifies the transformer block data for the given
block equates the value stored in the collect method.
"""
assert (
block_structure.get_transformer_block_field(
block_key,
self,
self.block_key()
) == self.block_val(block_key)
)
for block_key in block_structure.topological_traversal():
assert_collected_value(block_key)
def setUp(self):
super(TestBlockCache, self).setUp()
self.children_map = self.SIMPLE_CHILDREN_MAP
self.usage_info = None
self.mock_cache = MockCache()
self.modulestore = MockModulestoreFactory.create(self.children_map)
self.transformers = [self.TestTransformer1()]
def test_get_blocks(self, mock_available_transforms):
mock_available_transforms.return_value = {transformer.name(): transformer for transformer in self.transformers}
block_structure = get_blocks(
self.mock_cache, self.modulestore, self.usage_info, root_block_usage_key=0, transformers=self.transformers
)
self.assert_block_structure(block_structure, self.children_map)
def test_unregistered_transformers(self, mock_available_transforms):
mock_available_transforms.return_value = {}
with self.assertRaisesRegexp(TransformerException, "requested transformers are not registered"):
get_blocks(
self.mock_cache,
self.modulestore,
self.usage_info,
root_block_usage_key=0,
transformers=self.transformers,
)
@skipUnless(settings.ROOT_URLCONF == 'lms.urls', 'Test only valid in lms')
def test_block_caching(self, mock_available_transforms):
mock_available_transforms.return_value = {transformer.name(): transformer for transformer in self.transformers}
cache = get_cache('block_cache')
for iteration in range(2):
self.modulestore.get_items_call_count = 0
block_structure = get_blocks(
cache, self.modulestore, self.usage_info, root_block_usage_key=0, transformers=self.transformers
)
self.assert_block_structure(block_structure, self.children_map)
if iteration == 0:
self.assertGreater(self.modulestore.get_items_call_count, 0)
else:
self.assertEquals(self.modulestore.get_items_call_count, 0)
openedx/core/lib/block_cache/tests/test_block_structure_factory.py deleted 100644 → 0
"""
Tests for block_structure_factory.py
"""
# pylint: disable=protected-access
from mock import patch
from unittest import TestCase
from ..block_structure_factory import BlockStructureFactory
from .test_utils import (
MockCache, MockModulestoreFactory, MockTransformer, ChildrenMapTestMixin
)
class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin):
"""
Tests for BlockStructureFactory
"""
def setUp(self):
super(TestBlockStructureFactory, self).setUp()
self.children_map = self.SIMPLE_CHILDREN_MAP
self.modulestore = MockModulestoreFactory.create(self.children_map)
self.block_structure = BlockStructureFactory.create_from_modulestore(
root_block_usage_key=0, modulestore=self.modulestore
)
self.transformers = [MockTransformer]
mock_registry = patch(
'openedx.core.lib.block_cache.transformer_registry.TransformerRegistry.get_available_plugins'
)
mock_registry.return_value = {transformer.name(): transformer for transformer in self.transformers}
self.addCleanup(mock_registry.stop)
mock_registry.start()
def add_transformers(self):
"""
Add each registered transformer to the block structure.
Mimic collection by setting test transformer block data.
"""
for transformer in self.transformers:
self.block_structure._add_transformer(transformer)
self.block_structure.set_transformer_block_field(
usage_key=0, transformer=transformer, key='test', value='{} val'.format(transformer.name())
)
def test_create_from_modulestore(self):
self.assert_block_structure(self.block_structure, self.children_map)
def test_not_in_cache(self):
cache = MockCache()
self.assertIsNone(
BlockStructureFactory.create_from_cache(
root_block_usage_key=0,
cache=cache,
transformers=self.transformers,
)
)
def test_uncollected_transformers(self):
cache = MockCache()
# serialize the structure to cache, but without collecting any transformer data
BlockStructureFactory.serialize_to_cache(self.block_structure, cache)
with patch('openedx.core.lib.block_cache.block_structure_factory.logger.info') as mock_logger:
# cached data does not have collected information for all registered transformers
self.assertIsNone(
BlockStructureFactory.create_from_cache(
root_block_usage_key=0,
cache=cache,
transformers=self.transformers,
)
)
self.assertTrue(mock_logger.called)
def test_cache(self):
cache = MockCache()
# collect transformer data
self.add_transformers()
# serialize to cache
BlockStructureFactory.serialize_to_cache(self.block_structure, cache)
# test re-create from cache
self.modulestore.get_items_call_count = 0
from_cache_block_structure = BlockStructureFactory.create_from_cache(
root_block_usage_key=0,
cache=cache,
transformers=self.transformers,
)
self.assertIsNotNone(from_cache_block_structure)
self.assert_block_structure(from_cache_block_structure, self.children_map)
self.assertEquals(self.modulestore.get_items_call_count, 0)
def test_remove_from_cache(self):
cache = MockCache()
# collect transformer data
self.add_transformers()
# serialize to cache
BlockStructureFactory.serialize_to_cache(self.block_structure, cache)
# remove from cache
BlockStructureFactory.remove_from_cache(root_block_usage_key=0, cache=cache)
self.assertIsNone(
BlockStructureFactory.create_from_cache(
root_block_usage_key=0,
cache=cache,
transformers=self.transformers
)
)
openedx/core/lib/block_cache/__init__.py → openedx/core/lib/block_structure/__init__.py
"""
The block_cache django app provides an extensible framework for caching
The block_structure django app provides an extensible framework for caching
data of block structures from the modulestore.
Dual-Phase. The framework is meant to be used in 2 phases.
......
openedx/core/lib/block_cache/block_structure.py → openedx/core/lib/block_structure/block_structure.py
......@@ -177,7 +177,7 @@ class BlockStructure(object):
)
#--- Internal methods ---#
# To be used within the block_cache framework or by tests.
# To be used within the block_structure framework or by tests.
def _prune_unreachable(self):
"""
......@@ -488,18 +488,8 @@ class BlockStructureBlockData(BlockStructure):
for _ in self.topological_traversal(filter_func=filter_func, **kwargs):
pass
def get_block_keys(self):
"""
Returns the block keys in the block structure.
Returns:
iterator(UsageKey) - An iterator of the usage
keys of all the blocks in the block structure.
"""
return self._block_relations.iterkeys()
#--- Internal methods ---#
# To be used within the block_cache framework or by tests.
# To be used within the block_structure framework or by tests.
def _get_transformer_data_version(self, transformer):
"""
......@@ -571,7 +561,7 @@ class BlockStructureModulestoreData(BlockStructureBlockData):
return self._xblock_map[usage_key]
#--- Internal methods ---#
# To be used within the block_cache framework or by tests.
# To be used within the block_structure framework or by tests.
def _add_xblock(self, usage_key, xblock):
"""
......
openedx/core/lib/block_cache/block_structure_factory.py → openedx/core/lib/block_structure/cache.py
"""
Module for factory class for BlockStructure objects.
Module for the Cache class for BlockStructure objects.
"""
# pylint: disable=protected-access
from logging import getLogger
from openedx.core.lib.cache_utils import zpickle, zunpickle
from .block_structure import BlockStructureBlockData, BlockStructureModulestoreData
from .block_structure import BlockStructureModulestoreData
logger = getLogger(__name__) # pylint: disable=C0103
class BlockStructureFactory(object):
class BlockStructureCache(object):
"""
Factory class for BlockStructure objects.
Cache for BlockStructure objects.
"""
@classmethod
def create_from_modulestore(cls, root_block_usage_key, modulestore):
def __init__(self, cache):
"""
Creates and returns a block structure from the modulestore
starting at the given root_block_usage_key.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be created.
modulestore (ModuleStoreRead) - The modulestore that
contains the data for the xBlocks within the block
structure starting at root_block_usage_key.
Returns:
BlockStructureModulestoreData - The created block structure
with instantiated xBlocks from the given modulestore
starting at root_block_usage_key.
cache (django.core.cache.backends.base.BaseCache) - The
cache into which cacheable data of the block structure
is to be serialized.
"""
# Create block structure.
block_structure = BlockStructureModulestoreData(root_block_usage_key)
# Create internal set of blocks visited to use when recursing.
blocks_visited = set()
def build_block_structure(xblock):
"""
Recursively update the block structure with the given xBlock
and its descendants.
"""
# Check if the xblock was already visited (can happen in
# DAGs).
if xblock.location in blocks_visited:
return
# Add the xBlock.
blocks_visited.add(xblock.location)
block_structure._add_xblock(xblock.location, xblock)
# Add relations with its children and recurse.
for child in xblock.get_children():
block_structure._add_relation(xblock.location, child.location)
build_block_structure(child)
root_xblock = modulestore.get_item(root_block_usage_key, depth=None)
build_block_structure(root_xblock)
return block_structure
self._cache = cache
@classmethod
def serialize_to_cache(cls, block_structure, cache):
def add(self, block_structure):
"""
Store a compressed and pickled serialization of the given
block structure into the given cache.
......@@ -77,10 +37,6 @@ class BlockStructureFactory(object):
Arguments:
block_structure (BlockStructure) - The block structure
that is to be serialized to the given cache.
cache (django.core.cache.backends.base.BaseCache) - The
cache into which cacheable data of the block structure
is to be serialized.
"""
data_to_cache = (
block_structure._block_relations,
......@@ -88,8 +44,8 @@ class BlockStructureFactory(object):
block_structure._block_data_map
)
zp_data_to_cache = zpickle(data_to_cache)
cache.set(
cls._encode_root_cache_key(block_structure.root_block_usage_key),
self._cache.set(
self._encode_root_cache_key(block_structure.root_block_usage_key),
zp_data_to_cache
)
logger.debug(
......@@ -98,8 +54,7 @@ class BlockStructureFactory(object):
len(zp_data_to_cache),
)
@classmethod
def create_from_cache(cls, root_block_usage_key, cache, transformers):
def get(self, root_block_usage_key):
"""
Deserializes and returns the block structure starting at
root_block_usage_key from the given cache, if it's found in the cache.
......@@ -112,28 +67,18 @@ class BlockStructureFactory(object):
of the block structure that is to be deserialized from
the given cache.
cache (django.core.cache.backends.base.BaseCache) - The
cache from which the block structure is to be
deserialized.
transformers ([BlockStructureTransformer]) - A list of
transformers for which the block structure will be
transformed.
Returns:
BlockStructure - The deserialized block structure starting
at root_block_usage_key, if found in the cache.
NoneType - If the root_block_usage_key is not found in the cache
or if the cached data is outdated for one or more of the
given transformers.
NoneType - If the root_block_usage_key is not found in the cache.
"""
# Find root_block_usage_key in the cache.
zp_data_from_cache = cache.get(cls._encode_root_cache_key(root_block_usage_key))
zp_data_from_cache = self._cache.get(self._encode_root_cache_key(root_block_usage_key))
if not zp_data_from_cache:
logger.debug(
"BlockStructure %r not found in the cache.",
"Did not find BlockStructure %r in the cache.",
root_block_usage_key,
)
return None
......@@ -146,47 +91,28 @@ class BlockStructureFactory(object):
# Deserialize and construct the block structure.
block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache)
block_structure = BlockStructureBlockData(root_block_usage_key)
block_structure = BlockStructureModulestoreData(root_block_usage_key)
block_structure._block_relations = block_relations
block_structure._transformer_data = transformer_data
block_structure._block_data_map = block_data_map
# Verify that the cached data for all the given transformers are
# for their latest versions.
outdated_transformers = {}
for transformer in transformers:
cached_transformer_version = block_structure._get_transformer_data_version(transformer)
if transformer.VERSION != cached_transformer_version:
outdated_transformers[transformer.name()] = "version: {}, cached: {}".format(
transformer.VERSION,
cached_transformer_version,
)
if outdated_transformers:
logger.info(
"Collected data for the following transformers are outdated:\n%s.",
'\n'.join([t_name + ": " + t_value for t_name, t_value in outdated_transformers.iteritems()]),
)
return None
return block_structure
@classmethod
def remove_from_cache(cls, root_block_usage_key, cache):
def delete(self, root_block_usage_key):
"""
Removes the block structure for the given root_block_usage_key
Deletes the block structure for the given root_block_usage_key
from the given cache.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be removed from
the given cache.
cache (django.core.cache.backends.base.BaseCache) - The
cache from which the block structure is to be
removed.
the cache.
"""
cache.delete(cls._encode_root_cache_key(root_block_usage_key))
# TODO also remove all block data?
self._cache.delete(self._encode_root_cache_key(root_block_usage_key))
logger.debug(
"Deleted BlockStructure %r from the cache.",
root_block_usage_key,
)
@classmethod
def _encode_root_cache_key(cls, root_block_usage_key):
......
openedx/core/lib/block_structure/factory.py 0 → 100644
"""
Module for factory class for BlockStructure objects.
"""
from .block_structure import BlockStructureModulestoreData
class BlockStructureFactory(object):
"""
Factory class for BlockStructure objects.
"""
@classmethod
def create_from_modulestore(cls, root_block_usage_key, modulestore):
"""
Creates and returns a block structure from the modulestore
starting at the given root_block_usage_key.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be created.
modulestore (ModuleStoreRead) - The modulestore that
contains the data for the xBlocks within the block
structure starting at root_block_usage_key.
Returns:
BlockStructureModulestoreData - The created block structure
with instantiated xBlocks from the given modulestore
starting at root_block_usage_key.
Raises:
xmodule.modulestore.exceptions.ItemNotFoundError if a block for
root_block_usage_key is not found in the modulestore.
"""
block_structure = BlockStructureModulestoreData(root_block_usage_key)
blocks_visited = set()
def build_block_structure(xblock):
"""
Recursively update the block structure with the given xBlock
and its descendants.
"""
# Check if the xblock was already visited (can happen in
# DAGs).
if xblock.location in blocks_visited:
return
# Add the xBlock.
blocks_visited.add(xblock.location)
block_structure._add_xblock(xblock.location, xblock) # pylint: disable=protected-access
# Add relations with its children and recurse.
for child in xblock.get_children():
block_structure._add_relation(xblock.location, child.location) # pylint: disable=protected-access
build_block_structure(child)
root_xblock = modulestore.get_item(root_block_usage_key, depth=None)
build_block_structure(root_xblock)
return block_structure
@classmethod
def create_from_cache(cls, root_block_usage_key, block_structure_cache):
"""
Deserializes and returns the block structure starting at
root_block_usage_key from the given cache, if it's found in the cache.
The given root_block_usage_key must equate the root_block_usage_key
previously passed to serialize_to_cache.
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is to be deserialized from
the given cache.
block_structure_cache (BlockStructureCache) - The
cache from which the block structure is to be
deserialized.
Returns:
BlockStructure - The deserialized block structure starting
at root_block_usage_key, if found in the cache.
NoneType - If the root_block_usage_key is not found in the cache.
"""
return block_structure_cache.get(root_block_usage_key)
openedx/core/lib/block_structure/manager.py 0 → 100644
"""
Top-level module for the Block Structure framework with a class for managing
BlockStructures.
"""
from .factory import BlockStructureFactory
from .cache import BlockStructureCache
from .transformers import BlockStructureTransformers
class BlockStructureManager(object):
"""
Top-level class for managing Block Structures.
"""
def __init__(self, root_block_usage_key, modulestore, cache):
"""
Arguments:
root_block_usage_key (UsageKey) - The usage_key for the root
of the block structure that is being accessed.
modulestore (ModuleStoreRead) - The modulestore that
contains the data for the xBlock objects corresponding to
the block structure.
cache (django.core.cache.backends.base.BaseCache) - The
cache to use for storing/retrieving the block structure's
collected data.
"""
self.root_block_usage_key = root_block_usage_key
self.modulestore = modulestore
self.block_structure_cache = BlockStructureCache(cache)
def get_transformed(self, transformers):
"""
Returns the transformed Block Structure for the root_block_usage_key,
getting block data from the cache and modulestore, as needed.
Details: Same as the get_collected method, except the transformers'
transform methods are also called.
Arguments:
transformers (BlockStructureTransformers) - Collection of
transformers to apply.
Returns:
BlockStructureBlockData - A transformed block structure,
starting at self.root_block_usage_key.
"""
block_structure = self.get_collected()
transformers.transform(block_structure)
return block_structure
def get_collected(self):
"""
Returns the collected Block Structure for the root_block_usage_key,
getting block data from the cache and modulestore, as needed.
Details: The cache is updated if needed (if outdated or empty),
the modulestore is accessed if needed (at cache miss), and the
transformers data is collected if needed.
Returns:
BlockStructureBlockData - A collected block structure,
starting at root_block_usage_key, with collected data
from each registered transformer.
"""
block_structure = BlockStructureFactory.create_from_cache(
self.root_block_usage_key,
self.block_structure_cache
)
cache_miss = block_structure is None
if cache_miss or BlockStructureTransformers.is_collected_outdated(block_structure):
block_structure = BlockStructureFactory.create_from_modulestore(
self.root_block_usage_key,
self.modulestore
)
BlockStructureTransformers.collect(block_structure)
self.block_structure_cache.add(block_structure)
return block_structure
def update_collected(self):
"""
Updates the collected Block Structure for the root_block_usage_key.
Details: The cache is cleared and updated by collecting transformers
data from the modulestore.
"""
self.clear()
self.get_collected()
def clear(self):
"""
Removes cached data for the block structure associated with the given
root block key.
"""
self.block_structure_cache.delete(self.root_block_usage_key)
openedx/core/lib/block_cache/tests/test_utils.py → openedx/core/lib/block_structure/tests/helpers.py
"""
Common utilities for tests in block_cache module
Common utilities for tests in block_structure module
"""
# pylint: disable=protected-access
from contextlib import contextmanager
from mock import patch
from xmodule.modulestore.exceptions import ItemNotFoundError
from ..block_structure import BlockStructureBlockData
from ..transformer import BlockStructureTransformer
......@@ -55,9 +59,14 @@ class MockModulestore(object):
"""
Returns the mock XBlock (MockXBlock) associated with the
given block_key.
Raises ItemNotFoundError if the item is not found.
"""
self.get_items_call_count += 1
return self.blocks.get(block_key)
item = self.blocks.get(block_key)
if not item:
raise ItemNotFoundError
return item
class MockCache(object):
......@@ -68,11 +77,13 @@ class MockCache(object):
def __init__(self):
# An in-memory map of cache keys to cache values.
self.map = {}
self.set_call_count = 0
def set(self, key, val):
"""
Associates the given key with the given value in the cache.
"""
self.set_call_count += 1
self.map[key] = val
def get(self, key, default=None):
......@@ -82,20 +93,6 @@ class MockCache(object):
"""
return self.map.get(key, default)
def set_many(self, map_):
"""
For each dictionary entry in the given map, updates the cache
with that entry.
"""
for key, val in map_.iteritems():
self.set(key, val)
def get_many(self, keys):
"""
Returns a dictionary of entries for each key found in the cache.
"""
return {key: self.map[key] for key in keys if key in self.map}
def delete(self, key):
"""
Deletes the given key from the cache.
......@@ -141,6 +138,18 @@ class MockTransformer(BlockStructureTransformer):
pass
@contextmanager
def mock_registered_transformers(transformers):
"""
Context manager for mocking the transformer registry to return the given transformers.
"""
with patch(
'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
) as mock_available_transforms:
mock_available_transforms.return_value = {transformer for transformer in transformers}
yield
class ChildrenMapTestMixin(object):
"""
A Test Mixin with utility methods for testing with block structures
......@@ -172,7 +181,7 @@ class ChildrenMapTestMixin(object):
# 5 6
DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []]
def create_block_structure(self, block_structure_cls, children_map):
def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData):
"""
Factory method for creating and returning a block structure
for the given children_map.
......@@ -183,7 +192,7 @@ class ChildrenMapTestMixin(object):
# _add_relation
for parent, children in enumerate(children_map):
for child in children:
block_structure._add_relation(parent, child)
block_structure._add_relation(parent, child) # pylint: disable=protected-access
return block_structure
def get_parents_map(self, children_map):
......
openedx/core/lib/block_cache/tests/test_block_structure.py → openedx/core/lib/block_structure/tests/test_block_structure.py
......@@ -10,9 +10,9 @@ from unittest import TestCase
from openedx.core.lib.graph_traversals import traverse_post_order
from ..block_structure import BlockStructure, BlockStructureModulestoreData, BlockStructureBlockData
from ..block_structure import BlockStructure, BlockStructureModulestoreData
from ..exceptions import TransformerException
from .test_utils import MockXBlock, MockTransformer, ChildrenMapTestMixin
from .helpers import MockXBlock, MockTransformer, ChildrenMapTestMixin
@ddt.ddt
......@@ -27,7 +27,7 @@ class TestBlockStructure(TestCase, ChildrenMapTestMixin):
ChildrenMapTestMixin.DAG_CHILDREN_MAP,
)
def test_relations(self, children_map):
block_structure = self.create_block_structure(BlockStructure, children_map)
block_structure = self.create_block_structure(children_map, BlockStructure)
# get_children
for parent, children in enumerate(children_map):
......@@ -167,7 +167,7 @@ class TestBlockStructureData(TestCase, ChildrenMapTestMixin):
return
### create structure
block_structure = self.create_block_structure(BlockStructureBlockData, children_map)
block_structure = self.create_block_structure(children_map)
parents_map = self.get_parents_map(children_map)
### verify blocks pre-exist
......@@ -213,6 +213,6 @@ class TestBlockStructureData(TestCase, ChildrenMapTestMixin):
self.assert_block_structure(block_structure, pruned_children_map, missing_blocks)
def test_remove_block_if(self):
block_structure = self.create_block_structure(BlockStructureBlockData, ChildrenMapTestMixin.LINEAR_CHILDREN_MAP)
block_structure = self.create_block_structure(ChildrenMapTestMixin.LINEAR_CHILDREN_MAP)
block_structure.remove_block_if(lambda block: block == 2)
self.assert_block_structure(block_structure, [[1], [], [], []], missing_blocks=[2])
openedx/core/lib/block_structure/tests/test_cache.py 0 → 100644
"""
Tests for block_structure/cache.py
"""
from unittest import TestCase
from ..cache import BlockStructureCache
from .helpers import ChildrenMapTestMixin, MockCache, MockTransformer
class TestBlockStructureCache(ChildrenMapTestMixin, TestCase):
"""
Tests for BlockStructureFactory
"""
def setUp(self):
super(TestBlockStructureCache, self).setUp()
self.children_map = self.SIMPLE_CHILDREN_MAP
self.block_structure = self.create_block_structure(self.children_map)
self.cache = BlockStructureCache(MockCache())
def add_transformers(self):
"""
Add each registered transformer to the block structure.
Mimic collection by setting test transformer block data.
"""
for transformer in [MockTransformer]:
self.block_structure._add_transformer(transformer) # pylint: disable=protected-access
self.block_structure.set_transformer_block_field(
usage_key=0, transformer=transformer, key='test', value='{} val'.format(transformer.name())
)
def test_add(self):
self.add_transformers()
self.cache.add(self.block_structure)
cached_value = self.cache.get(self.block_structure.root_block_usage_key)
self.assertIsNotNone(cached_value)
self.assert_block_structure(cached_value, self.children_map)
def test_get_none(self):
self.assertIsNone(
self.cache.get(self.block_structure.root_block_usage_key)
)
def test_delete(self):
self.add_transformers()
self.cache.add(self.block_structure)
self.cache.delete(self.block_structure.root_block_usage_key)
self.assertIsNone(
self.cache.get(self.block_structure.root_block_usage_key)
)
openedx/core/lib/block_structure/tests/test_factory.py 0 → 100644
"""
Tests for block_structure_factory.py
"""
from unittest import TestCase
from xmodule.modulestore.exceptions import ItemNotFoundError
from ..cache import BlockStructureCache
from ..factory import BlockStructureFactory
from .helpers import (
MockCache, MockModulestoreFactory, ChildrenMapTestMixin
)
class TestBlockStructureFactory(TestCase, ChildrenMapTestMixin):
"""
Tests for BlockStructureFactory
"""
def setUp(self):
super(TestBlockStructureFactory, self).setUp()
self.children_map = self.SIMPLE_CHILDREN_MAP
self.modulestore = MockModulestoreFactory.create(self.children_map)
def test_from_modulestore(self):
block_structure = BlockStructureFactory.create_from_modulestore(
root_block_usage_key=0, modulestore=self.modulestore
)
self.assert_block_structure(block_structure, self.children_map)
def test_from_modulestore_fail(self):
with self.assertRaises(ItemNotFoundError):
BlockStructureFactory.create_from_modulestore(
root_block_usage_key=len(self.children_map) + 1,
modulestore=self.modulestore,
)
def test_from_cache(self):
cache = BlockStructureCache(MockCache())
block_structure = self.create_block_structure(self.children_map)
cache.add(block_structure)
from_cache_block_structure = BlockStructureFactory.create_from_cache(
block_structure.root_block_usage_key,
cache,
)
self.assertIsNotNone(from_cache_block_structure)
self.assert_block_structure(from_cache_block_structure, self.children_map)
def test_from_cache_none(self):
cache = BlockStructureCache(MockCache())
self.assertIsNone(
BlockStructureFactory.create_from_cache(
root_block_usage_key=0,
block_structure_cache=cache,
)
)
openedx/core/lib/block_structure/tests/test_manager.py 0 → 100644
"""
Tests for manager.py
"""
from unittest import TestCase
from ..manager import BlockStructureManager
from ..transformers import BlockStructureTransformers
from .helpers import (
MockModulestoreFactory, MockCache, MockTransformer, ChildrenMapTestMixin, mock_registered_transformers
)
class TestTransformer1(MockTransformer):
"""
Test Transformer class with basic functionality to verify collected and
transformed data.
"""
collect_data_key = 't1.collect'
transform_data_key = 't1.transform'
collect_call_count = 0
@classmethod
def collect(cls, block_structure):
"""
Collects block data for the block structure.
"""
cls._set_block_values(block_structure, cls.collect_data_key)
cls.collect_call_count += 1
def transform(self, usage_info, block_structure):
"""
Transforms the block structure.
"""
self._set_block_values(block_structure, self.transform_data_key)
@classmethod
def assert_collected(cls, block_structure):
"""
Asserts data was collected for the block structure.
"""
cls._assert_block_values(block_structure, cls.collect_data_key)
@classmethod
def assert_transformed(cls, block_structure):
"""
Asserts the block structure was transformed.
"""
cls._assert_block_values(block_structure, cls.transform_data_key)
@classmethod
def _set_block_values(cls, block_structure, data_key):
"""
Sets a value for each block in the given structure, using the given
data key.
"""
for block_key in block_structure.topological_traversal():
block_structure.set_transformer_block_field(
block_key, cls, data_key, cls._create_block_value(block_key, data_key)
)
@classmethod
def _assert_block_values(cls, block_structure, data_key):
"""
Verifies the value for each block in the given structure, for the given
data key.
"""
for block_key in block_structure.topological_traversal():
assert (
block_structure.get_transformer_block_field(
block_key,
cls,
data_key,
) == cls._create_block_value(block_key, data_key)
)
@classmethod
def _create_block_value(cls, block_key, data_key):
"""
Returns a unique deterministic value for the given block key
and data key.
"""
return data_key + 't1.val1.' + unicode(block_key)
class TestBlockStructureManager(TestCase, ChildrenMapTestMixin):
"""
Test class for BlockStructureManager.
"""
def setUp(self):
super(TestBlockStructureManager, self).setUp()
TestTransformer1.collect_call_count = 0
self.registered_transformers = [TestTransformer1()]
with mock_registered_transformers(self.registered_transformers):
self.transformers = BlockStructureTransformers(self.registered_transformers)
self.children_map = self.SIMPLE_CHILDREN_MAP
self.modulestore = MockModulestoreFactory.create(self.children_map)
self.cache = MockCache()
self.bs_manager = BlockStructureManager(
root_block_usage_key=0,
modulestore=self.modulestore,
cache=self.cache,
)
def collect_and_verify(self, expect_modulestore_called, expect_cache_updated):
"""
Calls the manager's get_collected method and verifies its result
and behavior.
"""
self.modulestore.get_items_call_count = 0
self.cache.set_call_count = 0
with mock_registered_transformers(self.registered_transformers):
block_structure = self.bs_manager.get_collected()
self.assert_block_structure(block_structure, self.children_map)
TestTransformer1.assert_collected(block_structure)
if expect_modulestore_called:
self.assertGreater(self.modulestore.get_items_call_count, 0)
else:
self.assertEquals(self.modulestore.get_items_call_count, 0)
self.assertEquals(self.cache.set_call_count, 1 if expect_cache_updated else 0)
def test_get_transformed(self):
with mock_registered_transformers(self.registered_transformers):
block_structure = self.bs_manager.get_transformed(self.transformers)
self.assert_block_structure(block_structure, self.children_map)
TestTransformer1.assert_collected(block_structure)
TestTransformer1.assert_transformed(block_structure)
def test_get_collected_cached(self):
self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True)
self.collect_and_verify(expect_modulestore_called=False, expect_cache_updated=False)
self.assertEquals(TestTransformer1.collect_call_count, 1)
def test_get_collected_outdated_data(self):
self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True)
TestTransformer1.VERSION += 1
self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True)
self.assertEquals(TestTransformer1.collect_call_count, 2)
def test_clear(self):
self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True)
self.bs_manager.clear()
self.collect_and_verify(expect_modulestore_called=True, expect_cache_updated=True)
self.assertEquals(TestTransformer1.collect_call_count, 2)
openedx/core/lib/block_cache/tests/test_transformer_registry.py → openedx/core/lib/block_structure/tests/test_transformer_registry.py
......@@ -3,11 +3,10 @@ Tests for transformer_registry.py
"""
import ddt
from mock import patch
from unittest import TestCase
from ..transformer_registry import TransformerRegistry
from .test_utils import MockTransformer
from .helpers import MockTransformer, mock_registered_transformers
class TestTransformer1(MockTransformer):
......@@ -55,14 +54,8 @@ class TransformerRegistryTestCase(TestCase):
@ddt.unpack
def test_find_unregistered(self, transformers, expected_unregistered):
with (
patch('openedx.core.lib.block_cache.transformer_registry.TransformerRegistry.get_available_plugins')
) as mock_registry:
mock_registry.return_value = {
transformer.name(): transformer
for transformer in [TestTransformer1, TestTransformer2]
}
with mock_registered_transformers([TestTransformer1, TestTransformer2]):
self.assertSetEqual(
TransformerRegistry.find_unregistered(transformers), set(expected_unregistered)
TransformerRegistry.find_unregistered(transformers),
set(expected_unregistered),
)
openedx/core/lib/block_structure/tests/test_transformers.py 0 → 100644
"""
Tests for transformers.py
"""
from mock import MagicMock, patch
from unittest import TestCase
from ..block_structure import BlockStructureModulestoreData
from ..exceptions import TransformerException
from ..transformers import BlockStructureTransformers
from .helpers import (
ChildrenMapTestMixin, MockTransformer, mock_registered_transformers
)
class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
"""
Test class for testing BlockStructureTransformers
"""
class UnregisteredTransformer(MockTransformer):
"""
Mock transformer that is not registered.
"""
pass
def setUp(self):
super(TestBlockStructureTransformers, self).setUp()
self.transformers = BlockStructureTransformers(usage_info=MagicMock())
self.registered_transformers = [MockTransformer]
def add_mock_transformer(self):
"""
Adds the registered transformers to the self.transformers collection.
"""
with mock_registered_transformers(self.registered_transformers):
self.transformers += self.registered_transformers
def test_add_registered(self):
self.add_mock_transformer()
self.assertIn(MockTransformer, self.transformers._transformers) # pylint: disable=protected-access
def test_add_unregistered(self):
with self.assertRaises(TransformerException):
self.transformers += [self.UnregisteredTransformer]
self.assertEquals(self.transformers._transformers, []) # pylint: disable=protected-access
def test_collect(self):
with mock_registered_transformers(self.registered_transformers):
with patch(
'openedx.core.lib.block_structure.tests.helpers.MockTransformer.collect'
) as mock_collect_call:
self.transformers.collect(block_structure=MagicMock())
self.assertTrue(mock_collect_call.called)
def test_transform(self):
self.add_mock_transformer()
with patch(
'openedx.core.lib.block_structure.tests.helpers.MockTransformer.transform'
) as mock_transform_call:
self.transformers.transform(block_structure=MagicMock())
self.assertTrue(mock_transform_call.called)
def test_is_collected_outdated(self):
block_structure = self.create_block_structure(
self.SIMPLE_CHILDREN_MAP,
BlockStructureModulestoreData
)
with mock_registered_transformers(self.registered_transformers):
self.assertTrue(self.transformers.is_collected_outdated(block_structure))
self.transformers.collect(block_structure)
self.assertFalse(self.transformers.is_collected_outdated(block_structure))
openedx/core/lib/block_cache/transformer.py → openedx/core/lib/block_structure/transformer.py
......@@ -14,7 +14,7 @@ class BlockStructureTransformer(object):
# attribute. While the value for the base class is set to 0,
# the value for each concrete transformer should be 1 or higher.
#
# A transformer's version attribute is used by the block_cache
# A transformer's version attribute is used by the block_structure
# framework in order to determine whether any collected data for a
# transformer is outdated. When a transformer's data is collected
# and cached, it's version number at the time of collection is
......@@ -115,7 +115,7 @@ class BlockStructureTransformer(object):
Arguments:
usage_info (any negotiated type) - A usage-specific object
that is passed to the block_cache and forwarded to all
that is passed to the block_structure and forwarded to all
requested Transformers in order to apply a
usage-specific transform. For example, an instance of
usage_info would contain a user object for which the
......
openedx/core/lib/block_cache/transformer_registry.py → openedx/core/lib/block_structure/transformer_registry.py
......@@ -14,6 +14,7 @@ class TransformerRegistry(PluginManager):
`BlockStructureTransformer`.
"""
NAMESPACE = 'openedx.block_structure_transformer'
USE_PLUGIN_MANAGER = True
@classmethod
def get_registered_transformers(cls):
......@@ -24,7 +25,10 @@ class TransformerRegistry(PluginManager):
{BlockStructureTransformer} - All transformers that are
registered with the platform's PluginManager.
"""
return set(cls.get_available_plugins().itervalues())
if cls.USE_PLUGIN_MANAGER:
return set(cls.get_available_plugins().itervalues())
else:
return set()
@classmethod
def find_unregistered(cls, transformers):
......@@ -38,7 +42,7 @@ class TransformerRegistry(PluginManager):
transformers to check in the registry.
Returns:
[string] - The names of a subset of the given
set([string]) - Set of names of a subset of the given
transformers that weren't found in the registry.
"""
registered_transformer_names = set(reg_trans.name() for reg_trans in cls.get_registered_transformers())
......
openedx/core/lib/block_structure/transformers.py 0 → 100644
"""
Module for a collection of BlockStructureTransformers.
"""
from logging import getLogger
from .exceptions import TransformerException
from .transformer_registry import TransformerRegistry
logger = getLogger(__name__) # pylint: disable=C0103
class BlockStructureTransformers(object):
"""
The BlockStructureTransformers class encapsulates an ordered list of block
structure transformers. It uses the Transformer Registry to verify the
the registration status of added transformers and to collect their data.
It provides aggregate functionality for collection and ordered
transformation of the transformers.
Clients are expected to access the list of transformers through the
class' interface rather than directly.
"""
def __init__(self, transformers=None, usage_info=None):
"""
Arguments:
transformers ([BlockStructureTransformer]) - List of transformers
to add to the collection.
usage_info (any negotiated type) - A usage-specific object
that is passed to the block_structure and forwarded to all
requested Transformers in order to apply a
usage-specific transform. For example, an instance of
usage_info would contain a user object for which the
transform should be applied.
Raises:
TransformerException - if any transformer is not registered in the
Transformer Registry.
"""
self.usage_info = usage_info
self._transformers = []
if transformers:
self.__iadd__(transformers)
def __iadd__(self, transformers):
"""
Adds the given transformers to the collection.
Args:
transformers ([BlockStructureTransformer]) - List of transformers
to add to the collection.
Raises:
TransformerException - if any transformer is not registered in the
Transformer Registry.
"""
unregistered_transformers = TransformerRegistry.find_unregistered(transformers)
if unregistered_transformers:
raise TransformerException(
"The following requested transformers are not registered: {}".format(unregistered_transformers)
)
self._transformers.extend(transformers)
return self
@classmethod
def collect(cls, block_structure):
"""
Collects data for each registered transformer.
"""
for transformer in TransformerRegistry.get_registered_transformers():
block_structure._add_transformer(transformer) # pylint: disable=protected-access
transformer.collect(block_structure)
# Collect all fields that were requested by the transformers.
block_structure._collect_requested_xblock_fields() # pylint: disable=protected-access
def transform(self, block_structure):
"""
The given block structure is transformed by each transformer in the
collection, in the order that the transformers were added.
"""
for transformer in self._transformers:
transformer.transform(self.usage_info, block_structure)
# Prune the block structure to remove any unreachable blocks.
block_structure._prune_unreachable() # pylint: disable=protected-access
@classmethod
def is_collected_outdated(cls, block_structure):
"""
Returns whether the collected data in the block structure is outdated.
"""
outdated_transformers = []
for transformer in TransformerRegistry.get_registered_transformers():
version_in_block_structure = block_structure._get_transformer_data_version(transformer) # pylint: disable=protected-access
if transformer.VERSION != version_in_block_structure:
outdated_transformers.append(transformer)
if outdated_transformers:
logger.debug(
"Collected Block Structure data for the following transformers is outdated: '%s'.",
[(transformer.name(), transformer.VERSION) for transformer in outdated_transformers],
)
return bool(outdated_transformers)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment