Merge pull request #12567 from edx/tnl/grading

Grading uses Block Transformers

cms/djangoapps/contentstore/tests/test_orphan.py

@@ -101,7 +101,7 @@ class TestOrphan(TestOrphanBase):
 
     @ddt.data(
         (ModuleStoreEnum.Type.split, 9, 6),
-        (ModuleStoreEnum.Type.mongo, 30, 13),
+        (ModuleStoreEnum.Type.mongo, 34, 13),
     )
     @ddt.unpack
     def test_delete_orphans(self, default_store, max_mongo_calls, min_mongo_calls):

common/lib/xmodule/xmodule/block_metadata_utils.py

+"""
+Simple utility functions that operate on block metadata.
+
+This is a place to put simple functions that operate on block metadata. It
+allows us to share code between the XModuleMixin and CourseOverview and
+BlockStructure.
+"""
+
+
+def url_name_for_block(block):
+    """
+    Given a block, returns the block's URL name.
+
+    Arguments:
+        block (XModuleMixin|CourseOverview|BlockStructureBlockData):
+            Block that is being accessed
+    """
+    return block.location.name
+
+
+def display_name_with_default(block):
+    """
+    Calculates the display name for a block.
+
+    Default to the display_name if it isn't None, else fall back to creating
+    a name based on the URL.
+
+    Unlike the rest of this module's functions, this function takes an entire
+    course descriptor/overview as a parameter. This is because a few test cases
+    (specifically, {Text|Image|Video}AnnotationModuleTestCase.test_student_view)
+    create scenarios where course.display_name is not None but course.location
+    is None, which causes calling course.url_name to fail. So, although we'd
+    like to just pass course.display_name and course.url_name as arguments to
+    this function, we can't do so without breaking those tests.
+
+    Note: This method no longer escapes as it once did, so the caller must
+    ensure it is properly escaped where necessary.
+
+    Arguments:
+        block (XModuleMixin|CourseOverview|BlockStructureBlockData):
+            Block that is being accessed
+    """
+    return (
+        block.display_name if block.display_name is not None
+        else url_name_for_block(block).replace('_', ' ')
+    )
+
+
+def display_name_with_default_escaped(block):
+    """
+    DEPRECATED: use display_name_with_default
+
+    Calculates the display name for a block with some HTML escaping.
+    This follows the same logic as display_name_with_default, with
+    the addition of the escaping.
+
+    Here is an example of how to move away from this method in Mako html:
+        Before:
+        <span class="course-name">${course.display_name_with_default_escaped}</span>
+
+        After:
+        <span class="course-name">${course.display_name_with_default | h}</span>
+    If the context is Javascript in Mako, you'll need to follow other best practices.
+
+    Note: Switch to display_name_with_default, and ensure the caller
+    properly escapes where necessary.
+
+    Note: This newly introduced method should not be used.  It was only
+    introduced to enable a quick search/replace and the ability to slowly
+    migrate and test switching to display_name_with_default, which is no
+    longer escaped.
+
+    Arguments:
+        block (XModuleMixin|CourseOverview|BlockStructureBlockData):
+            Block that is being accessed
+    """
+    # This escaping is incomplete.  However, rather than switching this to use
+    # markupsafe.escape() and fixing issues, better to put that energy toward
+    # migrating away from this method altogether.
+    return display_name_with_default(block).replace('<', '&lt;').replace('>', '&gt;')

common/lib/xmodule/xmodule/course_metadata_utils.py

@@ -32,78 +32,6 @@ def clean_course_key(course_key, padding_char):
     )
 
 
-def url_name_for_course_location(location):
-    """
-    Given a course's usage locator, returns the course's URL name.
-
-    Arguments:
-        location (BlockUsageLocator): The course's usage locator.
-    """
-    return location.name
-
-
-def display_name_with_default(course):
-    """
-    Calculates the display name for a course.
-
-    Default to the display_name if it isn't None, else fall back to creating
-    a name based on the URL.
-
-    Unlike the rest of this module's functions, this function takes an entire
-    course descriptor/overview as a parameter. This is because a few test cases
-    (specifically, {Text|Image|Video}AnnotationModuleTestCase.test_student_view)
-    create scenarios where course.display_name is not None but course.location
-    is None, which causes calling course.url_name to fail. So, although we'd
-    like to just pass course.display_name and course.url_name as arguments to
-    this function, we can't do so without breaking those tests.
-
-    Note: This method no longer escapes as it once did, so the caller must
-    ensure it is properly escaped where necessary.
-
-    Arguments:
-        course (CourseDescriptor|CourseOverview): descriptor or overview of
-            said course.
-    """
-    return (
-        course.display_name if course.display_name is not None
-        else course.url_name.replace('_', ' ')
-    )
-
-
-def display_name_with_default_escaped(course):
-    """
-    DEPRECATED: use display_name_with_default
-
-    Calculates the display name for a course with some HTML escaping.
-    This follows the same logic as display_name_with_default, with
-    the addition of the escaping.
-
-    Here is an example of how to move away from this method in Mako html:
-        Before:
-        <span class="course-name">${course.display_name_with_default_escaped}</span>
-
-        After:
-        <span class="course-name">${course.display_name_with_default | h}</span>
-    If the context is Javascript in Mako, you'll need to follow other best practices.
-
-    Note: Switch to display_name_with_default, and ensure the caller
-    properly escapes where necessary.
-
-    Note: This newly introduced method should not be used.  It was only
-    introduced to enable a quick search/replace and the ability to slowly
-    migrate and test switching to display_name_with_default, which is no
-    longer escaped.
-
-    Arguments:
-        course (CourseDescriptor|CourseOverview): descriptor or overview of
-            said course.
-    """
-    # This escaping is incomplete.  However, rather than switching this to use
-    # markupsafe.escape() and fixing issues, better to put that energy toward
-    # migrating away from this method altogether.
-    return course.display_name_with_default.replace('<', '&lt;').replace('>', '&gt;')
-
-
 def number_for_course_location(location):
     """
     Given a course's block usage locator, returns the course's number.

common/lib/xmodule/xmodule/course_module.py

@@ -11,12 +11,10 @@ from django.utils.timezone import UTC
 from lazy import lazy
 from lxml import etree
 from path import Path as path
-from xblock.core import XBlock
 from xblock.fields import Scope, List, String, Dict, Boolean, Integer, Float
 
 from xmodule import course_metadata_utils
 from xmodule.course_metadata_utils import DEFAULT_START_DATE
-from xmodule.exceptions import UndefinedContext
 from xmodule.graders import grader_from_conf
 from xmodule.mixin import LicenseMixin
 from xmodule.seq_module import SequenceDescriptor, SequenceModule
@@ -1183,83 +1181,6 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin):
         """
         return course_metadata_utils.sorting_score(self.start, self.advertised_start, self.announcement)
 
-    @lazy
-    def grading_context(self):
-        """
-        This returns a dictionary with keys necessary for quickly grading
-        a student. They are used by grades.grade()
-
-        The grading context has two keys:
-        graded_sections - This contains the sections that are graded, as
-            well as all possible children modules that can affect the
-            grading. This allows some sections to be skipped if the student
-            hasn't seen any part of it.
-
-            The format is a dictionary keyed by section-type. The values are
-            arrays of dictionaries containing
-                "section_descriptor" : The section descriptor
-                "xmoduledescriptors" : An array of xmoduledescriptors that
-                    could possibly be in the section, for any student
-
-        all_descriptors - This contains a list of all xmodules that can
-            effect grading a student. This is used to efficiently fetch
-            all the xmodule state for a FieldDataCache without walking
-            the descriptor tree again.
-
-
-        """
-        # If this descriptor has been bound to a student, return the corresponding
-        # XModule. If not, just use the descriptor itself
-        try:
-            module = getattr(self, '_xmodule', None)
-            if not module:
-                module = self
-        except UndefinedContext:
-            module = self
-
-        def possibly_scored(usage_key):
-            """Can this XBlock type can have a score or children?"""
-            return usage_key.block_type in self.block_types_affecting_grading
-
-        all_descriptors = []
-        graded_sections = {}
-
-        def yield_descriptor_descendents(module_descriptor):
-            for child in module_descriptor.get_children(usage_key_filter=possibly_scored):
-                yield child
-                for module_descriptor in yield_descriptor_descendents(child):
-                    yield module_descriptor
-
-        for chapter in self.get_children():
-            for section in chapter.get_children():
-                if section.graded:
-                    xmoduledescriptors = list(yield_descriptor_descendents(section))
-                    xmoduledescriptors.append(section)
-
-                    # The xmoduledescriptors included here are only the ones that have scores.
-                    section_description = {
-                        'section_descriptor': section,
-                        'xmoduledescriptors': [child for child in xmoduledescriptors if child.has_score]
-                    }
-
-                    section_format = section.format if section.format is not None else ''
-                    graded_sections[section_format] = graded_sections.get(section_format, []) + [section_description]
-
-                    all_descriptors.extend(xmoduledescriptors)
-                    all_descriptors.append(section)
-
-        return {'graded_sections': graded_sections,
-                'all_descriptors': all_descriptors, }
-
-    @lazy
-    def block_types_affecting_grading(self):
-        """Return all block types that could impact grading (i.e. scored, or having children)."""
-        return frozenset(
-            cat for (cat, xblock_class) in XBlock.load_classes() if (
-                getattr(xblock_class, 'has_score', False) or getattr(xblock_class, 'has_children', False)
-            )
-        )
-
     @staticmethod
     def make_id(org, course, url_name):
         return '/'.join([org, course, url_name])

common/lib/xmodule/xmodule/graders.py

@@ -173,7 +173,7 @@ class WeightedSubsectionsGrader(CourseGrader):
     All items in section_breakdown for each subgrader will be combined. A grade_breakdown will be
     composed using the score from each grader.
 
-    Note that the sum of the weights is not take into consideration. If the weights add up to
+    Note that the sum of the weights is not taken into consideration. If the weights add up to
     a value > 1, the student may end up with a percent > 100%. This allows for sections that
     are extra credit.
     """

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -2950,10 +2950,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
         output_fields = dict(jsonfields)
         for field_name, value in output_fields.iteritems():
             if value:
-                field = xblock_class.fields.get(field_name)
-                if field is None:
+                try:
+                    field = xblock_class.fields.get(field_name)
+                except AttributeError:
                     continue
-                elif isinstance(field, Reference):
+                if isinstance(field, Reference):
                     output_fields[field_name] = robust_usage_key(value)
                 elif isinstance(field, ReferenceList):
                     output_fields[field_name] = [robust_usage_key(ele) for ele in value]

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -218,7 +218,7 @@ class ModuleStoreIsolationMixin(CacheIsolationMixin):
 
     MODULESTORE = functools.partial(mixed_store_config, mkdtemp_clean(), {})
     CONTENTSTORE = functools.partial(contentstore_config)
-    ENABLED_CACHES = ['mongo_metadata_inheritance', 'loc_cache']
+    ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
     __settings_overrides = []
     __old_modulestores = []
     __old_contentstores = []

common/lib/xmodule/xmodule/tests/test_course_metadata_utils.py

@@ -7,11 +7,13 @@ from unittest import TestCase
 
 from django.utils.timezone import UTC
 
-from xmodule.course_metadata_utils import (
-    clean_course_key,
-    url_name_for_course_location,
+from xmodule.block_metadata_utils import (
+    url_name_for_block,
     display_name_with_default,
     display_name_with_default_escaped,
+)
+from xmodule.course_metadata_utils import (
+    clean_course_key,
     number_for_course_location,
     has_course_started,
     has_course_ended,
@@ -130,9 +132,9 @@ class CourseMetadataUtilsTestCase(TestCase):
                     "course_MNXXK4TTMUWXMMJ2KVXGS5TFOJZWS5DZLAVUGUZNGIYDGK2ZGIYDSNQ~"
                 ),
             ]),
-            FunctionTest(url_name_for_course_location, [
-                TestScenario((self.demo_course.location,), self.demo_course.location.name),
-                TestScenario((self.html_course.location,), self.html_course.location.name),
+            FunctionTest(url_name_for_block, [
+                TestScenario((self.demo_course,), self.demo_course.location.name),
+                TestScenario((self.html_course,), self.html_course.location.name),
             ]),
             FunctionTest(display_name_with_default_escaped, [
                 # Test course with no display name.

common/lib/xmodule/xmodule/x_module.py

@@ -27,7 +27,7 @@ from xblock.fields import (
 
 from xblock.fragment import Fragment
 from xblock.runtime import Runtime, IdReader, IdGenerator
-from xmodule import course_metadata_utils
+from xmodule import block_metadata_utils
 from xmodule.fields import RelativeTime
 from xmodule.errortracker import exc_info_to_str
 from xmodule.modulestore.exceptions import ItemNotFoundError
@@ -340,7 +340,7 @@ class XModuleMixin(XModuleFields, XBlock):
 
     @property
     def url_name(self):
-        return course_metadata_utils.url_name_for_course_location(self.location)
+        return block_metadata_utils.url_name_for_block(self)
 
     @property
     def display_name_with_default(self):
@@ -348,7 +348,7 @@ class XModuleMixin(XModuleFields, XBlock):
         Return a display name for the module: use display_name if defined in
         metadata, otherwise convert the url name.
         """
-        return course_metadata_utils.display_name_with_default(self)
+        return block_metadata_utils.display_name_with_default(self)
 
     @property
     def display_name_with_default_escaped(self):
@@ -363,7 +363,7 @@ class XModuleMixin(XModuleFields, XBlock):
         migrate and test switching to display_name_with_default, which is no
         longer escaped.
         """
-        return course_metadata_utils.display_name_with_default_escaped(self)
+        return block_metadata_utils.display_name_with_default_escaped(self)
 
     @property
     def tooltip_title(self):

lms/djangoapps/ccx/overrides.py

@@ -50,12 +50,11 @@ class CustomCoursesForEdxOverrideProvider(FieldOverrideProvider):
         return default
 
     @classmethod
-    def enabled_for(cls, course):
-        """CCX field overrides are enabled per-course
-
-        protect against missing attributes
+    def enabled_for(cls, block):
+        """
+        CCX field overrides are enabled for CCX blocks.
         """
-        return getattr(course, 'enable_ccx', False)
+        return getattr(block.location, 'ccx', None) or getattr(block, 'enable_ccx', False)
 
 
 def get_current_ccx(course_key):
@@ -86,12 +85,9 @@ def get_override_for_ccx(ccx, block, name, default=None):
     """
     overrides = _get_overrides_for_ccx(ccx)
 
-    if isinstance(block.location, CCXBlockUsageLocator):
-        non_ccx_key = block.location.to_block_locator()
-    else:
-        non_ccx_key = block.location
+    clean_ccx_key = _clean_ccx_key(block.location)
 
-    block_overrides = overrides.get(non_ccx_key, {})
+    block_overrides = overrides.get(clean_ccx_key, {})
     if name in block_overrides:
         try:
             return block.fields[name].from_json(block_overrides[name])
@@ -101,6 +97,21 @@ def get_override_for_ccx(ccx, block, name, default=None):
         return default
 
 
+def _clean_ccx_key(block_location):
+    """
+    Converts the given BlockUsageKey from a CCX key to the
+    corresponding key for its parent course, while handling the case
+    where no conversion is needed.  Also strips any version and
+    branch information from the key.
+    Returns the cleaned key.
+    """
+    if isinstance(block_location, CCXBlockUsageLocator):
+        clean_key = block_location.to_block_locator()
+    else:
+        clean_key = block_location
+    return clean_key.version_agnostic().for_branch(None)
+
+
 def _get_overrides_for_ccx(ccx):
     """
     Returns a dictionary mapping field name to overriden value for any
@@ -136,6 +147,7 @@ def override_field_for_ccx(ccx, block, name, value):
     value_json = field.to_json(value)
     serialized_value = json.dumps(value_json)
     override_has_changes = False
+    clean_ccx_key = _clean_ccx_key(block.location)
 
     override = get_override_for_ccx(ccx, block, name + "_instance")
     if override:
@@ -149,7 +161,7 @@ def override_field_for_ccx(ccx, block, name, value):
             defaults={'value': serialized_value},
         )
         if created:
-            _get_overrides_for_ccx(ccx).setdefault(block.location, {})[name + "_id"] = override.id
+            _get_overrides_for_ccx(ccx).setdefault(clean_ccx_key, {})[name + "_id"] = override.id
         else:
             override_has_changes = serialized_value != override.value
 
@@ -157,8 +169,8 @@ def override_field_for_ccx(ccx, block, name, value):
         override.value = serialized_value
         override.save()
 
-    _get_overrides_for_ccx(ccx).setdefault(block.location, {})[name] = value_json
-    _get_overrides_for_ccx(ccx).setdefault(block.location, {})[name + "_instance"] = override
+    _get_overrides_for_ccx(ccx).setdefault(clean_ccx_key, {})[name] = value_json
+    _get_overrides_for_ccx(ccx).setdefault(clean_ccx_key, {})[name + "_instance"] = override
 
 
 def clear_override_for_ccx(ccx, block, name):
@@ -185,7 +197,8 @@ def clear_ccx_field_info_from_ccx_map(ccx, block, name):  # pylint: disable=inva
     Remove field information from ccx overrides mapping dictionary
     """
     try:
-        ccx_override_map = _get_overrides_for_ccx(ccx).setdefault(block.location, {})
+        clean_ccx_key = _clean_ccx_key(block.location)
+        ccx_override_map = _get_overrides_for_ccx(ccx).setdefault(clean_ccx_key, {})
         ccx_override_map.pop(name)
         ccx_override_map.pop(name + "_id")
         ccx_override_map.pop(name + "_instance")

lms/djangoapps/ccx/tests/test_field_override_performance.py

@@ -9,6 +9,7 @@ from nose.plugins.skip import SkipTest
 
 from courseware.views.views import progress
 from courseware.field_overrides import OverrideFieldData
+from courseware.testutils import FieldOverrideTestMixin
 from datetime import datetime
 from django.conf import settings
 from django.core.cache import caches
@@ -20,13 +21,13 @@ from request_cache.middleware import RequestCache
 from student.models import CourseEnrollment
 from student.tests.factories import UserFactory
 from xblock.core import XBlock
-from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, \
     TEST_DATA_SPLIT_MODULESTORE, TEST_DATA_MONGO_MODULESTORE
 from xmodule.modulestore.tests.factories import check_mongo_calls_range, CourseFactory, check_sum_of_calls
 from xmodule.modulestore.tests.utils import ProceduralCourseTestMixin
 from ccx_keys.locator import CCXLocator
 from lms.djangoapps.ccx.tests.factories import CcxFactory
+from openedx.core.djangoapps.content.block_structure.api import get_course_in_cache
 
 
 @attr('shard_3')
@@ -38,8 +39,7 @@ from lms.djangoapps.ccx.tests.factories import CcxFactory
     }
 )
 @ddt.ddt
-class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
-                                       ModuleStoreTestCase):
+class FieldOverridePerformanceTestCase(FieldOverrideTestMixin, ProceduralCourseTestMixin, ModuleStoreTestCase):
     """
     Base class for instrumenting SQL queries and Mongo reads for field override
     providers.
@@ -51,8 +51,6 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
     # TEST_DATA must be overridden by subclasses
     TEST_DATA = None
 
-    ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
-
     def setUp(self):
         """
         Create a test client, course, and user.
@@ -172,7 +170,7 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
                 caches[cache].clear()
 
             # Refill the metadata inheritance cache
-            modulestore().get_course(self.course.id, depth=None)
+            get_course_in_cache(self.course.id)
 
             # We clear the request cache to simulate a new request in the LMS.
             RequestCache.clear_request_cache()
@@ -190,7 +188,8 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
     @ddt.data(*itertools.product(('no_overrides', 'ccx'), range(1, 4), (True, False), (True, False)))
     @ddt.unpack
     @override_settings(
-        FIELD_OVERRIDE_PROVIDERS=(),
+        XBLOCK_FIELD_DATA_WRAPPERS=[],
+        MODULESTORE_FIELD_OVERRIDE_PROVIDERS=[],
     )
     def test_field_overrides(self, overrides, course_width, enable_ccx, view_as_ccx):
         """
@@ -209,7 +208,10 @@ class FieldOverridePerformanceTestCase(ProceduralCourseTestMixin,
         if self.MODULESTORE == TEST_DATA_MONGO_MODULESTORE and view_as_ccx:
             raise SkipTest("Can't use a MongoModulestore test as a CCX course")
 
-        with self.settings(FIELD_OVERRIDE_PROVIDERS=providers[overrides]):
+        with self.settings(
+            XBLOCK_FIELD_DATA_WRAPPERS=['lms.djangoapps.courseware.field_overrides:OverrideModulestoreFieldData.wrap'],
+            MODULESTORE_FIELD_OVERRIDE_PROVIDERS=providers[overrides],
+        ):
             default_queries, history_queries, reads, xblocks = self.TEST_DATA[
                 (overrides, course_width, enable_ccx, view_as_ccx)
             ]
@@ -232,24 +234,24 @@ class TestFieldOverrideMongoPerformance(FieldOverridePerformanceTestCase):
         #     # of mongo queries,
         #     # of xblocks
         # )
-        ('no_overrides', 1, True, False): (47, 1, 6, 13),
-        ('no_overrides', 2, True, False): (119, 16, 6, 84),
-        ('no_overrides', 3, True, False): (399, 81, 6, 335),
-        ('ccx', 1, True, False): (47, 1, 6, 13),
-        ('ccx', 2, True, False): (119, 16, 6, 84),
-        ('ccx', 3, True, False): (399, 81, 6, 335),
-        ('ccx', 1, True, True): (47, 1, 6, 13),
-        ('ccx', 2, True, True): (119, 16, 6, 84),
-        ('ccx', 3, True, True): (399, 81, 6, 335),
-        ('no_overrides', 1, False, False): (47, 1, 6, 13),
-        ('no_overrides', 2, False, False): (119, 16, 6, 84),
-        ('no_overrides', 3, False, False): (399, 81, 6, 335),
-        ('ccx', 1, False, False): (47, 1, 6, 13),
-        ('ccx', 2, False, False): (119, 16, 6, 84),
-        ('ccx', 3, False, False): (399, 81, 6, 335),
-        ('ccx', 1, False, True): (47, 1, 6, 13),
-        ('ccx', 2, False, True): (119, 16, 6, 84),
-        ('ccx', 3, False, True): (399, 81, 6, 335),
+        ('no_overrides', 1, True, False): (34, 0, 6, 1),
+        ('no_overrides', 2, True, False): (40, 0, 6, 1),
+        ('no_overrides', 3, True, False): (50, 0, 6, 1),
+        ('ccx', 1, True, False): (34, 0, 6, 1),
+        ('ccx', 2, True, False): (40, 0, 6, 1),
+        ('ccx', 3, True, False): (50, 0, 6, 1),
+        ('ccx', 1, True, True): (47, 0, 6, 1),
+        ('ccx', 2, True, True): (40, 0, 6, 1),
+        ('ccx', 3, True, True): (50, 0, 6, 1),
+        ('no_overrides', 1, False, False): (34, 0, 6, 1),
+        ('no_overrides', 2, False, False): (40, 0, 6, 1),
+        ('no_overrides', 3, False, False): (50, 0, 6, 1),
+        ('ccx', 1, False, False): (34, 0, 6, 1),
+        ('ccx', 2, False, False): (40, 0, 6, 1),
+        ('ccx', 3, False, False): (50, 0, 6, 1),
+        ('ccx', 1, False, True): (47, 0, 6, 1),
+        ('ccx', 2, False, True): (40, 0, 6, 1),
+        ('ccx', 3, False, True): (50, 0, 6, 1),
     }
 
 
@@ -261,22 +263,22 @@ class TestFieldOverrideSplitPerformance(FieldOverridePerformanceTestCase):
     __test__ = True
 
     TEST_DATA = {
-        ('no_overrides', 1, True, False): (47, 1, 4, 9),
-        ('no_overrides', 2, True, False): (119, 16, 19, 54),
-        ('no_overrides', 3, True, False): (399, 81, 84, 215),
-        ('ccx', 1, True, False): (47, 1, 4, 9),
-        ('ccx', 2, True, False): (119, 16, 19, 54),
-        ('ccx', 3, True, False): (399, 81, 84, 215),
-        ('ccx', 1, True, True): (49, 1, 4, 13),
-        ('ccx', 2, True, True): (121, 16, 19, 84),
-        ('ccx', 3, True, True): (401, 81, 84, 335),
-        ('no_overrides', 1, False, False): (47, 1, 4, 9),
-        ('no_overrides', 2, False, False): (119, 16, 19, 54),
-        ('no_overrides', 3, False, False): (399, 81, 84, 215),
-        ('ccx', 1, False, False): (47, 1, 4, 9),
-        ('ccx', 2, False, False): (119, 16, 19, 54),
-        ('ccx', 3, False, False): (399, 81, 84, 215),
-        ('ccx', 1, False, True): (47, 1, 4, 9),
-        ('ccx', 2, False, True): (119, 16, 19, 54),
-        ('ccx', 3, False, True): (399, 81, 84, 215),
+        ('no_overrides', 1, True, False): (34, 0, 4, 1),
+        ('no_overrides', 2, True, False): (40, 0, 19, 1),
+        ('no_overrides', 3, True, False): (50, 0, 84, 1),
+        ('ccx', 1, True, False): (34, 0, 4, 1),
+        ('ccx', 2, True, False): (40, 0, 19, 1),
+        ('ccx', 3, True, False): (50, 0, 84, 1),
+        ('ccx', 1, True, True): (35, 0, 5, 6),
+        ('ccx', 2, True, True): (41, 0, 20, 47),
+        ('ccx', 3, True, True): (51, 0, 85, 202),
+        ('no_overrides', 1, False, False): (34, 0, 4, 1),
+        ('no_overrides', 2, False, False): (40, 0, 19, 1),
+        ('no_overrides', 3, False, False): (50, 0, 84, 1),
+        ('ccx', 1, False, False): (34, 0, 4, 1),
+        ('ccx', 2, False, False): (40, 0, 19, 1),
+        ('ccx', 3, False, False): (50, 0, 84, 1),
+        ('ccx', 1, False, True): (46, 0, 4, 1),
+        ('ccx', 2, False, True): (118, 0, 19, 1),
+        ('ccx', 3, False, True): (398, 0, 84, 1),
     }

lms/djangoapps/ccx/tests/test_overrides.py

@@ -7,7 +7,10 @@ import mock
 import pytz
 from nose.plugins.attrib import attr
 
+from ccx_keys.locator import CCXLocator
+from courseware.courses import get_course_by_id
 from courseware.field_overrides import OverrideFieldData
+from courseware.testutils import FieldOverrideTestMixin
 from django.test.utils import override_settings
 from lms.djangoapps.courseware.tests.test_field_overrides import inject_field_overrides
 from request_cache.middleware import RequestCache
@@ -24,9 +27,11 @@ from lms.djangoapps.ccx.tests.utils import flatten, iter_blocks
 
 
 @attr('shard_1')
-@override_settings(FIELD_OVERRIDE_PROVIDERS=(
-    'ccx.overrides.CustomCoursesForEdxOverrideProvider',))
-class TestFieldOverrides(SharedModuleStoreTestCase):
+@override_settings(
+    XBLOCK_FIELD_DATA_WRAPPERS=['lms.djangoapps.courseware.field_overrides:OverrideModulestoreFieldData.wrap'],
+    MODULESTORE_FIELD_OVERRIDE_PROVIDERS=['ccx.overrides.CustomCoursesForEdxOverrideProvider'],
+)
+class TestFieldOverrides(FieldOverrideTestMixin, SharedModuleStoreTestCase):
     """
     Make sure field overrides behave in the expected manner.
     """
@@ -77,6 +82,9 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
 
         inject_field_overrides(iter_blocks(ccx.course), self.course, AdminFactory.create())
 
+        self.ccx_key = CCXLocator.from_course_locator(self.course.id, ccx.id)
+        self.ccx_course = get_course_by_id(self.ccx_key, depth=None)
+
         def cleanup_provider_classes():
             """
             After everything is done, clean up by un-doing the change to the
@@ -90,7 +98,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         Test that overriding start date on a chapter works.
         """
         ccx_start = datetime.datetime(2014, 12, 25, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         override_field_for_ccx(self.ccx, chapter, 'start', ccx_start)
         self.assertEquals(chapter.start, ccx_start)
 
@@ -99,7 +107,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         Test that for creating new field executed only create query
         """
         ccx_start = datetime.datetime(2014, 12, 25, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         # One outer SAVEPOINT/RELEASE SAVEPOINT pair around everything caused by the
         # transaction.atomic decorator wrapping override_field_for_ccx.
         # One SELECT and one INSERT.
@@ -114,7 +122,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         """
         ccx_start = datetime.datetime(2014, 12, 25, 00, 00, tzinfo=pytz.UTC)
         new_ccx_start = datetime.datetime(2015, 12, 25, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         override_field_for_ccx(self.ccx, chapter, 'start', ccx_start)
         with self.assertNumQueries(3):
             override_field_for_ccx(self.ccx, chapter, 'start', new_ccx_start)
@@ -124,7 +132,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         Test that if value of field does not changed no query execute.
         """
         ccx_start = datetime.datetime(2014, 12, 25, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         override_field_for_ccx(self.ccx, chapter, 'start', ccx_start)
         with self.assertNumQueries(2):      # 2 savepoints
             override_field_for_ccx(self.ccx, chapter, 'start', ccx_start)
@@ -134,7 +142,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         Test no extra queries when accessing an overriden field more than once.
         """
         ccx_start = datetime.datetime(2014, 12, 25, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         # One outer SAVEPOINT/RELEASE SAVEPOINT pair around everything caused by the
         # transaction.atomic decorator wrapping override_field_for_ccx.
         # One SELECT and one INSERT.
@@ -148,7 +156,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         Test that sequentials inherit overridden start date from chapter.
         """
         ccx_start = datetime.datetime(2014, 12, 25, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         override_field_for_ccx(self.ccx, chapter, 'start', ccx_start)
         self.assertEquals(chapter.get_children()[0].start, ccx_start)
         self.assertEquals(chapter.get_children()[1].start, ccx_start)
@@ -160,7 +168,7 @@ class TestFieldOverrides(SharedModuleStoreTestCase):
         the mooc.
         """
         ccx_due = datetime.datetime(2015, 1, 1, 00, 00, tzinfo=pytz.UTC)
-        chapter = self.ccx.course.get_children()[0]
+        chapter = self.ccx_course.get_children()[0]
         chapter.display_name = 'itsme!'
         override_field_for_ccx(self.ccx, chapter, 'due', ccx_due)
         vertical = chapter.get_children()[0].get_children()[0]

lms/djangoapps/ccx/tests/test_views.py

@@ -15,6 +15,7 @@ from courseware.courses import get_course_by_id
 from courseware.tests.factories import StudentModuleFactory
 from courseware.tests.helpers import LoginEnrollmentTestCase
 from courseware.tabs import get_course_tab_list
+from courseware.testutils import FieldOverrideTestMixin
 from instructor.access import (
     allow_access,
     list_with_level,
@@ -921,10 +922,12 @@ def patched_get_children(self, usage_key_filter=None):
 
 
 @attr('shard_1')
-@override_settings(FIELD_OVERRIDE_PROVIDERS=(
-    'ccx.overrides.CustomCoursesForEdxOverrideProvider',))
+@override_settings(
+    XBLOCK_FIELD_DATA_WRAPPERS=['lms.djangoapps.courseware.field_overrides:OverrideModulestoreFieldData.wrap'],
+    MODULESTORE_FIELD_OVERRIDE_PROVIDERS=['ccx.overrides.CustomCoursesForEdxOverrideProvider'],
+)
 @patch('xmodule.x_module.XModuleMixin.get_children', patched_get_children, spec=True)
-class TestCCXGrades(SharedModuleStoreTestCase, LoginEnrollmentTestCase):
+class TestCCXGrades(FieldOverrideTestMixin, SharedModuleStoreTestCase, LoginEnrollmentTestCase):
     """
     Tests for Custom Courses views.
     """

lms/djangoapps/ccx/utils.py

@@ -282,12 +282,6 @@ def ccx_students_enrolling_center(action, identifiers, email_students, course_ke
 
 def prep_course_for_grading(course, request):
     """Set up course module for overrides to function properly"""
-    field_data_cache = FieldDataCache.cache_for_descriptor_descendents(
-        course.id, request.user, course, depth=2)
-    course = get_module_for_descriptor(
-        request.user, request, course, field_data_cache, course.id, course=course
-    )
-
     course._field_data_cache = {}  # pylint: disable=protected-access
     course.set_grading_policy(course.grading_policy)
 

lms/djangoapps/certificates/management/commands/fix_ungraded_certs.py

@@ -51,7 +51,7 @@ class Command(BaseCommand):
 
         for cert in ungraded:
             # grade the student
-            grade = grades.grade(cert.user, request, course)
+            grade = grades.grade(cert.user, course)
             print "grading {0} - {1}".format(cert.user, grade['percent'])
             cert.grade = grade['percent']
             if not options['noop']:

lms/djangoapps/certificates/queue.py

@@ -257,7 +257,7 @@ class XQueueCertInterface(object):
         self.request.session = {}
 
         is_whitelisted = self.whitelist.filter(user=student, course_id=course_id, whitelist=True).exists()
-        grade = grades.grade(student, self.request, course)
+        grade = grades.grade(student, course)
         enrollment_mode, __ = CourseEnrollment.enrollment_mode_for_user(student, course_id)
         mode_is_verified = enrollment_mode in GeneratedCertificate.VERIFIED_CERTS_MODES
         user_is_verified = SoftwareSecurePhotoVerification.user_is_verified(student)

lms/djangoapps/certificates/tests/test_cert_management.py

@@ -13,7 +13,7 @@ from badges.events.course_complete import get_completion_badge
 from badges.models import BadgeAssertion
 from badges.tests.factories import BadgeAssertionFactory, CourseCompleteImageConfigurationFactory
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from xmodule.modulestore.tests.factories import CourseFactory, check_mongo_calls
+from xmodule.modulestore.tests.factories import CourseFactory, check_mongo_calls, ItemFactory
 from student.tests.factories import UserFactory, CourseEnrollmentFactory
 from certificates.management.commands import resubmit_error_certificates, regenerate_user, ungenerated_certs
 from certificates.models import GeneratedCertificate, CertificateStatuses
@@ -33,6 +33,9 @@ class CertificateManagementTest(ModuleStoreTestCase):
             CourseFactory.create()
             for __ in range(3)
         ]
+        for course in self.courses:
+            chapter = ItemFactory.create(parent_location=course.location)
+            ItemFactory.create(parent_location=chapter.location, category='sequential', graded=True)
         CourseCompleteImageConfigurationFactory.create()
 
     def _create_cert(self, course_key, user, status, mode=CourseMode.HONOR):

lms/djangoapps/course_api/blocks/serializers.py

@@ -17,10 +17,14 @@ class BlockSerializer(serializers.Serializer):  # pylint: disable=abstract-metho
         Get the field value requested.  The field may be an XBlock field, a
         transformer block field, or an entire tranformer block data dict.
         """
+        value = None
         if transformer is None:
             value = self.context['block_structure'].get_xblock_field(block_key, field_name)
         elif field_name is None:
-            value = self.context['block_structure'].get_transformer_block_data(block_key, transformer)
+            try:
+                value = self.context['block_structure'].get_transformer_block_data(block_key, transformer).fields
+            except KeyError:
+                pass
         else:
             value = self.context['block_structure'].get_transformer_block_field(block_key, transformer, field_name)
 

lms/djangoapps/course_api/blocks/tests/test_api.py

@@ -4,7 +4,6 @@ Tests for Blocks api.py
 
 from django.test.client import RequestFactory
 
-from openedx.core.djangoapps.content.block_structure.tests.helpers import EnableTransformerRegistryMixin
 from student.tests.factories import UserFactory
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
@@ -13,7 +12,7 @@ from xmodule.modulestore.tests.factories import SampleCourseFactory
 from ..api import get_blocks
 
 
-class TestGetBlocks(EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
+class TestGetBlocks(SharedModuleStoreTestCase):
     """
     Tests for the get_blocks function
     """

lms/djangoapps/course_api/blocks/tests/test_forms.py

@@ -7,7 +7,6 @@ from urllib import urlencode
 from rest_framework.exceptions import PermissionDenied
 
 from opaque_keys.edx.locator import CourseLocator
-from openedx.core.djangoapps.content.block_structure.tests.helpers import EnableTransformerRegistryMixin
 from openedx.core.djangoapps.util.test_forms import FormTestMixin
 from student.models import CourseEnrollment
 from student.tests.factories import UserFactory, CourseEnrollmentFactory
@@ -18,7 +17,7 @@ from ..forms import BlockListGetForm
 
 
 @ddt.ddt
-class TestBlockListGetForm(EnableTransformerRegistryMixin, FormTestMixin, SharedModuleStoreTestCase):
+class TestBlockListGetForm(FormTestMixin, SharedModuleStoreTestCase):
     """
     Tests for BlockListGetForm
     """

lms/djangoapps/course_api/blocks/tests/test_serializers.py

@@ -3,7 +3,6 @@ Tests for Course Blocks serializers
 """
 from mock import MagicMock
 
-from openedx.core.djangoapps.content.block_structure.tests.helpers import EnableTransformerRegistryMixin
 from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
 from student.tests.factories import UserFactory
 from xmodule.modulestore import ModuleStoreEnum
@@ -17,7 +16,7 @@ from ..serializers import BlockSerializer, BlockDictSerializer
 from .helpers import deserialize_usage_key
 
 
-class TestBlockSerializerBase(EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
+class TestBlockSerializerBase(SharedModuleStoreTestCase):
     """
     Base class for testing BlockSerializer and BlockDictSerializer
     """
@@ -42,10 +41,11 @@ class TestBlockSerializerBase(EnableTransformerRegistryMixin, SharedModuleStoreT
             block_types_to_count=['video'],
             requested_student_view_data=['video'],
         )
+        self.transformers = BlockStructureTransformers(COURSE_BLOCK_ACCESS_TRANSFORMERS + [blocks_api_transformer])
         self.block_structure = get_course_blocks(
             self.user,
             self.course.location,
-            BlockStructureTransformers(COURSE_BLOCK_ACCESS_TRANSFORMERS + [blocks_api_transformer]),
+            self.transformers,
         )
         self.serializer_context = {
             'request': MagicMock(),
@@ -93,7 +93,7 @@ class TestBlockSerializerBase(EnableTransformerRegistryMixin, SharedModuleStoreT
             {
                 'id', 'type', 'lms_web_url', 'student_view_url',
                 'display_name', 'graded',
-                'block_counts', 'student_view_multi_device',
+                'student_view_multi_device',
                 'lti_url',
                 'visible_to_staff_only',
             },
@@ -109,6 +109,13 @@ class TestBlockSerializerBase(EnableTransformerRegistryMixin, SharedModuleStoreT
             self.assertIn('student_view_multi_device', serialized_block)
             self.assertTrue(serialized_block['student_view_multi_device'])
 
+        # chapters with video should have block_counts
+        if serialized_block['type'] == 'chapter':
+            if serialized_block['display_name'] not in ('poll_test', 'handout_container'):
+                self.assertIn('block_counts', serialized_block)
+            else:
+                self.assertNotIn('block_counts', serialized_block)
+
     def create_staff_context(self):
         """
         Create staff user and course blocks accessible by that user
@@ -120,7 +127,7 @@ class TestBlockSerializerBase(EnableTransformerRegistryMixin, SharedModuleStoreT
         block_structure = get_course_blocks(
             staff_user,
             self.course.location,
-            BlockStructureTransformers(COURSE_BLOCK_ACCESS_TRANSFORMERS),
+            self.transformers,
         )
         return {
             'request': MagicMock(),
@@ -157,12 +164,14 @@ class TestBlockSerializer(TestBlockSerializerBase):
         serializer = self.create_serializer()
         for serialized_block in serializer.data:
             self.assert_basic_block(serialized_block['id'], serialized_block)
+        self.assertEquals(len(serializer.data), 28)
 
     def test_additional_requested_fields(self):
         self.add_additional_requested_fields()
         serializer = self.create_serializer()
         for serialized_block in serializer.data:
             self.assert_extended_block(serialized_block)
+        self.assertEquals(len(serializer.data), 28)
 
     def test_staff_fields(self):
         """
@@ -174,6 +183,7 @@ class TestBlockSerializer(TestBlockSerializerBase):
         for serialized_block in serializer.data:
             self.assert_extended_block(serialized_block)
             self.assert_staff_fields(serialized_block)
+        self.assertEquals(len(serializer.data), 29)
 
 
 class TestBlockDictSerializer(TestBlockSerializerBase):
@@ -201,12 +211,14 @@ class TestBlockDictSerializer(TestBlockSerializerBase):
         for block_key_string, serialized_block in serializer.data['blocks'].iteritems():
             self.assertEquals(serialized_block['id'], block_key_string)
             self.assert_basic_block(block_key_string, serialized_block)
+        self.assertEquals(len(serializer.data['blocks']), 28)
 
     def test_additional_requested_fields(self):
         self.add_additional_requested_fields()
         serializer = self.create_serializer()
         for serialized_block in serializer.data['blocks'].itervalues():
             self.assert_extended_block(serialized_block)
+        self.assertEquals(len(serializer.data['blocks']), 28)
 
     def test_staff_fields(self):
         """
@@ -218,3 +230,4 @@ class TestBlockDictSerializer(TestBlockSerializerBase):
         for serialized_block in serializer.data['blocks'].itervalues():
             self.assert_extended_block(serialized_block)
             self.assert_staff_fields(serialized_block)
+        self.assertEquals(len(serializer.data['blocks']), 29)

lms/djangoapps/course_api/blocks/tests/test_views.py

@@ -8,7 +8,6 @@ from urllib import urlencode
 from urlparse import urlunparse
 
 from opaque_keys.edx.locator import CourseLocator
-from openedx.core.djangoapps.content.block_structure.tests.helpers import EnableTransformerRegistryMixin
 from student.models import CourseEnrollment
 from student.tests.factories import AdminFactory, CourseEnrollmentFactory, UserFactory
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
@@ -17,7 +16,7 @@ from xmodule.modulestore.tests.factories import ToyCourseFactory
 from .helpers import deserialize_usage_key
 
 
-class TestBlocksView(EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
+class TestBlocksView(SharedModuleStoreTestCase):
     """
     Test class for BlocksView
     """

lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py

@@ -38,13 +38,13 @@ class TestBlockCountsTransformer(ModuleStoreTestCase):
         )
 
         # verify count of chapters
-        self.assertEquals(block_counts_for_course['chapter'], 2)
+        self.assertEquals(block_counts_for_course.chapter, 2)
 
         # verify count of problems
-        self.assertEquals(block_counts_for_course['problem'], 6)
-        self.assertEquals(block_counts_for_chapter_x['problem'], 3)
+        self.assertEquals(block_counts_for_course.problem, 6)
+        self.assertEquals(block_counts_for_chapter_x.problem, 3)
 
         # verify other block types are not counted
         for block_type in ['course', 'html', 'video']:
-            self.assertNotIn(block_type, block_counts_for_course)
-            self.assertNotIn(block_type, block_counts_for_chapter_x)
+            self.assertFalse(hasattr(block_counts_for_course, block_type))
+            self.assertFalse(hasattr(block_counts_for_chapter_x, block_type))

lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py

@@ -15,8 +15,6 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
     """
     Tests generate course blocks management command.
     """
-    ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
-
     def setUp(self):
         """
         Create courses in modulestore.

lms/djangoapps/course_blocks/transformers/tests/helpers.py

@@ -36,8 +36,6 @@ class CourseStructureTestCase(TransformerRegistryTestMixin, ModuleStoreTestCase)
     """
     Helper for test cases that need to build course structures.
     """
-    ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
-
     def setUp(self):
         """
         Create users.

lms/djangoapps/courseware/model_data.py

@@ -940,7 +940,6 @@ class ScoresClient(object):
     Score = namedtuple('Score', 'correct total')
 
     def __init__(self, course_key, user_id):
-        """Basic constructor. from_field_data_cache() is more appopriate for most uses."""
         self.course_key = course_key
         self.user_id = user_id
         self._locations_to_scores = {}
@@ -983,10 +982,10 @@ class ScoresClient(object):
         return self._locations_to_scores.get(location.replace(version=None, branch=None))
 
     @classmethod
-    def from_field_data_cache(cls, fd_cache):
-        """Create a ScoresClient from a populated FieldDataCache."""
-        client = cls(fd_cache.course_id, fd_cache.user.id)
-        client.fetch_scores(fd_cache.scorable_locations)
+    def create_for_locations(cls, course_id, user_id, scorable_locations):
+        """Create a ScoresClient with pre-fetched data for the given locations."""
+        client = cls(course_id, user_id)
+        client.fetch_scores(scorable_locations)
         return client
 
 

lms/djangoapps/courseware/tests/__init__.py

@@ -82,9 +82,6 @@ class BaseTestXmodule(ModuleStoreTestCase):
 
         self.item_descriptor.xmodule_runtime = self.new_module_runtime()
 
-        #self.item_module = self.item_descriptor.xmodule_runtime.xmodule_instance
-        #self.item_module is None at this time
-
         self.item_url = unicode(self.item_descriptor.location)
 
     def setup_course(self):

lms/djangoapps/courseware/tests/test_discussion_module.py

@@ -6,7 +6,6 @@ from mock import Mock
 from . import BaseTestXmodule
 from course_api.blocks.tests.helpers import deserialize_usage_key
 from courseware.module_render import get_module_for_descriptor_internal
-from openedx.core.djangoapps.content.block_structure.tests.helpers import EnableTransformerRegistryMixin
 from student.tests.factories import UserFactory, CourseEnrollmentFactory
 from xmodule.discussion_module import DiscussionModule
 from xmodule.modulestore import ModuleStoreEnum
@@ -16,7 +15,7 @@ from xmodule.modulestore.tests.factories import ToyCourseFactory, ItemFactory
 
 
 @ddt.ddt
-class DiscussionModuleTest(BaseTestXmodule, EnableTransformerRegistryMixin, SharedModuleStoreTestCase):
+class DiscussionModuleTest(BaseTestXmodule, SharedModuleStoreTestCase):
     """Logic tests for Discussion Xmodule."""
     CATEGORY = "discussion"
 

lms/djangoapps/courseware/tests/test_field_overrides.py

@@ -17,6 +17,7 @@ from ..field_overrides import (
     OverrideFieldData,
     OverrideModulestoreFieldData,
 )
+from ..testutils import FieldOverrideTestMixin
 
 
 TESTUSER = "testuser"
@@ -128,15 +129,7 @@ class OverrideFieldDataTests(SharedModuleStoreTestCase):
 @override_settings(
     MODULESTORE_FIELD_OVERRIDE_PROVIDERS=['courseware.tests.test_field_overrides.TestOverrideProvider']
 )
-class OverrideModulestoreFieldDataTests(OverrideFieldDataTests):
-    def setUp(self):
-        super(OverrideModulestoreFieldDataTests, self).setUp()
-        OverrideModulestoreFieldData.provider_classes = None
-
-    def tearDown(self):
-        super(OverrideModulestoreFieldDataTests, self).tearDown()
-        OverrideModulestoreFieldData.provider_classes = None
-
+class OverrideModulestoreFieldDataTests(FieldOverrideTestMixin, OverrideFieldDataTests):
     def make_one(self):
         return OverrideModulestoreFieldData.wrap(self.course, DictFieldData({
             'foo': 'bar',

lms/djangoapps/courseware/tests/test_grades.py

@@ -11,7 +11,6 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
 
 from courseware.grades import (
-    field_data_cache_for_grading,
     grade,
     iterate_grades_for,
     MaxScoresCache,
@@ -31,7 +30,7 @@ from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
 
 
-def _grade_with_errors(student, request, course, keep_raw_scores=False):
+def _grade_with_errors(student, course, keep_raw_scores=False):
     """This fake grade method will throw exceptions for student3 and
     student4, but allow any other students to go through normal grading.
 
@@ -42,7 +41,7 @@ def _grade_with_errors(student, request, course, keep_raw_scores=False):
     if student.username in ['student3', 'student4']:
         raise Exception("I don't like {}".format(student.username))
 
-    return grade(student, request, course, keep_raw_scores=keep_raw_scores)
+    return grade(student, course, keep_raw_scores=keep_raw_scores)
 
 
 @attr('shard_1')
@@ -217,15 +216,6 @@ class TestFieldDataCacheScorableLocations(SharedModuleStoreTestCase):
 
         CourseEnrollment.enroll(self.student, self.course.id)
 
-    def test_field_data_cache_scorable_locations(self):
-        """Only scorable locations should be in FieldDataCache.scorable_locations."""
-        fd_cache = field_data_cache_for_grading(self.course, self.student)
-        block_types = set(loc.block_type for loc in fd_cache.scorable_locations)
-        self.assertNotIn('video', block_types)
-        self.assertNotIn('html', block_types)
-        self.assertNotIn('discussion', block_types)
-        self.assertIn('problem', block_types)
-
 
 class TestProgressSummary(TestCase):
     """

lms/djangoapps/courseware/tests/test_module_render.py

@@ -339,16 +339,16 @@ class ModuleRenderTestCase(SharedModuleStoreTestCase, LoginEnrollmentTestCase):
         )
 
     @override_settings(FIELD_OVERRIDE_PROVIDERS=(
-        'ccx.overrides.CustomCoursesForEdxOverrideProvider',
+        'courseware.student_field_overrides.IndividualStudentOverrideProvider',
     ))
-    def test_rebind_different_users_ccx(self):
+    def test_rebind_different_users(self):
         """
         This tests the rebinding a descriptor to a student does not result
-        in overly nested _field_data when CCX is enabled.
+        in overly nested _field_data.
         """
         request = self.request_factory.get('')
         request.user = self.mock_user
-        course = CourseFactory.create(enable_ccx=True)
+        course = CourseFactory.create()
 
         descriptor = ItemFactory(category='html', parent=course)
         field_data_cache = FieldDataCache(

lms/djangoapps/courseware/tests/test_submitting_problems.py

@@ -256,13 +256,7 @@ class TestSubmittingProblems(ModuleStoreTestCase, LoginEnrollmentTestCase, Probl
         - grade_breakdown : A breakdown of the major components that
             make up the final grade. (For display)
         """
-
-        fake_request = self.factory.get(
-            reverse('progress', kwargs={'course_id': self.course.id.to_deprecated_string()})
-        )
-
-        fake_request.user = self.student_user
-        return grades.grade(self.student_user, fake_request, self.course)
+        return grades.grade(self.student_user, self.course)
 
     def get_progress_summary(self):
         """
@@ -275,15 +269,7 @@ class TestSubmittingProblems(ModuleStoreTestCase, LoginEnrollmentTestCase, Probl
         ungraded problems, and is good for displaying a course summary with due dates,
         etc.
         """
-
-        fake_request = self.factory.get(
-            reverse('progress', kwargs={'course_id': self.course.id.to_deprecated_string()})
-        )
-
-        progress_summary = grades.progress_summary(
-            self.student_user, fake_request, self.course
-        )
-        return progress_summary
+        return grades.progress_summary(self.student_user, self.course)
 
     def check_grade_percent(self, percent):
         """

lms/djangoapps/courseware/tests/test_video_mongo.py

@@ -482,23 +482,7 @@ class TestGetHtmlMethod(BaseTestXmodule):
         # it'll just fall back to the values in the VideoDescriptor.
         self.assertIn("example_source.mp4", self.item_descriptor.render(STUDENT_VIEW).content)
 
-    @patch('edxval.api.get_video_info')
-    def test_get_html_with_mocked_edx_video_id(self, mock_get_video_info):
-        mock_get_video_info.return_value = {
-            'url': '/edxval/video/example',
-            'edx_video_id': u'example',
-            'duration': 111.0,
-            'client_video_id': u'The example video',
-            'encoded_videos': [
-                {
-                    'url': u'http://www.meowmix.com',
-                    'file_size': 25556,
-                    'bitrate': 9600,
-                    'profile': u'desktop_mp4'
-                }
-            ]
-        }
-
+    def test_get_html_with_mocked_edx_video_id(self):
         SOURCE_XML = """
             <video show_captions="true"
             display_name="A Name"
@@ -558,7 +542,23 @@ class TestGetHtmlMethod(BaseTestXmodule):
             edx_video_id=data['edx_video_id']
         )
         self.initialize_module(data=DATA)
-        context = self.item_descriptor.render(STUDENT_VIEW).content
+
+        with patch('edxval.api.get_video_info') as mock_get_video_info:
+            mock_get_video_info.return_value = {
+                'url': '/edxval/video/example',
+                'edx_video_id': u'example',
+                'duration': 111.0,
+                'client_video_id': u'The example video',
+                'encoded_videos': [
+                    {
+                        'url': u'http://www.meowmix.com',
+                        'file_size': 25556,
+                        'bitrate': 9600,
+                        'profile': u'desktop_mp4'
+                    }
+                ]
+            }
+            context = self.item_descriptor.render(STUDENT_VIEW).content
 
         expected_context = dict(initial_context)
         expected_context['metadata'].update({

lms/djangoapps/courseware/tests/test_views.py

@@ -1340,7 +1340,7 @@ class ProgressPageTests(ModuleStoreTestCase):
         self.assertContains(resp, u"Download Your Certificate")
 
     @ddt.data(
-        *itertools.product(((55, 4, True), (55, 4, False)), (True, False))
+        *itertools.product(((46, 4, True), (46, 4, False)), (True, False))
     )
     @ddt.unpack
     def test_query_counts(self, (sql_calls, mongo_calls, self_paced), self_paced_enabled):

lms/djangoapps/courseware/testutils.py

@@ -8,6 +8,7 @@ import ddt
 from mock import patch
 from urllib import urlencode
 
+from lms.djangoapps.courseware.field_overrides import OverrideModulestoreFieldData
 from lms.djangoapps.courseware.url_helpers import get_redirect_url
 from student.tests.factories import AdminFactory, UserFactory, CourseEnrollmentFactory
 from xmodule.modulestore import ModuleStoreEnum
@@ -197,3 +198,16 @@ class RenderXBlockTestMixin(object):
         self.setup_course()
         self.setup_user(admin=False, enroll=True, login=True)
         self.verify_response(url_params={'view': 'author_view'}, expected_response_code=400)
+
+
+class FieldOverrideTestMixin(object):
+    """
+    A Mixin helper class for classes that test Field Overrides.
+    """
+    def setUp(self):
+        super(FieldOverrideTestMixin, self).setUp()
+        OverrideModulestoreFieldData.provider_classes = None
+
+    def tearDown(self):
+        super(FieldOverrideTestMixin, self).tearDown()
+        OverrideModulestoreFieldData.provider_classes = None

lms/djangoapps/courseware/views/views.py

@@ -38,6 +38,7 @@ from instructor.views.api import require_global_staff
 import shoppingcart
 import survey.utils
 import survey.views
+from lms.djangoapps.ccx.utils import prep_course_for_grading
 from certificates import api as certs_api
 from openedx.core.djangoapps.models.course_details import CourseDetails
 from commerce.utils import EcommerceService
@@ -681,6 +682,7 @@ def _progress(request, course_key, student_id):
             raise Http404
 
     course = get_course_with_access(request.user, 'load', course_key, depth=None, check_if_enrolled=True)
+    prep_course_for_grading(course, request)
 
     # check to see if there is a required survey that must be taken before
     # the user can access the course.
@@ -714,16 +716,8 @@ def _progress(request, course_key, student_id):
     # additional DB lookup (this kills the Progress page in particular).
     student = User.objects.prefetch_related("groups").get(id=student.id)
 
-    with outer_atomic():
-        field_data_cache = grades.field_data_cache_for_grading(course, student)
-        scores_client = ScoresClient.from_field_data_cache(field_data_cache)
-
-    courseware_summary = grades.progress_summary(
-        student, request, course, field_data_cache=field_data_cache, scores_client=scores_client
-    )
-    grade_summary = grades.grade(
-        student, request, course, field_data_cache=field_data_cache, scores_client=scores_client
-    )
+    courseware_summary = grades.progress_summary(student, course)
+    grade_summary = grades.grade(student, course)
     studio_url = get_studio_url(course, 'settings/grading')
 
     if courseware_summary is None:
@@ -1056,7 +1050,7 @@ def is_course_passed(course, grade_summary=None, student=None, request=None):
     success_cutoff = min(nonzero_cutoffs) if nonzero_cutoffs else None
 
     if grade_summary is None:
-        grade_summary = grades.grade(student, request, course)
+        grade_summary = grades.grade(student, course)
 
     return success_cutoff and grade_summary['percent'] >= success_cutoff
 

lms/djangoapps/django_comment_client/forum/tests.py

@@ -342,11 +342,11 @@ class SingleThreadQueryCountTestCase(ModuleStoreTestCase):
 
     @ddt.data(
         # old mongo with cache
-        (ModuleStoreEnum.Type.mongo, 1, 6, 4, 18, 10),
-        (ModuleStoreEnum.Type.mongo, 50, 6, 4, 18, 10),
+        (ModuleStoreEnum.Type.mongo, 1, 6, 4, 17, 8),
+        (ModuleStoreEnum.Type.mongo, 50, 6, 4, 17, 8),
         # split mongo: 3 queries, regardless of thread response size.
-        (ModuleStoreEnum.Type.split, 1, 3, 3, 18, 10),
-        (ModuleStoreEnum.Type.split, 50, 3, 3, 18, 10),
+        (ModuleStoreEnum.Type.split, 1, 3, 3, 17, 8),
+        (ModuleStoreEnum.Type.split, 50, 3, 3, 17, 8),
     )
     @ddt.unpack
     def test_number_of_mongo_queries(

lms/djangoapps/instructor/features/data_download.py

@@ -63,7 +63,7 @@ Graded sections:
 Listing grading context for course {}
 graded sections:
 []
-all descriptors:
+all graded blocks:
 length=0""".format(world.course_key)
     assert_in(expected_config, world.css_text('#data-grade-config-text'))
 

lms/djangoapps/instructor/offline_gradecalc.py

@@ -50,7 +50,7 @@ def offline_grade_calculation(course_key):
         request.user = student
         request.session = {}
 
-        gradeset = grades.grade(student, request, course, keep_raw_scores=True)
+        gradeset = grades.grade(student, course, keep_raw_scores=True)
         # Convert Score namedtuples to dicts:
         totaled_scores = gradeset['totaled_scores']
         for section in totaled_scores:
@@ -89,7 +89,7 @@ def student_grades(student, request, course, keep_raw_scores=False, use_offline=
     as use_offline.  If use_offline is True then this will look for an offline computed gradeset in the DB.
     '''
     if not use_offline:
-        return grades.grade(student, request, course, keep_raw_scores=keep_raw_scores)
+        return grades.grade(student, course, keep_raw_scores=keep_raw_scores)
 
     try:
         ocg = models.OfflineComputedGrade.objects.get(user=student, course_id=course.id)

lms/djangoapps/instructor/tests/test_offline_gradecalc.py

@@ -16,7 +16,7 @@ from xmodule.modulestore.tests.factories import CourseFactory
 from ..offline_gradecalc import offline_grade_calculation, student_grades
 
 
-def mock_grade(_student, _request, course, **_kwargs):
+def mock_grade(_student, course, **_kwargs):
     """ Return some fake grade data to mock grades.grade() """
     return {
         'grade': u'Pass',
@@ -104,4 +104,4 @@ class TestOfflineGradeCalc(ModuleStoreTestCase):
         offline_grade_calculation(self.course.id)
         with patch('courseware.grades.grade', side_effect=AssertionError('Should not re-grade')):
             result = student_grades(self.user, None, self.course, use_offline=True)
-        self.assertEqual(result, mock_grade(self.user, None, self.course))
+        self.assertEqual(result, mock_grade(self.user, self.course))

lms/djangoapps/instructor/tests/views/test_instructor_dashboard.py

@@ -392,6 +392,6 @@ class TestInstructorDashboardPerformance(ModuleStoreTestCase, LoginEnrollmentTes
 
         # check MongoDB calls count
         url = reverse('spoc_gradebook', kwargs={'course_id': self.course.id})
-        with check_mongo_calls(8):
+        with check_mongo_calls(7):
             response = self.client.get(url)
             self.assertEqual(response.status_code, 200)

lms/djangoapps/instructor_analytics/basic.py

@@ -24,6 +24,7 @@ from courseware.models import StudentModule
 from certificates.models import GeneratedCertificate
 from django.db.models import Count
 from certificates.models import CertificateStatuses
+from courseware.grades import grading_context_for_course
 
 
 STUDENT_FEATURES = ('id', 'username', 'first_name', 'last_name', 'is_staff', 'email')
@@ -490,14 +491,14 @@ def dump_grading_context(course):
     msg += hbar
     msg += "Listing grading context for course %s\n" % course.id.to_deprecated_string()
 
-    gcontext = course.grading_context
+    gcontext = grading_context_for_course(course)
     msg += "graded sections:\n"
 
-    msg += '%s\n' % gcontext['graded_sections'].keys()
-    for (gsomething, gsvals) in gcontext['graded_sections'].items():
+    msg += '%s\n' % gcontext['all_graded_sections'].keys()
+    for (gsomething, gsvals) in gcontext['all_graded_sections'].items():
         msg += "--> Section %s:\n" % (gsomething)
         for sec in gsvals:
-            sdesc = sec['section_descriptor']
+            sdesc = sec['section_block']
             frmat = getattr(sdesc, 'format', None)
             aname = ''
             if frmat in graders:
@@ -512,7 +513,7 @@ def dump_grading_context(course):
                 notes = ', score by attempt!'
             msg += "      %s (format=%s, Assignment=%s%s)\n"\
                 % (sdesc.display_name, frmat, aname, notes)
-    msg += "all descriptors:\n"
-    msg += "length=%d\n" % len(gcontext['all_descriptors'])
+    msg += "all graded blocks:\n"
+    msg += "length=%d\n" % len(gcontext['all_graded_blocks'])
     msg = '<pre>%s</pre>' % msg.replace('<', '&lt;')
     return msg

lms/djangoapps/instructor_task/tests/test_tasks_helper.py

@@ -285,7 +285,7 @@ class TestInstructorGradeReport(InstructorGradeReportTestCase):
             user_b.username,
             course.id,
             cohort_name_header,
-            ''
+            u'Default Group',
         )
 
     @patch('instructor_task.tasks_helper._get_current_task')
@@ -685,7 +685,7 @@ class TestProblemReportSplitTestContent(TestReportMixin, TestConditionalContent,
 
     def test_problem_grade_report(self):
         """
-        Test that we generate the correct the correct grade report when dealing with A/B tests.
+        Test that we generate the correct grade report when dealing with A/B tests.
 
         In order to verify that the behavior of the grade report is correct, we submit answers for problems
         that the student won't have access to. A/B tests won't restrict access to the problems, but it should

lms/envs/aws.py

@@ -731,7 +731,7 @@ COURSE_CATALOG_API_URL = ENV_TOKENS.get('COURSE_CATALOG_API_URL', COURSE_CATALOG
 ##### Custom Courses for EdX #####
 if FEATURES.get('CUSTOM_COURSES_EDX'):
     INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
-    FIELD_OVERRIDE_PROVIDERS += (
+    MODULESTORE_FIELD_OVERRIDE_PROVIDERS += (
         'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider',
     )
 CCX_MAX_STUDENTS_ALLOWED = ENV_TOKENS.get('CCX_MAX_STUDENTS_ALLOWED', CCX_MAX_STUDENTS_ALLOWED)

lms/envs/test.py

@@ -577,11 +577,6 @@ JWT_AUTH.update({
     'JWT_AUDIENCE': 'test-key',
 })
 
-# Disable the use of the plugin manager in the transformer registry for
-# better performant unit tests.
-from openedx.core.lib.block_structure.transformer_registry import TransformerRegistry
-TransformerRegistry.USE_PLUGIN_MANAGER = False
-
 # Set the default Oauth2 Provider Model so that migrations can run in
 # verbose mode
 OAUTH2_PROVIDER_APPLICATION_MODEL = 'oauth2_provider.Application'

lms/envs/yaml_config.py

@@ -299,7 +299,7 @@ GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE
 ##### Custom Courses for EdX #####
 if FEATURES.get('CUSTOM_COURSES_EDX'):
     INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
-    FIELD_OVERRIDE_PROVIDERS += (
+    MODULESTORE_FIELD_OVERRIDE_PROVIDERS += (
         'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider',
     )
 

openedx/core/djangoapps/content/block_structure/tests/helpers.py

@@ -3,28 +3,9 @@ Helpers for Course Blocks tests.
 """
 
 from openedx.core.lib.block_structure.cache import BlockStructureCache
-from openedx.core.lib.block_structure.transformer_registry import TransformerRegistry
 from ..api import get_cache
 
 
-class EnableTransformerRegistryMixin(object):
-    """
-    Mixin that enables the TransformerRegistry to USE_PLUGIN_MANAGER for
-    finding registered transformers.  USE_PLUGIN_MANAGER is set to False
-    for LMS unit tests to speed up performance of the unit tests, so all
-    registered transformers in the platform do not need to be collected.
-    This Mixin is expected to be used by Tests for integration testing
-    with all registered transformers.
-    """
-    def setUp(self, **kwargs):
-        super(EnableTransformerRegistryMixin, self).setUp(**kwargs)
-        TransformerRegistry.USE_PLUGIN_MANAGER = True
-
-    def tearDown(self):
-        super(EnableTransformerRegistryMixin, self).tearDown()
-        TransformerRegistry.USE_PLUGIN_MANAGER = False
-
-
 def is_course_in_block_structure_cache(course_key, store):
     """
     Returns whether the given course is in the Block Structure cache.

openedx/core/djangoapps/content/block_structure/tests/test_signals.py

@@ -7,15 +7,13 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 
 from ..api import get_block_structure_manager
-from .helpers import is_course_in_block_structure_cache, EnableTransformerRegistryMixin
+from .helpers import is_course_in_block_structure_cache
 
 
-class CourseBlocksSignalTest(EnableTransformerRegistryMixin, ModuleStoreTestCase):
+class CourseBlocksSignalTest(ModuleStoreTestCase):
     """
     Tests for the Course Blocks signal
     """
-    ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
-
     def setUp(self):
         super(CourseBlocksSignalTest, self).setUp()
         self.course = CourseFactory.create()

openedx/core/djangoapps/content/course_overviews/models.py

@@ -20,7 +20,7 @@ from lms.djangoapps import django_comment_client
 from openedx.core.djangoapps.models.course_details import CourseDetails
 from static_replace.models import AssetBaseUrlConfig
 from util.date_utils import strftime_localized
-from xmodule import course_metadata_utils
+from xmodule import course_metadata_utils, block_metadata_utils
 from xmodule.course_module import CourseDescriptor, DEFAULT_START_DATE
 from xmodule.error_module import ErrorDescriptor
 from xmodule.modulestore.django import modulestore
@@ -317,14 +317,14 @@ class CourseOverview(TimeStampedModel):
         """
         Returns this course's URL name.
         """
-        return course_metadata_utils.url_name_for_course_location(self.location)
+        return block_metadata_utils.url_name_for_block(self)
 
     @property
     def display_name_with_default(self):
         """
         Return reasonable display name for the course.
         """
-        return course_metadata_utils.display_name_with_default(self)
+        return block_metadata_utils.display_name_with_default(self)
 
     @property
     def display_name_with_default_escaped(self):
@@ -338,7 +338,7 @@ class CourseOverview(TimeStampedModel):
         migrate and test switching to display_name_with_default, which is no
         longer escaped.
         """
-        return course_metadata_utils.display_name_with_default_escaped(self)
+        return block_metadata_utils.display_name_with_default_escaped(self)
 
     def has_started(self):
         """

openedx/core/lib/block_structure/cache.py

@@ -40,7 +40,7 @@ class BlockStructureCache(object):
         """
         data_to_cache = (
             block_structure._block_relations,
-            block_structure._transformer_data,
+            block_structure.transformer_data,
             block_structure._block_data_map,
         )
         zp_data_to_cache = zpickle(data_to_cache)
@@ -99,7 +99,7 @@ class BlockStructureCache(object):
         block_relations, transformer_data, block_data_map = zunpickle(zp_data_from_cache)
         block_structure = BlockStructureModulestoreData(root_block_usage_key)
         block_structure._block_relations = block_relations
-        block_structure._transformer_data = transformer_data
+        block_structure.transformer_data = transformer_data
         block_structure._block_data_map = block_data_map
 
         return block_structure

openedx/core/lib/block_structure/manager.py

@@ -2,6 +2,8 @@
 Top-level module for the Block Structure framework with a class for managing
 BlockStructures.
 """
+from contextlib import contextmanager
+
 from .cache import BlockStructureCache
 from .factory import BlockStructureFactory
 from .exceptions import UsageKeyNotInBlockStructure
@@ -87,12 +89,13 @@ class BlockStructureManager(object):
         )
         cache_miss = block_structure is None
         if cache_miss or BlockStructureTransformers.is_collected_outdated(block_structure):
-            block_structure = BlockStructureFactory.create_from_modulestore(
-                self.root_block_usage_key,
-                self.modulestore
-            )
-            BlockStructureTransformers.collect(block_structure)
-            self.block_structure_cache.add(block_structure)
+            with self._bulk_operations():
+                block_structure = BlockStructureFactory.create_from_modulestore(
+                    self.root_block_usage_key,
+                    self.modulestore
+                )
+                BlockStructureTransformers.collect(block_structure)
+                self.block_structure_cache.add(block_structure)
         return block_structure
 
     def update_collected(self):
@@ -111,3 +114,15 @@ class BlockStructureManager(object):
         root block key.
         """
         self.block_structure_cache.delete(self.root_block_usage_key)
+
+    @contextmanager
+    def _bulk_operations(self):
+        """
+        A context manager for notifying the store of bulk operations.
+        """
+        try:
+            course_key = self.root_block_usage_key.course_key
+        except AttributeError:
+            course_key = None
+        with self.modulestore.bulk_operations(course_key):
+            yield

openedx/core/lib/block_structure/tests/helpers.py

@@ -68,6 +68,13 @@ class MockModulestore(object):
             raise ItemNotFoundError
         return item
 
+    @contextmanager
+    def bulk_operations(self, ignore):  # pylint: disable=unused-argument
+        """
+        A context manager for notifying the store of bulk operations.
+        """
+        yield
+
 
 class MockCache(object):
     """

openedx/core/lib/block_structure/tests/test_block_structure.py

@@ -138,17 +138,19 @@ class TestBlockStructureData(TestCase, ChildrenMapTestMixin):
 
         # verify fields have not been collected yet
         for block in blocks:
+            bs_block = block_structure[block.location]
             for field in fields:
-                self.assertIsNone(block_structure.get_xblock_field(block.location, field))
+                self.assertIsNone(getattr(bs_block, field, None))
 
         # collect fields
         block_structure._collect_requested_xblock_fields()
 
         # verify values of collected fields
         for block in blocks:
+            bs_block = block_structure[block.location]
             for field in fields:
                 self.assertEquals(
-                    block_structure.get_xblock_field(block.location, field),
+                    getattr(bs_block, field, None),
                     block.field_map.get(field),
                 )
 

openedx/core/lib/cache_utils.py

 """
 Utilities related to caching.
 """
+import collections
 import cPickle as pickle
 import functools
 import zlib
@@ -40,6 +41,48 @@ def memoize_in_request_cache(request_cache_attr_name=None):
     return _decorator
 
 
+class memoized(object):  # pylint: disable=invalid-name
+    """
+    Decorator. Caches a function's return value each time it is called.
+    If called later with the same arguments, the cached value is returned
+    (not reevaluated).
+    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
+
+    WARNING: Only use this memoized decorator for caching data that
+    is constant throughout the lifetime of a gunicorn worker process,
+    is costly to compute, and is required often.  Otherwise, it can lead to
+    unwanted memory leakage.
+    """
+
+    def __init__(self, func):
+        self.func = func
+        self.cache = {}
+
+    def __call__(self, *args):
+        if not isinstance(args, collections.Hashable):
+            # uncacheable. a list, for instance.
+            # better to not cache than blow up.
+            return self.func(*args)
+        if args in self.cache:
+            return self.cache[args]
+        else:
+            value = self.func(*args)
+            self.cache[args] = value
+            return value
+
+    def __repr__(self):
+        """
+        Return the function's docstring.
+        """
+        return self.func.__doc__
+
+    def __get__(self, obj, objtype):
+        """
+        Support instance methods.
+        """
+        return functools.partial(self.__call__, obj)
+
+
 def hashvalue(arg):
     """
     If arg is an xblock, use its location. otherwise just turn it into a string

setup.py

@@ -51,6 +51,7 @@ setup(
             "visibility = lms.djangoapps.course_blocks.transformers.visibility:VisibilityTransformer",
             "course_blocks_api = lms.djangoapps.course_api.blocks.transformers.blocks_api:BlocksAPITransformer",
             "proctored_exam = lms.djangoapps.course_api.blocks.transformers.proctored_exam:ProctoredExamTransformer",
+            "grades = lms.djangoapps.courseware.transformers.grades:GradesTransformer",
         ],
     }
 )