Architecture for arbitrary field overrides, field overrides for

individual students, and a reimplementation of the individual due date
feature.

This work introduces an architecture, used with the 'authored_data'
portion of LmsFieldData, which allows arbitrary field overrides to be
made for fields that are part of the course content or settings (Mongo
data).  The basic architecture is extensible by means of writing and
configuring arbitrary field override providers.

One concrete implementation of a field override provider is provided
which allows for overrides to be for individual students.  This provider
is then used as a basis for reimplementing the individual due date
extensions feature as a proof of concept for the design.

One can imagine writing override providers that provide overrides based
on a student's membership in a cohort or other similar idea.  This work
is being done, in fact, to pave the way for the Personal Online Courses
feature being developed by MIT, which will use an override provider very
much long those lines.

common/lib/xmodule/xmodule/capa_base.py

@@ -26,7 +26,6 @@ from xmodule.exceptions import NotFoundError
 from xblock.fields import Scope, String, Boolean, Dict, Integer, Float
 from .fields import Timedelta, Date
 from django.utils.timezone import UTC
-from .util.duedate import get_extended_due_date
 from xmodule.capa_base_constants import RANDOMIZATION, SHOWANSWER
 from django.conf import settings
 
@@ -107,14 +106,6 @@ class CapaFields(object):
         values={"min": 0}, scope=Scope.settings
     )
     due = Date(help=_("Date that this problem is due by"), scope=Scope.settings)
-    extended_due = Date(
-        help=_("Date that this problem is due by for a particular student. This "
-               "can be set by an instructor, and will override the global due "
-               "date if it is set to a date that is later than the global due "
-               "date."),
-        default=None,
-        scope=Scope.user_state,
-    )
     graceperiod = Timedelta(
         help=_("Amount of time after the due date that submissions will be accepted"),
         scope=Scope.settings
@@ -218,7 +209,7 @@ class CapaMixin(CapaFields):
     def __init__(self, *args, **kwargs):
         super(CapaMixin, self).__init__(*args, **kwargs)
 
-        due_date = get_extended_due_date(self)
+        due_date = self.due
 
         if self.graceperiod is not None and due_date:
             self.close_date = due_date + self.graceperiod

common/lib/xmodule/xmodule/combined_open_ended_module.py

@@ -23,7 +23,6 @@ V1_SETTINGS_ATTRIBUTES = [
     "accept_file_upload",
     "skip_spelling_checks",
     "due",
-    "extended_due",
     "graceperiod",
     "weight",
     "min_to_calibrate",
@@ -258,16 +257,6 @@ class CombinedOpenEndedFields(object):
         help=_("Date that this problem is due by"),
         scope=Scope.settings
     )
-    extended_due = Date(
-        help=_(
-            "Date that this problem is due by for a particular student. This "
-            "can be set by an instructor, and will override the global due "
-            "date if it is set to a date that is later than the global due "
-            "date."
-        ),
-        default=None,
-        scope=Scope.user_state,
-    )
     graceperiod = Timedelta(
         help=_("Amount of time after the due date that submissions will be accepted"),
         scope=Scope.settings

common/lib/xmodule/xmodule/foldit_module.py

@@ -8,7 +8,6 @@ from xmodule.x_module import XModule
 from xmodule.xml_module import XmlDescriptor
 from xblock.fields import Scope, Integer, String
 from .fields import Date
-from .util.duedate import get_extended_due_date
 
 
 log = logging.getLogger(__name__)
@@ -21,14 +20,6 @@ class FolditFields(object):
     required_level = Integer(default=4, scope=Scope.settings)
     required_sublevel = Integer(default=5, scope=Scope.settings)
     due = Date(help="Date that this problem is due by", scope=Scope.settings)
-    extended_due = Date(
-        help="Date that this problem is due by for a particular student. This "
-             "can be set by an instructor, and will override the global due "
-             "date if it is set to a date that is later than the global due "
-             "date.",
-        default=None,
-        scope=Scope.user_state,
-    )
 
     show_basic_score = String(scope=Scope.settings, default='false')
     show_leaderboard = String(scope=Scope.settings, default='false')
@@ -49,7 +40,7 @@ class FolditModule(FolditFields, XModule):
             show_leaderboard="false"/>
         """
         super(FolditModule, self).__init__(*args, **kwargs)
-        self.due_time = get_extended_due_date(self)
+        self.due_time = self.due
 
     def is_complete(self):
         """

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -44,14 +44,6 @@ class InheritanceMixin(XBlockMixin):
         help=_("Enter the default date by which problems are due."),
         scope=Scope.settings,
     )
-    extended_due = Date(
-        help="Date that this problem is due by for a particular student. This "
-             "can be set by an instructor, and will override the global due "
-             "date if it is set to a date that is later than the global due "
-             "date.",
-        default=None,
-        scope=Scope.user_state,
-    )
     visible_to_staff_only = Boolean(
         help=_("If true, can be seen only by course staff, regardless of start date."),
         default=False,

common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py

@@ -8,7 +8,6 @@ from xmodule.progress import Progress
 from xmodule.stringify import stringify_children
 from xmodule.open_ended_grading_classes import self_assessment_module
 from xmodule.open_ended_grading_classes import open_ended_module
-from xmodule.util.duedate import get_extended_due_date
 from .combined_open_ended_rubric import CombinedOpenEndedRubric, GRADER_TYPE_IMAGE_DICT, HUMAN_GRADER_TYPE, LEGEND_LIST
 from xmodule.open_ended_grading_classes.peer_grading_service import PeerGradingService, MockPeerGradingService
 from xmodule.open_ended_grading_classes.openendedchild import OpenEndedChild
@@ -150,7 +149,7 @@ class CombinedOpenEndedV1Module(object):
             'peer_grade_finished_submissions_when_none_pending', False
         )
 
-        due_date = get_extended_due_date(instance_state)
+        due_date = instance_state.get('due', None)
         grace_period_string = instance_state.get('graceperiod', None)
         try:
             self.timeinfo = TimeInfo(due_date, grace_period_string)

common/lib/xmodule/xmodule/peer_grading_module.py

@@ -11,7 +11,6 @@ from xmodule.fields import Date, Timedelta
 from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
 from xmodule.raw_module import RawDescriptor
 from xmodule.timeinfo import TimeInfo
-from xmodule.util.duedate import get_extended_due_date
 from xmodule.x_module import XModule, module_attr
 from xmodule.open_ended_grading_classes.peer_grading_service import PeerGradingService, MockPeerGradingService
 
@@ -52,14 +51,6 @@ class PeerGradingFields(object):
     due = Date(
         help=_("Due date that should be displayed."),
         scope=Scope.settings)
-    extended_due = Date(
-        help=_("Date that this problem is due by for a particular student. This "
-               "can be set by an instructor, and will override the global due "
-               "date if it is set to a date that is later than the global due "
-               "date."),
-        default=None,
-        scope=Scope.user_state,
-    )
     graceperiod = Timedelta(
         help=_("Amount of grace to give on the due date."),
         scope=Scope.settings
@@ -141,8 +132,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
                 self.linked_problem = self.system.get_module(linked_descriptors[0])
 
         try:
-            self.timeinfo = TimeInfo(
-                get_extended_due_date(self), self.graceperiod)
+            self.timeinfo = TimeInfo(self.due, self.graceperiod)
         except Exception:
             log.error("Error parsing due date information in location {0}".format(self.location))
             raise
@@ -570,7 +560,7 @@ class PeerGradingModule(PeerGradingFields, XModule):
             except (NoPathToItem, ItemNotFoundError):
                 continue
             if descriptor:
-                problem['due'] = get_extended_due_date(descriptor)
+                problem['due'] = descriptor.due
                 grace_period = descriptor.graceperiod
                 try:
                     problem_timeinfo = TimeInfo(problem['due'], grace_period)

common/lib/xmodule/xmodule/seq_module.py

@@ -36,14 +36,6 @@ class SequenceFields(object):
         help=_("Enter the date by which problems are due."),
         scope=Scope.settings,
     )
-    extended_due = Date(
-        help="Date that this problem is due by for a particular student. This "
-             "can be set by an instructor, and will override the global due "
-             "date if it is set to a date that is later than the global due "
-             "date.",
-        default=None,
-        scope=Scope.user_state,
-    )
 
     # Entrance Exam flag -- see cms/contentstore/views/entrance_exam.py for usage
     is_entrance_exam = Boolean(

common/lib/xmodule/xmodule/tests/test_capa_module.py

@@ -430,13 +430,6 @@ class CapaModuleTest(unittest.TestCase):
                                     due=self.yesterday_str)
         self.assertTrue(module.closed())
 
-    def test_due_date_extension(self):
-
-        module = CapaFactory.create(
-            max_attempts="1", attempts="0", due=self.yesterday_str,
-            extended_due=self.tomorrow_str)
-        self.assertFalse(module.closed())
-
     def test_parse_get_params(self):
 
         # Valid GET param dict
@@ -1742,7 +1735,7 @@ class TestProblemCheckTracking(unittest.TestCase):
         self.maxDiff = None
 
     def test_choice_answer_text(self):
-        factory = self.capa_factory_for_problem_xml("""\
+        xml = """\
             <problem display_name="Multiple Choice Questions">
               <p>What color is the open ocean on a sunny day?</p>
               <optionresponse>
@@ -1767,7 +1760,11 @@ class TestProblemCheckTracking(unittest.TestCase):
                 </checkboxgroup>
               </choiceresponse>
             </problem>
-            """)
+            """
+
+        # Whitespace screws up comparisons
+        xml = ''.join(line.strip() for line in xml.split('\n'))
+        factory = self.capa_factory_for_problem_xml(xml)
         module = factory.create()
 
         answer_input_dict = {

conf/locale/eo/LC_MESSAGES/django.po

@@ -1556,19 +1556,6 @@ msgstr "Däté thät thïs prößlém ïs düé ßý Ⱡ'σяєм ι#"
 
 #: common/lib/xmodule/xmodule/capa_base.py
 #: common/lib/xmodule/xmodule/combined_open_ended_module.py
-#: common/lib/xmodule/xmodule/peer_grading_module.py
-msgid ""
-"Date that this problem is due by for a particular student. This can be set "
-"by an instructor, and will override the global due date if it is set to a "
-"date that is later than the global due date."
-msgstr ""
-"Däté thät thïs prößlém ïs düé ßý för ä pärtïçülär stüdént. Thïs çän ßé sét "
-"ßý än ïnstrüçtör, änd wïll övérrïdé thé glößäl düé däté ïf ït ïs sét tö ä "
-"däté thät ïs lätér thän thé glößäl düé däté. Ⱡ'σяєм ιρѕυм ∂σłσя ѕιт αмєт, "
-"¢σηѕє¢тєтυя α∂ιριѕι¢ιηg єłιт, ѕє#"
-
-#: common/lib/xmodule/xmodule/capa_base.py
-#: common/lib/xmodule/xmodule/combined_open_ended_module.py
 msgid "Amount of time after the due date that submissions will be accepted"
 msgstr ""
 "Àmöünt öf tïmé äftér thé düé däté thät süßmïssïöns wïll ßé äççéptéd Ⱡ'σяєм "
@@ -11920,6 +11907,19 @@ msgstr "änd çhöösé ýöür stüdént träçk Ⱡ'σяєм #"
 msgid "and proceed to verification"
 msgstr "änd pröçééd tö vérïfïçätïön Ⱡ'σяєм#"
 
+#. Translators: This line appears next a checkbox which users can leave
+#. checked
+#. or uncheck in order
+#. to indicate whether they want to receive emails from the organization
+#. offering the course.
+#: lms/templates/courseware/mktg_course_about.html
+msgid ""
+"I would like to receive email about other {organization_full_name} programs "
+"and offers."
+msgstr ""
+"Ì wöüld lïké tö réçéïvé émäïl äßöüt öthér {organization_full_name} prögräms "
+"änd öfférs. Ⱡ'σяєм ιρѕυм ∂σłσя #"
+
 #: lms/templates/courseware/mktg_course_about.html
 msgid "Enrollment Is Closed"
 msgstr "Énröllmént Ìs Çlöséd Ⱡ'σя#"

lms/djangoapps/courseware/field_overrides.py

+"""
+This module provides a :class:`~xblock.field_data.FieldData` implementation
+which wraps an other `FieldData` object and provides overrides based on the
+user.  The use of providers allows for overrides that are arbitrarily
+extensible.  One provider is found in `courseware.student_field_overrides`
+which allows for fields to be overridden for individual students.  One can
+envision other providers being written that allow for fields to be overridden
+base on membership of a student in a cohort, or similar.  The use of an
+extensible, modular architecture allows for overrides being done in ways not
+envisioned by the authors.
+
+Currently, this module is used in the `module_render` module in this same
+package and is used to wrap the `authored_data` when constructing an
+`LmsFieldData`.  This means overrides will be in effect for all scopes covered
+by `authored_data`, e.g. course content and settings stored in Mongo.
+"""
+import threading
+
+from abc import ABCMeta, abstractmethod
+from contextlib import contextmanager
+from django.conf import settings
+from xblock.field_data import FieldData
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.inheritance import InheritanceMixin
+
+
+NOTSET = object()
+
+
+def resolve_dotted(name):
+    """
+    Given the dotted name for a Python object, performs any necessary imports
+    and returns the object.
+    """
+    names = name.split('.')
+    path = names.pop(0)
+    target = __import__(path)
+    while names:
+        segment = names.pop(0)
+        path += '.' + segment
+        try:
+            target = getattr(target, segment)
+        except AttributeError:
+            __import__(path)
+            target = getattr(target, segment)
+    return target
+
+
+class OverrideFieldData(FieldData):
+    """
+    A :class:`~xblock.field_data.FieldData` which wraps another `FieldData`
+    object and allows for fields handled by the wrapped `FieldData` to be
+    overriden by arbitrary providers.
+
+    Providers are configured by use of the Django setting,
+    `FIELD_OVERRIDE_PROVIDERS` which should be a tuple of dotted names of
+    :class:`FieldOverrideProvider` concrete implementations.  Note that order
+    is important for this setting.  Override providers will tried in the order
+    configured in the setting.  The first provider to find an override 'wins'
+    for a particular field lookup.
+    """
+    provider_classes = None
+
+    @classmethod
+    def wrap(cls, user, wrapped):
+        """
+        Will return a :class:`OverrideFieldData` which wraps the field data
+        given in `wrapped` for the given `user`, if override providers are
+        configred.  If no override providers are configured, using the Django
+        setting, `FIELD_OVERRIDE_PROVIDERS`, returns `wrapped`, eliminating
+        any performance impact of this feature if no override providers are
+        configured.
+        """
+        if cls.provider_classes is None:
+            cls.provider_classes = tuple(
+                (resolve_dotted(name) for name in
+                 settings.FIELD_OVERRIDE_PROVIDERS))
+
+        if cls.provider_classes:
+            return cls(user, wrapped)
+
+        return wrapped
+
+    def __init__(self, user, fallback):
+        self.fallback = fallback
+        self.providers = tuple((cls(user) for cls in self.provider_classes))
+
+    def get_override(self, block, name):
+        """
+        Checks for an override for the field identified by `name` in `block`.
+        Returns the overridden value or `NOTSET` if no override is found.
+        """
+        if not overrides_disabled():
+            for provider in self.providers:
+                value = provider.get(block, name, NOTSET)
+                if value is not NOTSET:
+                    return value
+        return NOTSET
+
+    def get(self, block, name):
+        value = self.get_override(block, name)
+        if value is not NOTSET:
+            return value
+        return self.fallback.get(block, name)
+
+    def set(self, block, name, value):
+        self.fallback.set(block, name, value)
+
+    def delete(self, block, name):
+        self.fallback.delete(block, name)
+
+    def has(self, block, name):
+        has = self.get_override(block, name)
+        if has is NOTSET:
+            # If this is an inheritable field and an override is set above,
+            # then we want to return False here, so the field_data uses the
+            # override and not the original value for this block.
+            inheritable = InheritanceMixin.fields.keys()
+            if name in inheritable:
+                for ancestor in _lineage(block):
+                    if self.get_override(ancestor, name) is not NOTSET:
+                        return False
+
+        return has is not NOTSET or self.fallback.has(block, name)
+
+    def set_many(self, block, update_dict):
+        return self.fallback.set_many(block, update_dict)
+
+    def default(self, block, name):
+        # The `default` method is overloaded by the field storage system to
+        # also handle inheritance.
+        if not overrides_disabled():
+            inheritable = InheritanceMixin.fields.keys()
+            if name in inheritable:
+                for ancestor in _lineage(block):
+                    value = self.get_override(ancestor, name)
+                    if value is not NOTSET:
+                        return value
+        return self.fallback.default(block, name)
+
+
+class _OverridesDisabled(threading.local):
+    """
+    A thread local used to manage state of overrides being disabled or not.
+    """
+    disabled = ()
+
+
+_OVERRIDES_DISABLED = _OverridesDisabled()
+
+
+@contextmanager
+def disable_overrides():
+    """
+    A context manager which disables field overrides inside the context of a
+    `with` statement, allowing code to get at the `original` value of a field.
+    """
+    prev = _OVERRIDES_DISABLED.disabled
+    _OVERRIDES_DISABLED.disabled += (True,)
+    yield
+    _OVERRIDES_DISABLED.disabled = prev
+
+
+def overrides_disabled():
+    """
+    Checks to see whether overrides are disabled in the current context.
+    Returns a boolean value.  See `disable_overrides`.
+    """
+    return bool(_OVERRIDES_DISABLED.disabled)
+
+
+class FieldOverrideProvider(object):
+    """
+    Abstract class which defines the interface that a `FieldOverrideProvider`
+    must provide.  In general, providers should derive from this class, but
+    it's not strictly necessary as long as they correctly implement this
+    interface.
+
+    A `FieldOverrideProvider` implementation is only responsible for looking up
+    field overrides. To set overrides, there will be a domain specific API for
+    the concrete override implementation being used.
+    """
+    __metaclass__ = ABCMeta
+
+    def __init__(self, user):
+        self.user = user
+
+    @abstractmethod
+    def get(self, block, name, default):  # pragma no cover
+        """
+        Look for an override value for the field named `name` in `block`.
+        Returns the overridden value or `default` if no override is found.
+        """
+        raise NotImplementedError
+
+
+def _lineage(block):
+    """
+    Returns an iterator over all ancestors of the given block, starting with
+    its immediate parent and ending at the root of the block tree.
+    """
+    location = modulestore().get_parent_location(block.location)
+    while location:
+        yield modulestore().get_item(location)
+        location = modulestore().get_parent_location(location)

lms/djangoapps/courseware/grades.py

@@ -20,7 +20,6 @@ from xmodule import graders
 from xmodule.graders import Score
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
-from xmodule.util.duedate import get_extended_due_date
 from .models import StudentModule
 from .module_render import get_module_for_descriptor
 from submissions import api as sub_api  # installed from the edx-submissions repository
@@ -373,7 +372,7 @@ def _progress_summary(student, request, course):
                     'scores': scores,
                     'section_total': section_total,
                     'format': module_format,
-                    'due': get_extended_due_date(section_module),
+                    'due': section_module.due,
                     'graded': graded,
                 })
 

lms/djangoapps/courseware/models.py

@@ -230,3 +230,20 @@ class OfflineComputedGradeLog(models.Model):
 
     def __unicode__(self):
         return "[OCGLog] %s: %s" % (self.course_id.to_deprecated_string(), self.created)  # pylint: disable=no-member
+
+
+class StudentFieldOverride(models.Model):
+    """
+    Holds the value of a specific field overriden for a student.  This is used
+    by the code in the `courseware.student_field_overrides` module to provide
+    overrides of xblock fields on a per user basis.
+    """
+    course_id = CourseKeyField(max_length=255, db_index=True)
+    location = LocationKeyField(max_length=255, db_index=True)
+    student = models.ForeignKey(User, db_index=True)
+
+    class Meta:   # pylint: disable=missing-docstring
+        unique_together = (('course_id', 'location', 'student'),)
+
+    field = models.CharField(max_length=255)
+    value = models.TextField(default='null')

lms/djangoapps/courseware/module_render.py

@@ -54,7 +54,6 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.django import modulestore, ModuleI18nService
 from xmodule.modulestore.exceptions import ItemNotFoundError
-from xmodule.util.duedate import get_extended_due_date
 from xmodule_modifiers import (
     replace_course_urls,
     replace_jump_to_id_urls,
@@ -71,6 +70,8 @@ from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip
 from util import milestones_helpers
 from util.module_utils import yield_dynamic_descriptor_descendents
 
+from .field_overrides import OverrideFieldData
+
 log = logging.getLogger(__name__)
 
 
@@ -170,7 +171,7 @@ def toc_for_course(request, course, active_chapter, active_section, field_data_c
                     sections.append({'display_name': section.display_name_with_default,
                                      'url_name': section.url_name,
                                      'format': section.format if section.format is not None else '',
-                                     'due': get_extended_due_date(section),
+                                     'due': section.due,
                                      'active': active,
                                      'graded': section.graded,
                                      })
@@ -496,11 +497,17 @@ def get_module_system_for_user(user, field_data_cache,
             request_token=request_token
         )
         # rebinds module to a different student.  We'll change system, student_data, and scope_ids
+        authored_data = OverrideFieldData.wrap(
+            real_user, module.descriptor._field_data  # pylint: disable=protected-access
+        )
         module.descriptor.bind_for_student(
             inner_system,
-            LmsFieldData(module.descriptor._field_data, inner_student_data),  # pylint: disable=protected-access
+            LmsFieldData(authored_data, inner_student_data),
             real_user.id,
         )
+        module.descriptor.scope_ids = (
+            module.descriptor.scope_ids._replace(user_id=real_user.id)  # pylint: disable=protected-access
+        )
         module.scope_ids = module.descriptor.scope_ids  # this is needed b/c NamedTuples are immutable
         # now bind the module to the new ModuleSystem instance and vice-versa
         module.runtime = inner_system
@@ -689,7 +696,9 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours
         request_token=request_token
     )
 
-    descriptor.bind_for_student(system, field_data, user.id)  # pylint: disable=protected-access
+    authored_data = OverrideFieldData.wrap(user, descriptor._field_data)  # pylint: disable=protected-access
+    descriptor.bind_for_student(system, LmsFieldData(authored_data, field_data), user.id)
+    descriptor.scope_ids = descriptor.scope_ids._replace(user_id=user.id)  # pylint: disable=protected-access
     return descriptor
 
 

lms/djangoapps/courseware/student_field_overrides.py

+"""
+API related to providing field overrides for individual students.  This is used
+by the individual due dates feature.
+"""
+import json
+
+from .field_overrides import FieldOverrideProvider
+from .models import StudentFieldOverride
+
+
+class IndividualStudentOverrideProvider(FieldOverrideProvider):
+    """
+    A concrete implementation of
+    :class:`~courseware.field_overrides.FieldOverrideProvider` which allows for
+    overrides to be made on a per user basis.
+    """
+    def get(self, block, name, default):
+        return get_override_for_user(self.user, block, name, default)
+
+
+def get_override_for_user(user, block, name, default=None):
+    """
+    Gets the value of the overridden field for the `user`.  `block` and `name`
+    specify the block and the name of the field.  If the field is not
+    overridden for the given user, returns `default`.
+    """
+    try:
+        override = StudentFieldOverride.objects.get(
+            course_id=block.runtime.course_id,
+            location=block.location,
+            student_id=user.id,
+            field=name
+        )
+        field = block.fields[name]
+        return field.from_json(json.loads(override.value))
+    except StudentFieldOverride.DoesNotExist:
+        pass
+    return default
+
+
+def override_field_for_user(user, block, name, value):
+    """
+    Overrides a field for the `user`.  `block` and `name` specify the block
+    and the name of the field on that block to override.  `value` is the
+    value to set for the given field.
+    """
+    override, _ = StudentFieldOverride.objects.get_or_create(
+        course_id=block.runtime.course_id,
+        location=block.location,
+        student_id=user.id,
+        field=name)
+    field = block.fields[name]
+    override.value = json.dumps(field.to_json(value))
+    override.save()
+
+
+def clear_override_for_user(user, block, name):
+    """
+    Clears a previously set field override for the `user`.  `block` and `name`
+    specify the block and the name of the field on that block to clear.
+    This function is idempotent--if no override is set, nothing action is
+    performed.
+    """
+    try:
+        StudentFieldOverride.objects.get(
+            course_id=block.runtime.course_id,
+            student_id=user.id,
+            location=block.location,
+            field=name).delete()
+    except StudentFieldOverride.DoesNotExist:
+        pass

lms/djangoapps/courseware/tests/animport.py

+"""
+A class which never gets imported except for in
+:meth:`~courseware.tests.test_field_overrides.ResolveDottedTests.test_import_something_that_isnt_already_loaded`.
+"""
+SOMENAME = 'bar'

lms/djangoapps/courseware/tests/test_field_overrides.py

+"""
+Tests for `field_overrides` module.
+"""
+import unittest
+
+from django.test import TestCase
+from django.test.utils import override_settings
+from xblock.field_data import DictFieldData
+
+from ..field_overrides import (
+    disable_overrides,
+    FieldOverrideProvider,
+    OverrideFieldData,
+    resolve_dotted,
+)
+
+
+TESTUSER = object()
+
+
+@override_settings(FIELD_OVERRIDE_PROVIDERS=(
+    'courseware.tests.test_field_overrides.TestOverrideProvider',))
+class OverrideFieldDataTests(TestCase):
+    """
+    Tests for `OverrideFieldData`.
+    """
+
+    def setUp(self):
+        OverrideFieldData.provider_classes = None
+
+    def tearDown(self):
+        OverrideFieldData.provider_classes = None
+
+    def make_one(self):
+        """
+        Factory method.
+        """
+        return OverrideFieldData.wrap(TESTUSER, DictFieldData({
+            'foo': 'bar',
+            'bees': 'knees',
+        }))
+
+    def test_get(self):
+        data = self.make_one()
+        self.assertEqual(data.get('block', 'foo'), 'fu')
+        self.assertEqual(data.get('block', 'bees'), 'knees')
+        with disable_overrides():
+            self.assertEqual(data.get('block', 'foo'), 'bar')
+
+    def test_set(self):
+        data = self.make_one()
+        data.set('block', 'foo', 'yowza')
+        self.assertEqual(data.get('block', 'foo'), 'fu')
+        with disable_overrides():
+            self.assertEqual(data.get('block', 'foo'), 'yowza')
+
+    def test_delete(self):
+        data = self.make_one()
+        data.delete('block', 'foo')
+        self.assertEqual(data.get('block', 'foo'), 'fu')
+        with disable_overrides():
+            # Since field_data is responsible for attribute access, you'd
+            # expect it to raise AttributeError. In fact, it raises KeyError,
+            # so we check for that.
+            with self.assertRaises(KeyError):
+                data.get('block', 'foo')
+
+    def test_has(self):
+        data = self.make_one()
+        self.assertTrue(data.has('block', 'foo'))
+        self.assertTrue(data.has('block', 'bees'))
+        self.assertTrue(data.has('block', 'oh'))
+        with disable_overrides():
+            self.assertFalse(data.has('block', 'oh'))
+
+    def test_many(self):
+        data = self.make_one()
+        data.set_many('block', {'foo': 'baz', 'ah': 'ic'})
+        self.assertEqual(data.get('block', 'foo'), 'fu')
+        self.assertEqual(data.get('block', 'ah'), 'ic')
+        with disable_overrides():
+            self.assertEqual(data.get('block', 'foo'), 'baz')
+
+    @override_settings(FIELD_OVERRIDE_PROVIDERS=())
+    def test_no_overrides_configured(self):
+        data = self.make_one()
+        self.assertIsInstance(data, DictFieldData)
+
+
+class ResolveDottedTests(unittest.TestCase):
+    """
+    Tests for `resolve_dotted`.
+    """
+
+    def test_bad_sub_import(self):
+        with self.assertRaises(ImportError):
+            resolve_dotted('courseware.tests.test_foo')
+
+    def test_bad_import(self):
+        with self.assertRaises(ImportError):
+            resolve_dotted('nosuchpackage')
+
+    def test_import_something_that_isnt_already_loaded(self):
+        self.assertEqual(
+            resolve_dotted('courseware.tests.animport.SOMENAME'),
+            'bar'
+        )
+
+
+class TestOverrideProvider(FieldOverrideProvider):
+    """
+    A concrete implementation of `FieldOverrideProvider` for testing.
+    """
+    def get(self, block, name, default):
+        assert self.user is TESTUSER
+        assert block == 'block'
+        if name == 'foo':
+            return 'fu'
+        if name == 'oh':
+            return 'man'
+        return default

lms/djangoapps/instructor/tests/test_api.py

@@ -50,6 +50,9 @@ from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+from xmodule.fields import Date
+
+from courseware.models import StudentFieldOverride
 
 import instructor_task.api
 import instructor.views.api
@@ -61,8 +64,8 @@ from instructor_task.api_helper import AlreadyRunningError
 from openedx.core.djangoapps.course_groups.cohorts import set_course_cohort_settings
 
 from .test_tools import msk_from_problem_urlname
-from ..views.tools import get_extended_due
 
+DATE_FIELD = Date()
 EXPECTED_CSV_HEADER = (
     '"code","redeem_code_url","course_id","company_name","created_by","redeemed_by","invoice_id","purchaser",'
     '"customer_reference_number","internal_reference"'
@@ -3114,6 +3117,24 @@ class TestInstructorAPIHelpers(TestCase):
         msk_from_problem_urlname(*args)
 
 
+def get_extended_due(course, unit, user):
+    """
+    Gets the overridden due date for the given user on the given unit.  Returns
+    `None` if there is no override set.
+    """
+    try:
+        override = StudentFieldOverride.objects.get(
+            course_id=course.id,
+            student=user,
+            location=unit.location,
+            field='due'
+        )
+        return DATE_FIELD.from_json(json.loads(override.value))
+    except StudentFieldOverride.DoesNotExist:
+        return None
+
+
+@override_settings(MODULESTORE=TEST_DATA_MOCK_MODULESTORE)
 class TestDueDateExtensions(ModuleStoreTestCase, LoginEnrollmentTestCase):
     """
     Test data dumps for reporting.

lms/djangoapps/instructor/tests/test_tools.py

@@ -3,14 +3,15 @@ Tests for views/tools.py.
 """
 
 import datetime
-import functools
 import mock
 import json
 import unittest
 
 from django.utils.timezone import utc
+from django.test.utils import override_settings
 
 from courseware.models import StudentModule
+from courseware.field_overrides import OverrideFieldData
 from student.tests.factories import UserFactory
 from xmodule.fields import Date
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
@@ -180,6 +181,10 @@ class TestTitleOrUrl(unittest.TestCase):
         self.assertEquals(tools.title_or_url(unit), 'test:hello')
 
 
+@override_settings(
+    FIELD_OVERRIDE_PROVIDERS=(
+        'courseware.student_field_overrides.IndividualStudentOverrideProvider',),
+)
 class TestSetDueDateExtension(ModuleStoreTestCase):
     """
     Test the set_due_date_extensions function.
@@ -189,53 +194,53 @@ class TestSetDueDateExtension(ModuleStoreTestCase):
         Fixtures.
         """
         super(TestSetDueDateExtension, self).setUp()
+        OverrideFieldData.provider_classes = None
 
-        due = datetime.datetime(2010, 5, 12, 2, 42, tzinfo=utc)
+        self.due = due = datetime.datetime(2010, 5, 12, 2, 42, tzinfo=utc)
         course = CourseFactory.create()
         week1 = ItemFactory.create(due=due, parent=course)
         week2 = ItemFactory.create(due=due, parent=course)
         week3 = ItemFactory.create(parent=course)
-
-        homework = ItemFactory.create(
-            parent=week1,
-            due=due
-        )
+        homework = ItemFactory.create(parent=week1)
+        assignment = ItemFactory.create(parent=homework, due=due)
 
         user = UserFactory.create()
-        StudentModule(
-            state='{}',
-            student_id=user.id,
-            course_id=course.id,
-            module_state_key=week1.location).save()
-        StudentModule(
-            state='{}',
-            student_id=user.id,
-            course_id=course.id,
-            module_state_key=homework.location).save()
 
         self.course = course
         self.week1 = week1
         self.homework = homework
+        self.assignment = assignment
         self.week2 = week2
         self.week3 = week3
         self.user = user
 
-        self.extended_due = functools.partial(
-            tools.get_extended_due, course, student=user)
+        # Apparently the test harness doesn't use LmsFieldStorage, and I'm not
+        # sure if there's a way to poke the test harness to do so.  So, we'll
+        # just inject the override field storage in this brute force manner.
+        for block in (course, week1, week2, week3, homework, assignment):
+            block._field_data = OverrideFieldData.wrap(  # pylint: disable=protected-access
+                user, block._field_data)  # pylint: disable=protected-access
+
+    def tearDown(self):
+        OverrideFieldData.provider_classes = None
+
+    def _clear_field_data_cache(self):
+        """
+        Clear field data cache for xblocks under test. Normally this would be
+        done by virtue of the fact that xblocks are reloaded on subsequent
+        requests.
+        """
+        for block in (self.week1, self.week2, self.week3,
+                      self.homework, self.assignment):
+            block.fields['due']._del_cached_value(block)  # pylint: disable=protected-access
 
     def test_set_due_date_extension(self):
         extended = datetime.datetime(2013, 12, 25, 0, 0, tzinfo=utc)
         tools.set_due_date_extension(self.course, self.week1, self.user, extended)
-        self.assertEqual(self.extended_due(self.week1), extended)
-        self.assertEqual(self.extended_due(self.homework), extended)
-
-    def test_set_due_date_extension_create_studentmodule(self):
-        extended = datetime.datetime(2013, 12, 25, 0, 0, tzinfo=utc)
-        user = UserFactory.create()  # No student modules for this user
-        tools.set_due_date_extension(self.course, self.week1, user, extended)
-        extended_due = functools.partial(tools.get_extended_due, self.course, student=user)
-        self.assertEqual(extended_due(self.week1), extended)
-        self.assertEqual(extended_due(self.homework), extended)
+        self._clear_field_data_cache()
+        self.assertEqual(self.week1.due, extended)
+        self.assertEqual(self.homework.due, extended)
+        self.assertEqual(self.assignment.due, extended)
 
     def test_set_due_date_extension_invalid_date(self):
         extended = datetime.datetime(2009, 1, 1, 0, 0, tzinfo=utc)
@@ -251,8 +256,7 @@ class TestSetDueDateExtension(ModuleStoreTestCase):
         extended = datetime.datetime(2013, 12, 25, 0, 0, tzinfo=utc)
         tools.set_due_date_extension(self.course, self.week1, self.user, extended)
         tools.set_due_date_extension(self.course, self.week1, self.user, None)
-        self.assertEqual(self.extended_due(self.week1), None)
-        self.assertEqual(self.extended_due(self.homework), None)
+        self.assertEqual(self.week1.due, self.due)
 
 
 class TestDataDumps(ModuleStoreTestCase):
@@ -278,51 +282,7 @@ class TestDataDumps(ModuleStoreTestCase):
         )
 
         user1 = UserFactory.create()
-        StudentModule(
-            state='{}',
-            student_id=user1.id,
-            course_id=course.id,
-            module_state_key=week1.location).save()
-        StudentModule(
-            state='{}',
-            student_id=user1.id,
-            course_id=course.id,
-            module_state_key=week2.location).save()
-        StudentModule(
-            state='{}',
-            student_id=user1.id,
-            course_id=course.id,
-            module_state_key=week3.location).save()
-        StudentModule(
-            state='{}',
-            student_id=user1.id,
-            course_id=course.id,
-            module_state_key=homework.location).save()
-
         user2 = UserFactory.create()
-        StudentModule(
-            state='{}',
-            student_id=user2.id,
-            course_id=course.id,
-            module_state_key=week1.location).save()
-        StudentModule(
-            state='{}',
-            student_id=user2.id,
-            course_id=course.id,
-            module_state_key=homework.location).save()
-
-        user3 = UserFactory.create()
-        StudentModule(
-            state='{}',
-            student_id=user3.id,
-            course_id=course.id,
-            module_state_key=week1.location).save()
-        StudentModule(
-            state='{}',
-            student_id=user3.id,
-            course_id=course.id,
-            module_state_key=homework.location).save()
-
         self.course = course
         self.week1 = week1
         self.homework = homework

lms/djangoapps/instructor/views/tools.py

@@ -10,7 +10,13 @@ from django.http import HttpResponseBadRequest
 from django.utils.timezone import utc
 from django.utils.translation import ugettext as _
 
-from courseware.models import StudentModule
+from courseware.models import StudentFieldOverride
+from courseware.field_overrides import disable_overrides
+from courseware.student_field_overrides import (
+    clear_override_for_user,
+    get_override_for_user,
+    override_field_for_user,
+)
 from xmodule.fields import Date
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.django import modulestore
@@ -175,22 +181,6 @@ def title_or_url(node):
     return title
 
 
-def get_extended_due(course, unit, student):
-    """
-    Get the extended due date out of a student's state for a particular unit.
-    """
-    student_module = StudentModule.objects.get(
-        student_id=student.id,
-        course_id=course.id,
-        module_state_key=unit.location
-    )
-
-    state = json.loads(student_module.state)
-    extended = state.get('extended_due', None)
-    if extended:
-        return DATE_FIELD.from_json(extended)
-
-
 def set_due_date_extension(course, unit, student, due_date):
     """
     Sets a due date extension. Raises DashboardError if the unit or extended
@@ -198,56 +188,22 @@ def set_due_date_extension(course, unit, student, due_date):
     """
     if due_date:
         # Check that the new due date is valid:
-        original_due_date = getattr(unit, 'due', None)
+        with disable_overrides():
+            original_due_date = getattr(unit, 'due', None)
 
         if not original_due_date:
             raise DashboardError(_("Unit {0} has no due date to extend.").format(unit.location))
         if due_date < original_due_date:
             raise DashboardError(_("An extended due date must be later than the original due date."))
+
+        override_field_for_user(student, unit, 'due', due_date)
+
     else:
         # We are deleting a due date extension. Check that it exists:
-        if not get_extended_due(course, unit, student):
+        if not get_override_for_user(student, unit, 'due'):
             raise DashboardError(_("No due date extension is set for that student and unit."))
 
-    def set_due_date(node):
-        """
-        Recursively set the due date on a node and all of its children.
-        """
-        try:
-            student_module = StudentModule.objects.get(
-                student_id=student.id,
-                course_id=course.id,
-                module_state_key=node.location
-            )
-            state = json.loads(student_module.state)
-
-        except StudentModule.DoesNotExist:
-            # Normally, a StudentModule is created as a side effect of assigning
-            # a value to a property in an XModule or XBlock which has a scope
-            # of 'Scope.user_state'.  Here, we want to alter user state but
-            # can't use the standard XModule/XBlock machinery to do so, because
-            # it fails to take into account that the state being altered might
-            # belong to a student other than the one currently logged in.  As a
-            # result, in our work around, we need to detect whether the
-            # StudentModule has been created for the given student on the given
-            # unit and create it if it is missing, so we can use it to store
-            # the extended due date.
-            student_module = StudentModule.objects.create(
-                student_id=student.id,
-                course_id=course.id,
-                module_state_key=node.location,
-                module_type=node.category
-            )
-            state = {}
-
-        state['extended_due'] = DATE_FIELD.to_json(due_date)
-        student_module.state = json.dumps(state)
-        student_module.save()
-
-        for child in node.get_children():
-            set_due_date(child)
-
-    set_due_date(unit)
+        clear_override_for_user(student, unit, 'due')
 
 
 def dump_module_extensions(course, unit):
@@ -257,20 +213,17 @@ def dump_module_extensions(course, unit):
     """
     data = []
     header = [_("Username"), _("Full Name"), _("Extended Due Date")]
-    query = StudentModule.objects.filter(
+    query = StudentFieldOverride.objects.filter(
         course_id=course.id,
-        module_state_key=unit.location)
-    for module in query:
-        state = json.loads(module.state)
-        extended_due = state.get("extended_due")
-        if not extended_due:
-            continue
-        extended_due = DATE_FIELD.from_json(extended_due)
-        extended_due = extended_due.strftime("%Y-%m-%d %H:%M")
-        fullname = module.student.profile.name
+        location=unit.location,
+        field='due')
+    for override in query:
+        due = DATE_FIELD.from_json(json.loads(override.value))
+        due = due.strftime("%Y-%m-%d %H:%M")
+        fullname = override.student.profile.name
         data.append(dict(zip(
             header,
-            (module.student.username, fullname, extended_due))))
+            (override.student.username, fullname, due))))
     data.sort(key=lambda x: x[header[0]])
     return {
         "header": header,
@@ -288,23 +241,19 @@ def dump_student_extensions(course, student):
     data = []
     header = [_("Unit"), _("Extended Due Date")]
     units = get_units_with_due_date(course)
-    units = dict([(u.location, u) for u in units])
-    query = StudentModule.objects.filter(
+    units = {u.location: u for u in units}
+    query = StudentFieldOverride.objects.filter(
         course_id=course.id,
-        student_id=student.id)
-    for module in query:
-        state = json.loads(module.state)
-        # temporary hack: module_state_key is missing the run but units are not. fix module_state_key
-        module_loc = module.module_state_key.map_into_course(module.course_id)
-        if module_loc not in units:
-            continue
-        extended_due = state.get("extended_due")
-        if not extended_due:
+        student=student,
+        field='due')
+    for override in query:
+        location = override.location.replace(course_key=course.id)
+        if location not in units:
             continue
-        extended_due = DATE_FIELD.from_json(extended_due)
-        extended_due = extended_due.strftime("%Y-%m-%d %H:%M")
-        title = title_or_url(units[module_loc])
-        data.append(dict(zip(header, (title, extended_due))))
+        due = DATE_FIELD.from_json(json.loads(override.value))
+        due = due.strftime("%Y-%m-%d %H:%M")
+        title = title_or_url(units[location])
+        data.append(dict(zip(header, (title, due))))
     return {
         "header": header,
         "title": _("Due date extensions for {0} {1} ({2})").format(

lms/envs/aws.py

@@ -343,6 +343,10 @@ if FEATURES.get('ENABLE_CORS_HEADERS') or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF
     CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = ENV_TOKENS.get('CROSS_DOMAIN_CSRF_COOKIE_DOMAIN')
 
 
+# Field overrides.  To use the IDDE feature, add
+# 'courseware.student_field_overrides.IndividualStudentOverrideProvider'.
+FIELD_OVERRIDE_PROVIDERS = tuple(ENV_TOKENS.get('FIELD_OVERRIDE_PROVIDERS', []))
+
 ############################## SECURE AUTH ITEMS ###############
 # Secret things: passwords, access keys, etc.
 

lms/envs/common.py

@@ -208,6 +208,10 @@ FEATURES = {
     'ENABLE_INSTRUCTOR_BACKGROUND_TASKS': True,
 
     # Enable instructor to assign individual due dates
+    # Note: In order for this feature to work, you must also add
+    # 'courseware.student_field_overrides.IndividualStudentOverrideProvider' to
+    # the setting FIELD_OVERRIDE_PROVIDERS, in addition to setting this flag to
+    # True.
     'INDIVIDUAL_DUE_DATES': False,
 
     # Enable legacy instructor dashboard
@@ -2225,3 +2229,9 @@ ECOMMERCE_API_TIMEOUT = 5
 
 # Reverification checkpoint name pattern
 CHECKPOINT_PATTERN = r'(?P<checkpoint_name>\w+)'
+
+# For the fields override feature
+# If using FEATURES['INDIVIDUAL_DUE_DATES'], you should add
+# 'courseware.student_field_overrides.IndividualStudentOverrideProvider' to
+# this setting.
+FIELD_OVERRIDE_PROVIDERS = ()