Merge remote-tracking branch 'origin/master' into arjun/javascript_response

Conflicts:
	common/lib/xmodule/xmodule/js/src/capa/display.coffee
	common/lib/xmodule/xmodule/tests/__init__.py
	common/lib/xmodule/xmodule/x_module.py
	lms/djangoapps/courseware/module_render.py

askbot @ e56ae380

-Subproject commit 1c3381046c78e055439ba1c78e0df48410fcc13e
+Subproject commit e56ae380846f7c6cdaeacfc58880fab103540491

cms/djangoapps/contentstore/views.py

@@ -249,7 +249,7 @@ def load_preview_module(request, preview_id, descriptor, instance_state, shared_
     module = descriptor.xmodule_constructor(system)(instance_state, shared_state)
     module.get_html = replace_static_urls(
         wrap_xmodule(module.get_html, module, "xmodule_display.html"),
-        module.metadata['data_dir']
+        module.metadata['data_dir'], module
     )
     save_preview_state(request, preview_id, descriptor.location.url(),
         module.get_instance_state(), module.get_shared_state())

cms/envs/common.py

@@ -83,7 +83,7 @@ TEMPLATE_CONTEXT_PROCESSORS = (
     'django.core.context_processors.request',
     'django.core.context_processors.static',
     'django.contrib.messages.context_processors.messages',
-    'django.core.context_processors.auth',  # this is required for admin
+    'django.contrib.auth.context_processors.auth',  # this is required for admin
     'django.core.context_processors.csrf',  # necessary for csrf protection
 )
 
@@ -121,6 +121,7 @@ MIDDLEWARE_CLASSES = (
 )
 
 ############################ SIGNAL HANDLERS ################################
+# This is imported to register the exception signal handling that logs exceptions
 import monitoring.exceptions  # noqa
 
 ############################ DJANGO_BUILTINS ################################

cms/envs/test.py

@@ -55,6 +55,17 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': ENV_ROOT / "db" / "cms.db",
+    },
+
+    # The following are for testing purposes...
+    'edX/toy/2012_Fall': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "course1.db",
+    },
+    
+    'edx/full/6.002_Spring_2012': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "course2.db",
     }
 }
 

cms/static/coffee/spec/main_spec.coffee

@@ -2,7 +2,7 @@ describe "CMS", ->
   beforeEach ->
     CMS.unbind()
 
-  it "should iniitalize Models", ->
+  it "should initialize Models", ->
     expect(CMS.Models).toBeDefined()
 
   it "should initialize Views", ->

cms/static/coffee/spec/models/module_spec.coffee

@@ -11,14 +11,25 @@ describe "CMS.Models.Module", ->
         @fakeModule = jasmine.createSpy("fakeModuleObject")
         window.FakeModule = jasmine.createSpy("FakeModule").andReturn(@fakeModule)
         @module = new CMS.Models.Module(type: "FakeModule")
-        @stubElement = $("<div>")
-        @module.loadModule(@stubElement)
+        @stubDiv = $('<div />')
+        @stubElement = $('<div class="xmodule_edit" />')
+        @stubElement.data('type', "FakeModule")
+
+        @stubDiv.append(@stubElement)
+        @module.loadModule(@stubDiv)
 
       afterEach ->
         window.FakeModule = undefined
 
       it "initialize the module", ->
-        expect(window.FakeModule).toHaveBeenCalledWith(@stubElement)
+        expect(window.FakeModule).toHaveBeenCalled()
+        # Need to compare underlying nodes, because jquery selectors
+        # aren't equal even when they point to the same node.
+        # http://stackoverflow.com/questions/9505437/how-to-test-jquery-with-jasmine-for-element-id-if-used-as-this
+        expectedNode = @stubElement[0]
+        actualNode = window.FakeModule.mostRecentCall.args[0][0]
+
+        expect(actualNode).toEqual(expectedNode)
         expect(@module.module).toEqual(@fakeModule)
 
     describe "when the module does not exists", ->
@@ -32,7 +43,8 @@ describe "CMS.Models.Module", ->
         window.console = @previousConsole
 
       it "print out error to log", ->
-        expect(window.console.error).toHaveBeenCalledWith("Unable to load HTML.")
+        expect(window.console.error).toHaveBeenCalled()
+        expect(window.console.error.mostRecentCall.args[0]).toMatch("^Unable to load")
 
 
   describe "editUrl", ->

cms/static/coffee/spec/views/module_edit_spec.coffee

@@ -8,11 +8,11 @@ describe "CMS.Views.ModuleEdit", ->
         <a href="#" class="cancel">cancel</a>
         <ol>
           <li>
-            <a href="#" class="module-edit" data-id="i4x://mitx.edu/course/module" data-type="html">submodule</a>
+            <a href="#" class="module-edit" data-id="i4x://mitx/course/html/module" data-type="html">submodule</a>
           </li>
         </ol>
       </div>
-      """
+      """ #"
     CMS.unbind()
 
   describe "defaults", ->
@@ -27,7 +27,7 @@ describe "CMS.Views.ModuleEdit", ->
       @stubModule.editUrl.andReturn("/edit_item?id=stub_module")
       new CMS.Views.ModuleEdit(el: $("#module-edit"), model: @stubModule)
 
-    it "load the edit from via ajax and pass to the model", ->
+    it "load the edit via ajax and pass to the model", ->
       expect($.fn.load).toHaveBeenCalledWith("/edit_item?id=stub_module", jasmine.any(Function))
       if $.fn.load.mostRecentCall
         $.fn.load.mostRecentCall.args[1]()
@@ -37,9 +37,9 @@ describe "CMS.Views.ModuleEdit", ->
     beforeEach ->
       @stubJqXHR = jasmine.createSpy("stubJqXHR")
       @stubJqXHR.success = jasmine.createSpy("stubJqXHR.success").andReturn(@stubJqXHR)
-      @stubJqXHR.error= jasmine.createSpy("stubJqXHR.success").andReturn(@stubJqXHR)
+      @stubJqXHR.error = jasmine.createSpy("stubJqXHR.error").andReturn(@stubJqXHR)
       @stubModule.save = jasmine.createSpy("stubModule.save").andReturn(@stubJqXHR)
-      new CMS.Views.ModuleEdit(el: $("#module-edit"), model: @stubModule)
+      new CMS.Views.ModuleEdit(el: $(".module-edit"), model: @stubModule)
       spyOn(window, "alert")
       $(".save-update").click()
 
@@ -77,5 +77,5 @@ describe "CMS.Views.ModuleEdit", ->
       expect(CMS.pushView).toHaveBeenCalledWith @view
       expect(CMS.Views.ModuleEdit).toHaveBeenCalledWith model: @model
       expect(CMS.Models.Module).toHaveBeenCalledWith
-        id: "i4x://mitx.edu/course/module"
+        id: "i4x://mitx/course/html/module"
         type: "html"

cms/static/coffee/spec/views/module_spec.coffee

 describe "CMS.Views.Module", ->
   beforeEach ->
     setFixtures """
-      <div id="module" data-id="i4x://mitx.edu/course/module" data-type="html">
+      <div id="module" data-id="i4x://mitx/course/html/module" data-type="html">
         <a href="#" class="module-edit">edit</a>
       </div>
       """
@@ -20,5 +20,5 @@ describe "CMS.Views.Module", ->
       expect(CMS.replaceView).toHaveBeenCalledWith @view
       expect(CMS.Views.ModuleEdit).toHaveBeenCalledWith model: @model
       expect(CMS.Models.Module).toHaveBeenCalledWith
-        id: "i4x://mitx.edu/course/module"
+        id: "i4x://mitx/course/html/module"
         type: "html"

cms/static/coffee/spec/views/week_spec.coffee

 describe "CMS.Views.Week", ->
   beforeEach ->
     setFixtures """
-      <div id="week" data-id="i4x://mitx.edu/course/week">
+      <div id="week" data-id="i4x://mitx/course/chapter/week">
         <div class="editable"></div>
         <textarea class="editable-textarea"></textarea>
         <a href="#" class="week-edit" >edit</a>

cms/static/coffee/src/models/module.coffee

@@ -4,7 +4,8 @@ class CMS.Models.Module extends Backbone.Model
     data: ''
 
   loadModule: (element) ->
-    @module = XModule.loadModule($(element).find('.xmodule_edit'))
+    elt = $(element).find('.xmodule_edit').first()
+    @module = XModule.loadModule(elt)
 
   editUrl: ->
     "/edit_item?#{$.param(id: @get('id'))}"

cms/static/coffee/src/views/module_edit.coffee

@@ -13,6 +13,16 @@ class CMS.Views.ModuleEdit extends Backbone.View
 
       # Load preview modules
       XModule.loadModules('display')
+      @enableDrag()
+
+  enableDrag: ->
+      # Enable dragging things in the #sortable div (if there is one)
+      if $("#sortable").length > 0
+        $("#sortable").sortable({
+          placeholder: "ui-state-highlight"
+          })
+        $("#sortable").disableSelection();
+
 
   save: (event) ->
     event.preventDefault()
@@ -32,6 +42,7 @@ class CMS.Views.ModuleEdit extends Backbone.View
   cancel: (event) ->
     event.preventDefault()
     CMS.popView()
+    @enableDrag()
 
   editSubmodule: (event) ->
     event.preventDefault()
@@ -42,3 +53,4 @@ class CMS.Views.ModuleEdit extends Backbone.View
             id: $(event.target).data('id')
             type: if moduleType == 'None' then null else moduleType
             previewType: if previewType == 'None' then null else previewType
+    @enableDrag()

cms/templates/widgets/sequence-edit.html

@@ -33,7 +33,7 @@
     <section class="modules">
       <ol>
         <li>
-          <ol>
+          <ol id="sortable">
             % for child in module.get_children():
             <li class="${module.category}">
               <a href="#" class="module-edit"

cms/urls.py

 from django.conf import settings
-from django.conf.urls.defaults import patterns, include, url
+from django.conf.urls import patterns, include, url
 
 import django.contrib.auth.views
 

common/djangoapps/pipeline_mako/__init__.py

-from staticfiles.storage import staticfiles_storage
-
 from mitxmako.shortcuts import render_to_string
 
 from pipeline.conf import settings
 from pipeline.packager import Packager
 from pipeline.utils import guess_type
+from static_replace import try_staticfiles_lookup
 
 
 def compressed_css(package_name):
@@ -25,9 +24,11 @@ def compressed_css(package_name):
 def render_css(package, path):
     template_name = package.template_name or "mako/css.html"
     context = package.extra_context
+
+    url = try_staticfiles_lookup(path)
     context.update({
         'type': guess_type(path, 'text/css'),
-        'url': staticfiles_storage.url(path)
+        'url': url,
     })
     return render_to_string(template_name, context)
 
@@ -58,7 +59,7 @@ def render_js(package, path):
     context = package.extra_context
     context.update({
         'type': guess_type(path, 'text/javascript'),
-        'url': staticfiles_storage.url(path)
+        'url': try_staticfiles_lookup(path)
     })
     return render_to_string(template_name, context)
 

common/djangoapps/pipeline_mako/templates/static_content.html

@@ -3,7 +3,13 @@ from staticfiles.storage import staticfiles_storage
 from pipeline_mako import compressed_css, compressed_js
 %>
 
-<%def name='url(file)'>${staticfiles_storage.url(file)}</%def>
+<%def name='url(file)'>
+<%
+try:
+    url = staticfiles_storage.url(file)
+except:
+    url = file
+%>${url}</%def>
 
 <%def name='css(group)'>
   % if settings.MITX_FEATURES['USE_DJANGO_PIPELINE']:

common/djangoapps/static_replace.py

-from staticfiles.storage import staticfiles_storage
+import logging
 import re
 
+from staticfiles.storage import staticfiles_storage
+from staticfiles import finders
+from django.conf import settings
+
+log = logging.getLogger(__name__)
+
+def try_staticfiles_lookup(path):
+    """
+    Try to lookup a path in staticfiles_storage.  If it fails, return
+    a dead link instead of raising an exception.
+    """
+    try:
+        url = staticfiles_storage.url(path)
+    except Exception as err:
+        log.warning("staticfiles_storage couldn't find path {}: {}".format(
+            path, str(err)))
+        # Just return a dead link--don't kill everything.
+        url = "file_not_found"
+    return url
+
 
 def replace(static_url, prefix=None):
     if prefix is None:
@@ -9,10 +29,19 @@ def replace(static_url, prefix=None):
         prefix = prefix + '/'
 
     quote = static_url.group('quote')
-    if staticfiles_storage.exists(static_url.group('rest')):
+
+    servable = (
+        # If in debug mode, we'll serve up anything that the finders can find
+        (settings.DEBUG and finders.find(static_url.group('rest'), True)) or
+        # Otherwise, we'll only serve up stuff that the storages can find
+        staticfiles_storage.exists(static_url.group('rest'))
+    )
+
+    if servable:
         return static_url.group(0)
     else:
-        url = staticfiles_storage.url(prefix + static_url.group('rest'))
+        # don't error if file can't be found
+        url = try_staticfiles_lookup(prefix + static_url.group('rest'))
         return "".join([quote, url, quote])
 
 

common/djangoapps/student/management/commands/create_random_users.py

+##
+## A script to create some dummy users
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User
+from student.models import UserProfile, CourseEnrollment
+
+from student.views import _do_create_account, get_random_post_override
+
+def create(n, course_id):
+    """Create n users, enrolling them in course_id if it's not None"""
+    for i in range(n):
+        (user, user_profile, _) = _do_create_account(get_random_post_override())
+        if course_id is not None:
+            CourseEnrollment.objects.create(user=user, course_id=course_id)
+
+class Command(BaseCommand):
+    help = """Create N new users, with random parameters.
+
+Usage: create_random_users.py N [course_id_to_enroll_in].
+
+Examples:
+  create_random_users.py 1
+  create_random_users.py 10 MITx/6.002x/2012_Fall
+  create_random_users.py 100 HarvardX/CS50x/2012
+"""
+
+    def handle(self, *args, **options):
+        if len(args) < 1 or len(args) > 2:
+            print Command.help
+            return
+
+        n = int(args[0])
+        course_id = args[1] if len(args) == 2 else None
+        create(n, course_id)

common/djangoapps/student/models.py

 """
-WE'RE USING MIGRATIONS!
+Models for Student Information
+
+Replication Notes
+
+In our live deployment, we intend to run in a scenario where there is a pool of
+Portal servers that hold the canoncial user information and that user 
+information is replicated to slave Course server pools. Each Course has a set of
+servers that serves only its content and has users that are relevant only to it.
+
+We replicate the following tables into the Course DBs where the user is 
+enrolled. Only the Portal servers should ever write to these models.
+* UserProfile
+* CourseEnrollment
+
+We do a partial replication of:
+* User -- Askbot extends this and uses the extra fields, so we replicate only
+          the stuff that comes with basic django_auth and ignore the rest.)
+
+There are a couple different scenarios:
+
+1. There's an update of User or UserProfile -- replicate it to all Course DBs
+   that the user is enrolled in (found via CourseEnrollment).
+2. There's a change in CourseEnrollment. We need to push copies of UserProfile,
+   CourseEnrollment, and the base fields in User
+
+Migration Notes
 
 If you make changes to this model, be sure to create an appropriate migration
 file and check it in at the same time as your model changes. To do that,
@@ -10,16 +35,41 @@ file and check it in at the same time as your model changes. To do that,
 """
 from datetime import datetime
 import json
+import logging
 import uuid
 
-from django.db import models
+from django.conf import settings
 from django.contrib.auth.models import User
+from django.db import models
+from django.db.models.signals import post_delete, post_save
+from django.dispatch import receiver
 from django_countries import CountryField
 
+from xmodule.modulestore.django import modulestore
+
 #from cache_toolbox import cache_model, cache_relation
 
+log = logging.getLogger(__name__)
 
 class UserProfile(models.Model):
+    """This is where we store all the user demographic fields. We have a 
+    separate table for this rather than extending the built-in Django auth_user.
+
+    Notes:
+        * Some fields are legacy ones from the first run of 6.002, from which 
+          we imported many users.
+        * Fields like name and address are intentionally open ended, to account
+          for international variations. An unfortunate side-effect is that we
+          cannot efficiently sort on last names for instance.
+
+    Replication:
+        * Only the Portal servers should ever modify this information.
+        * All fields are replicated into relevant Course databases
+
+    Some of the fields are legacy ones that were captured during the initial
+    MITx fall prototype.
+    """
+
     class Meta:
         db_table = "auth_userprofile"
 
@@ -203,3 +253,154 @@ def add_user_to_default_group(user, group):
         utg.save()
     utg.users.add(User.objects.get(username=user))
     utg.save()
+
+########################## REPLICATION SIGNALS #################################
+@receiver(post_save, sender=User)
+def replicate_user_save(sender, **kwargs):
+     user_obj = kwargs['instance']
+     return replicate_model(User.save, user_obj, user_obj.id)
+
+@receiver(post_save, sender=CourseEnrollment)
+def replicate_enrollment_save(sender, **kwargs):
+    """This is called when a Student enrolls in a course. It has to do the 
+    following:
+
+    1. Make sure the User is copied into the Course DB. It may already exist 
+       (someone deleting and re-adding a course). This has to happen first or
+       the foreign key constraint breaks.
+    2. Replicate the CourseEnrollment.
+    3. Replicate the UserProfile.
+    """
+    if not is_portal():
+        return
+
+    enrollment_obj = kwargs['instance']
+    log.debug("Replicating user because of new enrollment")
+    replicate_user(enrollment_obj.user, enrollment_obj.course_id)
+
+    log.debug("Replicating enrollment because of new enrollment")
+    replicate_model(CourseEnrollment.save, enrollment_obj, enrollment_obj.user_id)
+
+    log.debug("Replicating user profile because of new enrollment")
+    user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
+    replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
+ 
+@receiver(post_delete, sender=CourseEnrollment)
+def replicate_enrollment_delete(sender, **kwargs):
+     enrollment_obj = kwargs['instance']
+     return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
+ 
+@receiver(post_save, sender=UserProfile)
+def replicate_userprofile_save(sender, **kwargs):
+    """We just updated the UserProfile (say an update to the name), so push that
+    change to all Course DBs that we're enrolled in."""
+    user_profile_obj = kwargs['instance']
+    return replicate_model(UserProfile.save, user_profile_obj, user_profile_obj.user_id)
+
+ 
+######### Replication functions #########
+USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
+                       "password", "is_staff", "is_active", "is_superuser",
+                       "last_login", "date_joined"]
+
+def replicate_user(portal_user, course_db_name):
+    """Replicate a User to the correct Course DB. This is more complicated than
+    it should be because Askbot extends the auth_user table and adds its own 
+    fields. So we need to only push changes to the standard fields and leave
+    the rest alone so that Askbot changes at the Course DB level don't get 
+    overridden.
+    """
+    try:
+        # If the user exists in the Course DB, update the appropriate fields and
+        # save it back out to the Course DB.
+        course_user = User.objects.using(course_db_name).get(id=portal_user.id)
+        for field in USER_FIELDS_TO_COPY:
+            setattr(course_user, field, getattr(portal_user, field))
+
+        mark_handled(course_user)
+        log.debug("User {0} found in Course DB, replicating fields to {1}"
+                  .format(course_user, course_db_name))
+        course_user.save(using=course_db_name) # Just being explicit.
+
+    except User.DoesNotExist:
+        # Otherwise, just make a straight copy to the Course DB.
+        mark_handled(portal_user)
+        log.debug("User {0} not found in Course DB, creating copy in {1}"
+                  .format(portal_user, course_db_name))
+        portal_user.save(using=course_db_name)
+
+def replicate_model(model_method, instance, user_id):
+    """
+    model_method is the model action that we want replicated. For instance, 
+                 UserProfile.save
+    """
+    if not should_replicate(instance):
+        return
+
+    mark_handled(instance)
+    course_db_names = db_names_to_replicate_to(user_id)
+    log.debug("Replicating {0} for user {1} to DBs: {2}"
+              .format(model_method, user_id, course_db_names))
+
+    for db_name in course_db_names:
+        model_method(instance, using=db_name)
+
+######### Replication Helpers #########
+
+def is_valid_course_id(course_id):
+    """Right now, the only database that's not a course database is 'default'.
+    I had nicer checking in here originally -- it would scan the courses that
+    were in the system and only let you choose that. But it was annoying to run
+    tests with, since we don't have course data for some for our course test 
+    databases. Hence the lazy version.
+    """
+    return course_id != 'default'
+
+def is_portal():
+    """Are we in the portal pool? Only Portal servers are allowed to replicate
+    their changes. For now, only Portal servers see multiple DBs, so we use
+    that to decide."""
+    return len(settings.DATABASES) > 1
+
+def db_names_to_replicate_to(user_id):
+    """Return a list of DB names that this user_id is enrolled in."""
+    return [c.course_id
+            for c in CourseEnrollment.objects.filter(user_id=user_id)
+            if is_valid_course_id(c.course_id)]
+
+def marked_handled(instance):
+    """Have we marked this instance as being handled to avoid infinite loops
+    caused by saving models in post_save hooks for the same models?"""
+    return hasattr(instance, '_do_not_copy_to_course_db')
+
+def mark_handled(instance):
+    """You have to mark your instance with this function or else we'll go into
+    an infinite loop since we're putting listeners on Model saves/deletes and 
+    the act of replication requires us to call the same model method.
+
+    We create a _replicated attribute to differentiate the first save of this
+    model vs. the duplicate save we force on to the course database. Kind of
+    a hack -- suggestions welcome.
+    """
+    instance._do_not_copy_to_course_db = True
+
+def should_replicate(instance):
+    """Should this instance be replicated? We need to be a Portal server and
+    the instance has to not have been marked_handled."""
+    if marked_handled(instance):
+        # Basically, avoid an infinite loop. You should 
+        log.debug("{0} should not be replicated because it's been marked"
+                  .format(instance))
+        return False
+    if not is_portal():
+        log.debug("{0} should not be replicated because we're not a portal."
+                  .format(instance))
+        return False
+    return True
+
+
+
+
+
+
+

common/djangoapps/student/tests.py

@@ -4,13 +4,178 @@ when you run "manage.py test".
 
 Replace this with more appropriate tests for your application.
 """
+from datetime import datetime
 
 from django.test import TestCase
 
+from .models import User, UserProfile, CourseEnrollment, replicate_user, USER_FIELDS_TO_COPY
+
+COURSE_1 = 'edX/toy/2012_Fall'
+COURSE_2 = 'edx/full/6.002_Spring_2012'
+
+class ReplicationTest(TestCase):
+
+    multi_db = True
+
+    def test_user_replication(self):
+        """Test basic user replication."""
+        portal_user = User.objects.create_user('rusty', 'rusty@edx.org', 'fakepass')
+        portal_user.first_name='Rusty'
+        portal_user.last_name='Skids'
+        portal_user.is_staff=True
+        portal_user.is_active=True
+        portal_user.is_superuser=True
+        portal_user.last_login=datetime(2012, 1, 1)
+        portal_user.date_joined=datetime(2011, 1, 1)
+        # This is an Askbot field and will break if askbot is not included
+
+        if hasattr(portal_user, 'seen_response_count'):
+            portal_user.seen_response_count = 10
+
+        portal_user.save(using='default')
+
+        # We replicate this user to Course 1, then pull the same user and verify
+        # that the fields copied over properly.
+        replicate_user(portal_user, COURSE_1)
+        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
+
+        # Make sure the fields we care about got copied over for this user.
+        for field in USER_FIELDS_TO_COPY:
+            self.assertEqual(getattr(portal_user, field),
+                             getattr(course_user, field),
+                             "{0} not copied from {1} to {2}".format(
+                                 field, portal_user, course_user
+                             ))
+
+        if hasattr(portal_user, 'seen_response_count'):
+            # Since it's the first copy over of User data, we should have all of it
+            self.assertEqual(portal_user.seen_response_count,
+                             course_user.seen_response_count)
+
+        # But if we replicate again, the user already exists in the Course DB,
+        # so it shouldn't update the seen_response_count (which is Askbot 
+        # controlled).
+        # This hasattr lameness is here because we don't want this test to be
+        # triggered when we're being run by CMS tests (Askbot doesn't exist
+        # there, so the test will fail).
+        if hasattr(portal_user, 'seen_response_count'):
+            portal_user.seen_response_count = 20
+            replicate_user(portal_user, COURSE_1)
+            course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
+            self.assertEqual(portal_user.seen_response_count, 20)
+            self.assertEqual(course_user.seen_response_count, 10)
+
+        # Another replication should work for an email change however, since
+        # it's a field we care about.
+        portal_user.email = "clyde@edx.org"
+        replicate_user(portal_user, COURSE_1)
+        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
+        self.assertEqual(portal_user.email, course_user.email)
+
+        # During this entire time, the user data should never have made it over
+        # to COURSE_2
+        self.assertRaises(User.DoesNotExist, 
+                          User.objects.using(COURSE_2).get,
+                          id=portal_user.id)
+
+
+    def test_enrollment_for_existing_user_info(self):
+        """Test the effect of Enrolling in a class if you've already got user
+        data to be copied over."""
+        # Create our User
+        portal_user = User.objects.create_user('jack', 'jack@edx.org', 'fakepass')
+        portal_user.first_name = "Jack"
+        portal_user.save()
+
+        # Set up our UserProfile info
+        portal_user_profile = UserProfile.objects.create(
+                                  user=portal_user,
+                                  name="Jack Foo",
+                                  level_of_education=None,
+                                  gender='m',
+                                  mailing_address=None,
+                                  goals="World domination",
+                              )
+        portal_user_profile.save()
+
+        # Now let's see if creating a CourseEnrollment copies all the relevant
+        # data.
+        portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
+                                                            course_id=COURSE_1)
+        portal_enrollment.save()
+
+        # Grab all the copies we expect
+        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
+        self.assertEquals(portal_user, course_user)
+        self.assertRaises(User.DoesNotExist, 
+                          User.objects.using(COURSE_2).get,
+                          id=portal_user.id)
+
+        course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
+        self.assertEquals(portal_enrollment, course_enrollment)
+        self.assertRaises(CourseEnrollment.DoesNotExist, 
+                          CourseEnrollment.objects.using(COURSE_2).get,
+                          id=portal_enrollment.id)
+
+        course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
+        self.assertEquals(portal_user_profile, course_user_profile)
+        self.assertRaises(UserProfile.DoesNotExist, 
+                          UserProfile.objects.using(COURSE_2).get,
+                          id=portal_user_profile.id)
+
+
+    def test_enrollment_for_user_info_after_enrollment(self):
+        """Test the effect of modifying User data after you've enrolled."""
+        # Create our User
+        portal_user = User.objects.create_user('patty', 'patty@edx.org', 'fakepass')
+        portal_user.first_name = "Patty"
+        portal_user.save()
+
+        # Set up our UserProfile info
+        portal_user_profile = UserProfile.objects.create(
+                                  user=portal_user,
+                                  name="Patty Foo",
+                                  level_of_education=None,
+                                  gender='f',
+                                  mailing_address=None,
+                                  goals="World peace",
+                              )
+        portal_user_profile.save()
+
+        # Now let's see if creating a CourseEnrollment copies all the relevant
+        # data when things are saved.
+        portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
+                                                            course_id=COURSE_1)
+        portal_enrollment.save()
+
+        portal_user.last_name = "Bar"
+        portal_user.save()
+        portal_user_profile.gender = 'm'
+        portal_user_profile.save()
+      
+        # Grab all the copies we expect, and make sure it doesn't end up in 
+        # places we don't expect.
+        course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
+        self.assertEquals(portal_user, course_user)
+        self.assertRaises(User.DoesNotExist, 
+                          User.objects.using(COURSE_2).get,
+                          id=portal_user.id)
+
+        course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
+        self.assertEquals(portal_enrollment, course_enrollment)
+        self.assertRaises(CourseEnrollment.DoesNotExist, 
+                          CourseEnrollment.objects.using(COURSE_2).get,
+                          id=portal_enrollment.id)
+
+        course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
+        self.assertEquals(portal_user_profile, course_user_profile)
+        self.assertRaises(UserProfile.DoesNotExist, 
+                          UserProfile.objects.using(COURSE_2).get,
+                          id=portal_user_profile.id)
+
+
+
+
+
+
 
-class SimpleTest(TestCase):
-    def test_basic_addition(self):
-        """
-        Tests that 1 + 1 always equals 2.
-        """
-        self.assertEqual(1 + 1, 2)

common/djangoapps/util/json_request.py

@@ -9,7 +9,7 @@ def expect_json(view_function):
         if request.META['CONTENT_TYPE'] == "application/json":
             cloned_request = copy.copy(request)
             cloned_request.POST = cloned_request.POST.copy()
-            cloned_request.POST.update(json.loads(request.raw_post_data))
+            cloned_request.POST.update(json.loads(request.body))
             return view_function(cloned_request, *args, **kwargs)
         else:
             return view_function(request, *args, **kwargs)

common/djangoapps/util/middleware.py

-import logging
-
-from django.conf import settings
-from django.http import HttpResponseServerError
-
-log = logging.getLogger("mitx")
-
-
-class ExceptionLoggingMiddleware(object):
-    """Just here to log unchecked exceptions that go all the way up the Django
-    stack"""
-
-    if not settings.TEMPLATE_DEBUG:
-        def process_exception(self, request, exception):
-            log.exception(exception)
-            return HttpResponseServerError("Server Error - Please try again later.")

common/djangoapps/xmodule_modifiers.py

@@ -34,7 +34,7 @@ def wrap_xmodule(get_html, module, template):
     return _get_html
 
 
-def replace_static_urls(get_html, prefix):
+def replace_static_urls(get_html, prefix, module):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -69,14 +69,14 @@ def grade_histogram(module_id):
     return grades
 
 
-def add_histogram(get_html, module):
+def add_histogram(get_html, module, user):
     """
     Updates the supplied module with a new get_html function that wraps
     the output of the old get_html function with additional information
     for admin users only, including a histogram of student answers and the
     definition of the xmodule
 
-    Does nothing if module is a SequenceModule
+    Does nothing if module is a SequenceModule or a VerticalModule.
     """
     @wraps(get_html)
     def _get_html():
@@ -90,19 +90,27 @@ def add_histogram(get_html, module):
 
         # TODO (ichuang): Remove after fall 2012 LMS migration done
         if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
-            [filepath, filename] = module.definition.get('filename','')
+            [filepath, filename] = module.definition.get('filename', ['', None])
             osfs = module.system.filestore
             if filename is not None and osfs.exists(filename):
-                filepath = filename	# if original, unmangled filename exists then use it (github doesn't like symlinks)
+                # if original, unmangled filename exists then use it (github
+                # doesn't like symlinks)
+                filepath = filename
             data_dir = osfs.root_path.rsplit('/')[-1]
-            edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath)
+            giturl = module.metadata.get('giturl','https://github.com/MITx')
+            edit_link = "%s/%s/tree/master/%s" % (giturl,data_dir,filepath)
         else:
             edit_link = False
 
         staff_context = {'definition': module.definition.get('data'),
                          'metadata': json.dumps(module.metadata, indent=4),
-                         'element_id': module.location.html_id(),
+                         'location': module.location,
+                         'xqa_key': module.metadata.get('xqa_key',''),
+                         'category': str(module.__class__.__name__),
+                         'element_id': module.location.html_id().replace('-','_'),
                          'edit_link': edit_link,
+                         'user': user,
+                         'xqa_server' : settings.MITX_FEATURES.get('USE_XQA_SERVER','http://xqa:server@content-qa.mitx.mit.edu/xqa'),
                          'histogram': json.dumps(histogram),
                          'render_histogram': render_histogram,
                          'module_content': get_html()}

common/lib/capa/capa/capa_problem.py

@@ -203,8 +203,9 @@ class LoncapaProblem(object):
         cmap.update(self.correct_map)
         for responder in self.responders.values():
             if hasattr(responder, 'update_score'):
-                # Each LoncapaResponse will update the specific entries of 'cmap' that it's responsible for
-                cmap = responder.update_score(score_msg, cmap, queuekey)
+                # Each LoncapaResponse will update its specific entries in cmap
+                #   cmap is passed by reference
+                responder.update_score(score_msg, cmap, queuekey)
         self.correct_map.set_dict(cmap.get_dict())
         return cmap
 
@@ -228,14 +229,14 @@ class LoncapaProblem(object):
 
         Calls the Response for each question in this problem, to do the actual grading.
         '''
-        
+
         self.student_answers = convert_files_to_filenames(answers)
-        
+
         oldcmap = self.correct_map				# old CorrectMap
         newcmap = CorrectMap()					# start new with empty CorrectMap
         # log.debug('Responders: %s' % self.responders)
         for responder in self.responders.values():                  # Call each responsetype instance to do actual grading
-            if 'filesubmission' in responder.allowed_inputfields:   # File objects are passed only if responsetype 
+            if 'filesubmission' in responder.allowed_inputfields:   # File objects are passed only if responsetype
                                                                     #   explicitly allows for file submissions
                 results = responder.evaluate_answers(answers, oldcmap)
             else:
@@ -294,9 +295,9 @@ class LoncapaProblem(object):
                 try:
                     ifp = self.system.filestore.open(file)  # open using ModuleSystem OSFS filestore
                 except Exception as err:
-                    log.error('Error %s in problem xml include: %s' % (
+                    log.warning('Error %s in problem xml include: %s' % (
                             err, etree.tostring(inc, pretty_print=True)))
-                    log.error('Cannot find file %s in %s' % (
+                    log.warning('Cannot find file %s in %s' % (
                             file, self.system.filestore))
                     # if debugging, don't fail - just log error
                     # TODO (vshnayder): need real error handling, display to users
@@ -305,11 +306,11 @@ class LoncapaProblem(object):
                     else:
                         continue
                 try:
-                    incxml = etree.XML(ifp.read())		# read in and convert to XML
+                    incxml = etree.XML(ifp.read())    # read in and convert to XML
                 except Exception as err:
-                    log.error('Error %s in problem xml include: %s' % (
+                    log.warning('Error %s in problem xml include: %s' % (
                             err, etree.tostring(inc, pretty_print=True)))
-                    log.error('Cannot parse XML in %s' % (file))
+                    log.warning('Cannot parse XML in %s' % (file))
                     # if debugging, don't fail - just log error
                     # TODO (vshnayder): same as above
                     if not self.system.get('DEBUG'):
@@ -392,9 +393,10 @@ class LoncapaProblem(object):
             context['script_code'] += code		# store code source in context
             try:
                 exec code in context, context        	# use "context" for global context; thus defs in code are global within code
-            except Exception:
+            except Exception as err:
                 log.exception("Error while execing script code: " + code)
-                raise responsetypes.LoncapaProblemError("Error while executing script code")
+		msg = "Error while executing script code: %s" % str(err).replace('<','&lt;')
+                raise responsetypes.LoncapaProblemError(msg)
             finally:
                 sys.path = original_path
 

common/lib/capa/capa/inputtypes.py

@@ -205,7 +205,7 @@ def extract_choices(element):
             raise Exception("[courseware.capa.inputtypes.extract_choices] \
                              Expected a <choice> tag; got %s instead"
                              % choice.tag)
-        choice_text = ''.join([etree.tostring(x) for x in choice])
+        choice_text = ''.join([x.text for x in choice])
 
         choices.append((choice.get("name"), choice_text))
 
@@ -336,9 +336,19 @@ def filesubmission(element, value, status, render_template, msg=''):
     Upload a single file (e.g. for programming assignments)
     '''
     eid = element.get('id')
-    context = { 'id': eid, 'state': status, 'msg': msg, 'value': value, }
+
+    # Check if problem has been queued
+    queue_len = 0
+    if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
+        status = 'queued'
+        queue_len = msg
+        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+
+    context = { 'id': eid, 'state': status, 'msg': msg, 'value': value,
+                'queue_len': queue_len
+              }
     html = render_template("filesubmission.html", context)
-    return etree.XML(html) 
+    return etree.XML(html)
 
 
 #-----------------------------------------------------------------------------
@@ -359,9 +369,16 @@ def textbox(element, value, status, render_template, msg=''):
 
     if not value: value = element.text	 # if no student input yet, then use the default input given by the problem
 
+    # Check if problem has been queued
+    queue_len = 0
+    if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
+        status = 'queued'
+        queue_len = msg
+        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+
     # For CodeMirror
-    mode = element.get('mode')	or 'python' # mode, eg "python" or "xml"
-    linenumbers = element.get('linenumbers','true')	 # for CodeMirror
+    mode = element.get('mode','python')
+    linenumbers = element.get('linenumbers','true')
     tabsize = element.get('tabsize','4')
     tabsize = int(tabsize)
 
@@ -369,6 +386,7 @@ def textbox(element, value, status, render_template, msg=''):
                'mode': mode, 'linenumbers': linenumbers,
                'rows': rows, 'cols': cols,
                'hidden': hidden, 'tabsize': tabsize,
+               'queue_len': queue_len,
                }
     html = render_template("textbox.html", context)
     try:

common/lib/capa/capa/responsetypes.py

@@ -990,6 +990,12 @@ class CodeResponse(LoncapaResponse):
     '''
     Grade student code using an external queueing server, called 'xqueue'
 
+    Expects 'xqueue' dict in ModuleSystem with the following keys:
+        system.xqueue = { 'interface': XqueueInterface object,
+                          'callback_url': Per-StudentModule callback URL where results are posted (string),
+                          'default_queuename': Default queuename to submit request (string)
+                        }
+
     External requests are only submitted for student submission grading 
         (i.e. and not for getting reference answers)
     '''
@@ -999,10 +1005,27 @@ class CodeResponse(LoncapaResponse):
     max_inputfields = 1
 
     def setup_response(self):
+        '''
+        Configure CodeResponse from XML. Supports both CodeResponse and ExternalResponse XML
+
+        TODO: Determines whether in synchronous or asynchronous (queued) mode
+        '''
         xml = self.xml
+        self.url = xml.get('url', None) # XML can override external resource (grader/queue) URL
         self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
 
-        answer = xml.find('answer')
+        self._parse_externalresponse_xml()
+
+    def _parse_externalresponse_xml(self):
+        '''
+        VS[compat]: Suppport for old ExternalResponse XML format. When successful, sets:
+            self.code
+            self.tests
+            self.answer
+            self.initial_display
+        '''
+        answer = self.xml.find('answer')
+
         if answer is not None:
             answer_src = answer.get('src')
             if answer_src is not None:
@@ -1016,7 +1039,7 @@ class CodeResponse(LoncapaResponse):
                 msg += "\nSee XML source line %s" % getattr(self.xml, 'sourceline', '<unavailable>')
                 raise LoncapaProblemError(msg)
 
-        self.tests = xml.get('tests')
+        self.tests = self.xml.get('tests')
 
         # Extract 'answer' and 'initial_display' from XML. Note that the code to be exec'ed here is:
         #   (1) Internal edX code, i.e. NOT student submissions, and
@@ -1063,15 +1086,16 @@ class CodeResponse(LoncapaResponse):
                     'edX_cmd': 'get_score',
                     'edX_tests': self.tests,
                     'processor': self.code,
-                    'edX_student_response': unicode(submission), # unicode on File object returns its filename
                     }
         
-        # Submit request
-        if hasattr(submission, 'read'): # Test for whether submission is a file
+        # Submit request. When successful, 'msg' is the prior length of the queue
+        if is_file(submission):
+            contents.update({'edX_student_response': submission.name})
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents),
                                                     file_to_upload=submission)
         else:
+            contents.update({'edX_student_response': submission})
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents))
 
@@ -1080,33 +1104,31 @@ class CodeResponse(LoncapaResponse):
             cmap.set(self.answer_id, queuekey=None,
                      msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg)
         else:
-            # Non-null CorrectMap['queuekey'] indicates that the problem has been queued 
-            cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader. (Queue length: %s)' % msg)
+            # Queueing mechanism flags:
+            #   1) Backend: Non-null CorrectMap['queuekey'] indicates that the problem has been queued
+            #   2) Frontend: correctness='incomplete' eventually trickles down through inputtypes.textbox 
+            #       and .filesubmission to inform the browser to poll the LMS
+            cmap.set(self.answer_id, queuekey=queuekey, correctness='incomplete', msg=msg)
 
         return cmap
 
     def update_score(self, score_msg, oldcmap, queuekey):
-        # Parse 'score_msg' as XML
-        try:
-            rxml = etree.fromstring(score_msg)
-        except Exception as err:
-            msg = 'Error in CodeResponse %s: cannot parse response from xworker r.text=%s' % (err, score_msg)
-            raise Exception(err)
 
-        # The following process is lifted directly from ExternalResponse
-        ad = rxml.find('awarddetail').text
-        admap = {'EXACT_ANS': 'correct',         # TODO: handle other loncapa responses
-                 'WRONG_FORMAT': 'incorrect',
-                 }
-        self.context['correct'] = ['correct']
-        if ad in admap:
-            self.context['correct'][0] = admap[ad]
+        (valid_score_msg, correct, score, msg) = self._parse_score_msg(score_msg) 
+        if not valid_score_msg:
+            oldcmap.set(self.answer_id, msg='Error: Invalid grader reply.')
+            return oldcmap
+        
+        correctness = 'incorrect'
+        if correct:
+            correctness = 'correct'
+
+        self.context['correct'] = correctness # TODO: Find out how this is used elsewhere, if any
 
         # Replace 'oldcmap' with new grading results if queuekey matches.
         #   If queuekey does not match, we keep waiting for the score_msg whose key actually matches
         if oldcmap.is_right_queuekey(self.answer_id, queuekey):
-            msg = rxml.find('message').text.replace('&nbsp;', '&#160;')
-            oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None)  # Queuekey is consumed
+            oldcmap.set(self.answer_id, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuekey=None)  # Queuekey is consumed
         else:
             log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id))
 
@@ -1119,6 +1141,31 @@ class CodeResponse(LoncapaResponse):
     def get_initial_display(self):
         return {self.answer_id: self.initial_display}
 
+    def _parse_score_msg(self, score_msg):
+        '''
+         Grader reply is a JSON-dump of the following dict
+           { 'correct': True/False,
+             'score': # TODO -- Partial grading
+             'msg': grader_msg }
+
+        Returns (valid_score_msg, correct, score, msg):
+            valid_score_msg: Flag indicating valid score_msg format (Boolean)
+            correct:         Correctness of submission (Boolean)
+            score:           # TODO: Implement partial grading
+            msg:             Message from grader to display to student (string)
+        '''
+        fail = (False, False, -1, '')
+        try:
+            score_result = json.loads(score_msg)
+        except (TypeError, ValueError):
+            return fail
+        if not isinstance(score_result, dict):
+            return fail
+        for tag in ['correct', 'score', 'msg']:
+            if not score_result.has_key(tag):
+                return fail
+        return (True, score_result['correct'], score_result['score'], score_result['msg'])
+        
 
 #-----------------------------------------------------------------------------
 

common/lib/capa/capa/templates/filesubmission.html

@@ -6,8 +6,9 @@
     <span class="correct" id="status_${id}"></span>
     % elif state == 'incorrect':
     <span class="incorrect" id="status_${id}"></span>
-    % elif state == 'incomplete':
-    <span class="incorrect" id="status_${id}"></span>
+    % elif state == 'queued':
+    <span class="processing" id="status_${id}"></span>
+    <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
     <span class="debug">(${state})</span>
     <br/>

common/lib/capa/capa/templates/textbox.html

@@ -13,11 +13,12 @@
   <span class="correct" id="status_${id}"></span>
   % elif state == 'incorrect':
   <span class="incorrect" id="status_${id}"></span>
-  % elif state == 'incomplete':
-  <span class="incorrect" id="status_${id}"></span>
+  % elif state == 'queued':
+  <span class="processing" id="status_${id}"></span>
+  <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
   % endif
   % if hidden:
-      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+  <div style="display:none;" name="${hidden}" inputid="input_${id}" />
   % endif
   <br/>
   <span class="debug">(${state})</span>

common/lib/capa/capa/util.py

@@ -39,5 +39,18 @@ def convert_files_to_filenames(answers):
     '''
     new_answers = dict()
     for answer_id in answers.keys():
-        new_answers[answer_id] = unicode(answers[answer_id])
+        if is_file(answers[answer_id]):
+            new_answers[answer_id] = answers[answer_id].name
+        else:
+            new_answers[answer_id] = answers[answer_id]
     return new_answers
+
+def is_file(file_to_test):
+    '''
+    Duck typing to check if 'file_to_test' is a File object
+    '''
+    is_file = True
+    for method in ['read', 'name']:
+        if not hasattr(file_to_test, method):
+            is_file = False
+    return is_file 

common/lib/capa/capa/xqueue_interface.py

@@ -67,7 +67,6 @@ class XqueueInterface:
         self.url  = url
         self.auth = auth
         self.session = requests.session()
-        self._login()
         
     def send_to_queue(self, header, body, file_to_upload=None):
         '''

common/lib/xmodule/xmodule/abtest_module.py

@@ -11,18 +11,19 @@ DEFAULT = "_DEFAULT_GROUP"
 
 
 def group_from_value(groups, v):
-    ''' Given group: (('a',0.3),('b',0.4),('c',0.3)) And random value
+    """
+    Given group: (('a', 0.3), ('b', 0.4), ('c', 0.3)) and random value v
     in [0,1], return the associated group (in the above case, return
-    'a' if v<0.3, 'b' if 0.3<=v<0.7, and 'c' if v>0.7
-'''
+    'a' if v < 0.3, 'b' if 0.3 <= v < 0.7, and 'c' if v > 0.7
+    """
     sum = 0
     for (g, p) in groups:
         sum = sum + p
         if sum > v:
             return g
 
-    # Round off errors might cause us to run to the end of the list
-    # If the do, return the last element
+    # Round off errors might cause us to run to the end of the list.
+    # If the do, return the last element.
     return g
 
 
@@ -31,8 +32,8 @@ class ABTestModule(XModule):
     Implements an A/B test with an aribtrary number of competing groups
     """
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, descriptor, instance_state, shared_state, **kwargs)
 
         if shared_state is None:
 
@@ -103,7 +104,9 @@ class ABTestDescriptor(RawDescriptor, XmlDescriptor):
         experiment = xml_object.get('experiment')
 
         if experiment is None:
-            raise InvalidDefinitionError("ABTests must specify an experiment. Not found in:\n{xml}".format(xml=etree.tostring(xml_object, pretty_print=True)))
+            raise InvalidDefinitionError(
+                "ABTests must specify an experiment. Not found in:\n{xml}"
+                .format(xml=etree.tostring(xml_object, pretty_print=True)))
 
         definition = {
             'data': {
@@ -127,7 +130,9 @@ class ABTestDescriptor(RawDescriptor, XmlDescriptor):
             definition['data']['group_content'][name] = child_content_urls
             definition['children'].extend(child_content_urls)
 
-        default_portion = 1 - sum(portion for (name, portion) in definition['data']['group_portions'].items())
+        default_portion = 1 - sum(
+            portion for (name, portion) in definition['data']['group_portions'].items())
+
         if default_portion < 0:
             raise InvalidDefinitionError("ABTest portions must add up to less than or equal to 1")
 

common/lib/xmodule/xmodule/capa_module.py

@@ -11,13 +11,13 @@ from datetime import timedelta
 from lxml import etree
 from pkg_resources import resource_string
 
-from xmodule.x_module import XModule
-from xmodule.raw_module import RawDescriptor
-from xmodule.exceptions import NotFoundError
-from progress import Progress
 from capa.capa_problem import LoncapaProblem
 from capa.responsetypes import StudentInputError
 from capa.util import convert_files_to_filenames
+from progress import Progress
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
+from xmodule.exceptions import NotFoundError
 
 log = logging.getLogger("mitx.courseware")
 
@@ -80,9 +80,9 @@ class CapaModule(XModule):
     js_module_name = "Problem"
     css = {'scss': [resource_string(__name__, 'css/capa/display.scss')]}
 
-    def __init__(self, system, location, definition, instance_state=None,
+    def __init__(self, system, location, definition, descriptor, instance_state=None,
                  shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state,
+        XModule.__init__(self, system, location, definition, descriptor, instance_state,
                          shared_state, **kwargs)
 
         self.attempts = 0
@@ -119,9 +119,9 @@ class CapaModule(XModule):
         if self.show_answer == "":
             self.show_answer = "closed"
 
-        if instance_state != None:
+        if instance_state is not None:
             instance_state = json.loads(instance_state)
-        if instance_state != None and 'attempts' in instance_state:
+        if instance_state is not None and 'attempts' in instance_state:
             self.attempts = instance_state['attempts']
 
         self.name = only_one(dom2.xpath('/problem/@name'))
@@ -130,16 +130,18 @@ class CapaModule(XModule):
         if weight_string:
             self.weight = float(weight_string)
         else:
-            self.weight = 1
+            self.weight = None
 
         if self.rerandomize == 'never':
             seed = 1
-        elif self.rerandomize == "per_student" and hasattr(system, 'id'):
+        elif self.rerandomize == "per_student" and hasattr(self.system, 'id'):
             seed = system.id
         else:
             seed = None
 
         try:
+            # TODO (vshnayder): move as much as possible of this work and error
+            # checking to descriptor load time
             self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(),
                                       instance_state, seed=seed, system=self.system)
         except Exception as err:
@@ -148,7 +150,7 @@ class CapaModule(XModule):
             # TODO (vshnayder): do modules need error handlers too?
             # We shouldn't be switching on DEBUG.
             if self.system.DEBUG:
-                log.error(msg)
+                log.warning(msg)
                 # TODO (vshnayder): This logic should be general, not here--and may
                 # want to preserve the data instead of replacing it.
                 # e.g. in the CMS
@@ -238,7 +240,7 @@ class CapaModule(XModule):
         content = {'name': self.metadata['display_name'],
                    'html': html,
                    'weight': self.weight,
-                  }
+                   }
 
         # We using strings as truthy values, because the terminology of the
         # check button is context-specific.
@@ -426,7 +428,7 @@ class CapaModule(XModule):
         event_info = dict()
         event_info['state'] = self.lcp.get_state()
         event_info['problem_id'] = self.location.url()
-         
+
         answers = self.make_dict_of_responses(get)
         event_info['answers'] = convert_files_to_filenames(answers)
 
@@ -563,6 +565,14 @@ class CapaDescriptor(RawDescriptor):
 
     module_class = CapaModule
 
+    stores_state = True
+    has_score = True
+
+    # Capa modules have some additional metadata:
+    # TODO (vshnayder): do problems have any other metadata?  Do they
+    # actually use type and points?
+    metadata_attributes = RawDescriptor.metadata_attributes + ('type', 'points')
+
     # VS[compat]
     # TODO (cpennington): Delete this method once all fall 2012 course are being
     # edited in the cms
@@ -572,8 +582,3 @@ class CapaDescriptor(RawDescriptor):
             'problems/' + path[8:],
             path[8:],
         ]
-
-    @classmethod
-    def split_to_file(cls, xml_object):
-        '''Problems always written in their own files'''
-        return True

common/lib/xmodule/xmodule/course_module.py

@@ -3,6 +3,7 @@ import time
 import dateutil.parser
 import logging
 
+from xmodule.util.decorators import lazyproperty
 from xmodule.graders import load_grading_policy
 from xmodule.modulestore import Location
 from xmodule.seq_module import SequenceDescriptor, SequenceModule
@@ -12,13 +13,9 @@ log = logging.getLogger(__name__)
 
 class CourseDescriptor(SequenceDescriptor):
     module_class = SequenceModule
-    metadata_attributes = SequenceDescriptor.metadata_attributes + ('org', 'course')
 
     def __init__(self, system, definition=None, **kwargs):
         super(CourseDescriptor, self).__init__(system, definition, **kwargs)
-        
-        self._grader = None
-        self._grade_cutoffs = None
 
         msg = None
         try:
@@ -39,34 +36,84 @@ class CourseDescriptor(SequenceDescriptor):
 
     def has_started(self):
         return time.gmtime() > self.start
-    
+
     @property
     def grader(self):
-        self.__load_grading_policy()
-        return self._grader
-    
+        return self.__grading_policy['GRADER']
+
     @property
     def grade_cutoffs(self):
-        self.__load_grading_policy()
-        return self._grade_cutoffs
-    
-    
-    def __load_grading_policy(self):
-        if not self._grader or not self._grade_cutoffs:
-            policy_string = ""
-            
-            try:
-                with self.system.resources_fs.open("grading_policy.json") as grading_policy_file:
-                    policy_string = grading_policy_file.read()
-            except (IOError, ResourceNotFoundError):
-                log.warning("Unable to load course settings file from grading_policy.json in course " + self.id)
-            
-            grading_policy = load_grading_policy(policy_string)
-            
-            self._grader = grading_policy['GRADER']
-            self._grade_cutoffs = grading_policy['GRADE_CUTOFFS']
-    
-    
+        return self.__grading_policy['GRADE_CUTOFFS']
+
+    @lazyproperty
+    def __grading_policy(self):
+        policy_string = ""
+
+        try:
+            with self.system.resources_fs.open("grading_policy.json") as grading_policy_file:
+                policy_string = grading_policy_file.read()
+        except (IOError, ResourceNotFoundError):
+            log.warning("Unable to load course settings file from grading_policy.json in course " + self.id)
+
+        grading_policy = load_grading_policy(policy_string)
+
+        return grading_policy
+
+
+    @lazyproperty
+    def grading_context(self):
+        """
+        This returns a dictionary with keys necessary for quickly grading
+        a student. They are used by grades.grade()
+
+        The grading context has two keys:
+        graded_sections - This contains the sections that are graded, as
+            well as all possible children modules that can affect the
+            grading. This allows some sections to be skipped if the student
+            hasn't seen any part of it.
+
+            The format is a dictionary keyed by section-type. The values are
+            arrays of dictionaries containing
+                "section_descriptor" : The section descriptor
+                "xmoduledescriptors" : An array of xmoduledescriptors that
+                    could possibly be in the section, for any student
+
+        all_descriptors - This contains a list of all xmodules that can
+            effect grading a student. This is used to efficiently fetch
+            all the xmodule state for a StudentModuleCache without walking
+            the descriptor tree again.
+
+
+        """
+
+        all_descriptors = []
+        graded_sections = {}
+
+        def yield_descriptor_descendents(module_descriptor):
+            for child in module_descriptor.get_children():
+                yield child
+                for module_descriptor in yield_descriptor_descendents(child):
+                    yield module_descriptor
+
+        for c in self.get_children():
+            sections = []
+            for s in c.get_children():
+                if s.metadata.get('graded', False):
+                    xmoduledescriptors = list(yield_descriptor_descendents(s))
+                    
+                    # The xmoduledescriptors included here are only the ones that have scores.
+                    section_description = { 'section_descriptor' : s, 'xmoduledescriptors' : filter(lambda child: child.has_score, xmoduledescriptors) }
+
+                    section_format = s.metadata.get('format', "")
+                    graded_sections[ section_format ] = graded_sections.get( section_format, [] ) + [section_description]
+
+                    all_descriptors.extend(xmoduledescriptors)
+                    all_descriptors.append(s)
+
+        return { 'graded_sections' : graded_sections,
+                 'all_descriptors' : all_descriptors,}
+
+
     @staticmethod
     def id_to_location(course_id):
         '''Convert the given course_id (org/course/name) to a location object.

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -49,6 +49,8 @@ padding-left: flex-gutter(9);
   }
 }
 
+
+
 div {
   p.status {
     text-indent: -9999px;
@@ -64,6 +66,16 @@ div {
     }
   }
 
+  &.processing {
+    p.status {
+      @include inline-block();
+      background: url('../images/spinner.gif') center center no-repeat;
+      height: 20px;
+      width: 20px;
+      text-indent: -9999px;
+    }
+  }
+
   &.correct, &.ui-icon-check {
     p.status {
       @include inline-block();
@@ -134,6 +146,15 @@ div {
       width: 14px;
     }
 
+    &.processing, &.ui-icon-check {
+      @include inline-block();
+      background: url('../images/spinner.gif') center center no-repeat;
+      height: 20px;
+      position: relative;
+      top: 6px;
+      width: 25px;
+    }
+
     &.correct, &.ui-icon-check {
       @include inline-block();
       background: url('../images/correct-icon.png') center center no-repeat;

common/lib/xmodule/xmodule/css/sequence/display.scss

@@ -3,9 +3,9 @@ nav.sequence-nav {
   // import from external sources.
   @extend .topbar;
   border-bottom: 1px solid $border-color;
+  @include border-top-right-radius(4px);
   margin: (-(lh())) (-(lh())) lh() (-(lh()));
   position: relative;
-  @include border-top-right-radius(4px);
 
   ol {
     @include box-sizing(border-box);
@@ -242,9 +242,11 @@ nav.sequence-bottom {
     border: 1px solid $border-color;
     @include border-radius(3px);
     @include inline-block();
+    width: 100px;
 
     li {
       float: left;
+      width: 50%;
 
       &.prev, &.next {
         margin-bottom: 0;
@@ -252,12 +254,11 @@ nav.sequence-bottom {
         a {
           background-position: center center;
           background-repeat: no-repeat;
-          border-bottom: none;
+          border: none;
           display: block;
           padding: lh(.5) 4px;
           text-indent: -9999px;
           @include transition(all, .2s, $ease-in-out-quad);
-          width: 45px;
 
           &:hover {
             background-color: #ddd;
@@ -275,7 +276,7 @@ nav.sequence-bottom {
       &.prev {
         a {
           background-image: url('../images/sequence-nav/previous-icon.png');
-          border-right: 1px solid lighten($border-color, 10%);
+          border-right: 1px solid lighten(#c6c6c6, 10%);
 
           &:hover {
             background-color: none;

common/lib/xmodule/xmodule/css/video/display.scss

@@ -16,7 +16,6 @@ div.video {
       height: 0;
       overflow: hidden;
       padding-bottom: 56.25%;
-      padding-top: 30px;
       position: relative;
 
       object, iframe {
@@ -207,7 +206,7 @@ div.video {
             h3 {
               color: #999;
               float: left;
-              font-size: 12px;
+              font-size: em(14);
               font-weight: normal;
               letter-spacing: 1px;
               padding: 0 lh(.25) 0 lh(.5);
@@ -221,6 +220,7 @@ div.video {
               margin-bottom: 0;
               padding: 0 lh(.5) 0 0;
               line-height: 46px;
+              color: #fff;
             }
 
             &:hover, &:active, &:focus {
@@ -462,7 +462,8 @@ div.video {
     }
 
     ol.subtitles {
-      width: 0px;
+      width: 0;
+      height: 0;
     }
   }
 

common/lib/xmodule/xmodule/error_module.py

-import sys
+import hashlib
 import logging
+import random
+import string
+import sys
 
 from pkg_resources import resource_string
 from lxml import etree
@@ -24,6 +27,14 @@ class ErrorModule(XModule):
             'is_staff' : self.system.is_staff,
             })
 
+    def displayable_items(self):
+        """Hide errors in the profile and table of contents for non-staff
+        users.
+        """
+        if self.system.is_staff:
+            return [self]
+        return []
+
 class ErrorDescriptor(EditingDescriptor):
     """
     Module that provides a raw editing view of broken xml.
@@ -35,7 +46,8 @@ class ErrorDescriptor(EditingDescriptor):
                  error_msg='Error not available'):
         '''Create an instance of this descriptor from the supplied data.
 
-        Does not try to parse the data--just stores it.
+        Does not require that xml_data be parseable--just stores it and exports
+        as-is if not.
 
         Takes an extra, optional, parameter--the error that caused an
         issue.  (should be a string, or convert usefully into one).
@@ -45,6 +57,13 @@ class ErrorDescriptor(EditingDescriptor):
         definition = {'data': inner}
         inner['error_msg'] = str(error_msg)
 
+        # Pick a unique url_name -- the sha1 hash of the xml_data.
+        # NOTE: We could try to pull out the url_name of the errored descriptor,
+        # but url_names aren't guaranteed to be unique between descriptor types,
+        # and ErrorDescriptor can wrap any type.  When the wrapped module is fixed,
+        # it will be written out with the original url_name.
+        url_name = hashlib.sha1(xml_data).hexdigest()
+
         try:
             # If this is already an error tag, don't want to re-wrap it.
             xml_obj = etree.fromstring(xml_data)
@@ -63,8 +82,9 @@ class ErrorDescriptor(EditingDescriptor):
         inner['contents'] = xml_data
         # TODO (vshnayder): Do we need a unique slug here?  Just pick a random
         # 64-bit num?
-        location = ['i4x', org, course, 'error', 'slug']
-        metadata = {}  # stays in the xml_data
+        location = ['i4x', org, course, 'error', url_name]
+        # real metadata stays in the xml_data, but add a display name
+        metadata = {'display_name': 'Error ' + url_name}
 
         return cls(system, definition, location=location, metadata=metadata)
 

common/lib/xmodule/xmodule/html_module.py

@@ -13,13 +13,14 @@ from .html_checker import check_html
 
 log = logging.getLogger("mitx.courseware")
 
+
 class HtmlModule(XModule):
     def get_html(self):
         return self.html
 
-    def __init__(self, system, location, definition,
+    def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition,
+        XModule.__init__(self, system, location, definition, descriptor,
                          instance_state, shared_state, **kwargs)
         self.html = self.definition['data']
 
@@ -36,18 +37,19 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
     # are being edited in the cms
     @classmethod
     def backcompat_paths(cls, path):
-        origpath = path
         if path.endswith('.html.xml'):
-            path = path[:-9] + '.html'  #backcompat--look for html instead of xml
+            path = path[:-9] + '.html'  # backcompat--look for html instead of xml
         candidates = []
         while os.sep in path:
             candidates.append(path)
             _, _, path = path.partition(os.sep)
 
         # also look for .html versions instead of .xml
-        if origpath.endswith('.xml'):
-            candidates.append(origpath[:-4] + '.html')
-        return candidates
+        nc = []
+        for candidate in candidates:
+            if candidate.endswith('.xml'):
+                nc.append(candidate[:-4] + '.html')
+        return candidates + nc
 
     # NOTE: html descriptors are special.  We do not want to parse and
     # export them ourselves, because that can break things (e.g. lxml
@@ -69,7 +71,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
         if filename is None:
             definition_xml = copy.deepcopy(xml_object)
             cls.clean_metadata_from_xml(definition_xml)
-            return {'data' : stringify_children(definition_xml)}
+            return {'data': stringify_children(definition_xml)}
         else:
             filepath = cls._format_filepath(xml_object.tag, filename)
 
@@ -80,7 +82,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             # online and has imported all current (fall 2012) courses from xml
             if not system.resources_fs.exists(filepath):
                 candidates = cls.backcompat_paths(filepath)
-                #log.debug("candidates = {0}".format(candidates))
+                log.debug("candidates = {0}".format(candidates))
                 for candidate in candidates:
                     if system.resources_fs.exists(candidate):
                         filepath = candidate
@@ -95,7 +97,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
                         log.warning(msg)
                         system.error_tracker("Warning: " + msg)
 
-                    definition = {'data' : html}
+                    definition = {'data': html}
 
                     # TODO (ichuang): remove this after migration
                     # for Fall 2012 LMS migration: keep filename (and unmangled filename)
@@ -109,17 +111,11 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
                 # add more info and re-raise
                 raise Exception(msg), None, sys.exc_info()[2]
 
-    @classmethod
-    def split_to_file(cls, xml_object):
-        '''Never include inline html'''
-        return True
-
-
     # TODO (vshnayder): make export put things in the right places.
 
     def definition_to_xml(self, resource_fs):
         '''If the contents are valid xml, write them to filename.xml.  Otherwise,
-        write just the <html filename=""> tag to filename.xml, and the html
+        write just <html filename="" [meta-attrs="..."]> to filename.xml, and the html
         string to filename.html.
         '''
         try:
@@ -138,4 +134,3 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
         elt = etree.Element('html')
         elt.set("filename", self.url_name)
         return elt
-

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -13,7 +13,10 @@ class @Problem
   bind: =>
     MathJax.Hub.Queue ["Typeset", MathJax.Hub]
     window.update_schematics()
-    @inputs = @$("[id^=input_#{@element_id.replace(/problem_/, '')}_]")
+
+    problem_prefix = @element_id.replace(/problem_/,'')
+    @inputs = @$("[id^=input_#{problem_prefix}_]")
+    
     @$('section.action input:button').click @refreshAnswers
     @$('section.action input.check').click @check_fd
     #@$('section.action input.check').click @check
@@ -27,18 +30,40 @@ class @Problem
         @el.attr progress: response.progress_status
         @el.trigger('progressChanged')
 
+  queueing: =>
+    @queued_items = @$(".xqueue")
+    if @queued_items.length > 0
+      if window.queuePollerID # Only one poller 'thread' per Problem
+        window.clearTimeout(window.queuePollerID)
+      window.queuePollerID = window.setTimeout(@poll, 100)
+
+  poll: =>
+    $.postWithPrefix "#{@url}/problem_get", (response) =>
+      @el.html(response.html)
+      @executeProblemScripts()
+      @bind()
+
+      @queued_items = @$(".xqueue")
+      if @queued_items.length == 0 
+        delete window.queuePollerID
+      else
+        # TODO: Dynamically adjust timeout interval based on @queued_items.value
+        window.queuePollerID = window.setTimeout(@poll, 1000)
+
   render: (content) ->
     if content
       @el.html(content)
       @executeProblemScripts () =>
         @setupInputTypes()
         @bind()
+        @queueing()
     else
       $.postWithPrefix "#{@url}/problem_get", (response) =>
         @el.html(response.html)
         @executeProblemScripts () =>
           @setupInputTypes()
           @bind()
+          @queueing()
 
   # TODO add hooks for problem types here by inspecting response.html and doing
   # stuff if a div w a class is found

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -91,6 +91,13 @@ class @Sequence
     event.preventDefault()
     new_position = $(event.target).data('element')
     Logger.log "seq_goto", old: @position, new: new_position, id: @id
+
+    # On Sequence chage, destroy any existing polling thread 
+    #   for queued submissions, see ../capa/display.coffee
+    if window.queuePollerID
+      window.clearTimeout(window.queuePollerID)
+      delete window.queuePollerID
+
     @render new_position
 
   next: (event) =>

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -190,6 +190,13 @@ class Location(_LocationBase):
         return "Location%s" % repr(tuple(self))
 
 
+    @property
+    def course_id(self):
+        """Return the ID of the Course that this item belongs to by looking
+        at the location URL hierachy"""
+        return "/".join([self.org, self.course, self.name])
+
+
 class ModuleStore(object):
     """
     An abstract interface for a database backend that stores XModuleDescriptor

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -55,6 +55,9 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
         if json_data is None:
             return self.modulestore.get_item(location)
         else:
+            # TODO (vshnayder): metadata inheritance is somewhat broken because mongo, doesn't
+            # always load an entire course.  We're punting on this until after launch, and then
+            # will build a proper course policy framework.
             return XModuleDescriptor.load_from_json(json_data, self, self.default_class)
 
 

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -50,8 +50,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 # have been imported into the cms from xml
                 xml = clean_out_mako_templating(xml)
                 xml_data = etree.fromstring(xml)
-            except:
-                log.exception("Unable to parse xml: {xml}".format(xml=xml))
+            except Exception as err:
+                log.warning("Unable to parse xml: {err}, xml: {xml}".format(
+                    err=str(err), xml=xml))
                 raise
 
             # VS[compat]. Take this out once course conversion is done
@@ -188,26 +189,37 @@ class XMLModuleStore(ModuleStoreBase):
             course_file = StringIO(clean_out_mako_templating(course_file.read()))
 
             course_data = etree.parse(course_file).getroot()
+
             org = course_data.get('org')
 
             if org is None:
-                log.error("No 'org' attribute set for course in {dir}. "
+                msg = ("No 'org' attribute set for course in {dir}. "
                           "Using default 'edx'".format(dir=course_dir))
+                log.warning(msg)
+                tracker(msg)
                 org = 'edx'
 
             course = course_data.get('course')
 
             if course is None:
-                log.error("No 'course' attribute set for course in {dir}."
+                msg = ("No 'course' attribute set for course in {dir}."
                           " Using default '{default}'".format(
                         dir=course_dir,
                         default=course_dir
                         ))
+                log.warning(msg)
+                tracker(msg)
                 course = course_dir
 
             system = ImportSystem(self, org, course, course_dir, tracker)
 
             course_descriptor = system.process_xml(etree.tostring(course_data))
+            # NOTE: The descriptors end up loading somewhat bottom up, which
+            # breaks metadata inheritance via get_children().  Instead
+            # (actually, in addition to, for now), we do a final inheritance pass
+            # after we have the course descriptor.
+            XModuleDescriptor.compute_inherited_metadata(course_descriptor)
+
             log.debug('========> Done with course import from {0}'.format(course_dir))
             return course_descriptor
 

common/lib/xmodule/xmodule/seq_module.py

@@ -25,9 +25,9 @@ class SequenceModule(XModule):
     css = {'scss': [resource_string(__name__, 'css/sequence/display.scss')]}
     js_module_name = "Sequence"
 
-    def __init__(self, system, location, definition, instance_state=None,
+    def __init__(self, system, location, definition, descriptor, instance_state=None,
                  shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition,
+        XModule.__init__(self, system, location, definition, descriptor,
                          instance_state, shared_state, **kwargs)
         self.position = 1
 
@@ -107,6 +107,8 @@ class SequenceModule(XModule):
 class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
     mako_template = 'widgets/sequence-edit.html'
     module_class = SequenceModule
+    
+    stores_state = True # For remembering where in the sequence the student is
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):
@@ -122,16 +124,3 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
                 etree.fromstring(child.export_to_xml(resource_fs)))
         return xml_object
 
-    @classmethod
-    def split_to_file(cls, xml_object):
-        # Note: if we end up needing subclasses, can port this logic there.
-        yes = ('chapter',)
-        no = ('course',)
-
-        if xml_object.tag in yes:
-            return True
-        elif xml_object.tag in no:
-            return False
-
-        # otherwise maybe--delegate to superclass.
-        return XmlDescriptor.split_to_file(xml_object)

common/lib/xmodule/xmodule/template_module.py

@@ -26,12 +26,23 @@ class CustomTagModule(XModule):
         More information given in <a href="/book/234">the text</a>
     """
 
-    def __init__(self, system, location, definition,
+    def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition,
+        XModule.__init__(self, system, location, definition, descriptor,
                          instance_state, shared_state, **kwargs)
 
-        xmltree = etree.fromstring(self.definition['data'])
+    def get_html(self):
+        return self.descriptor.rendered_html
+
+
+class CustomTagDescriptor(RawDescriptor):
+    """ Descriptor for custom tags.  Loads the template when created."""
+    module_class = CustomTagModule
+
+    @staticmethod
+    def render_template(system, xml_data):
+        '''Render the template, given the definition xml_data'''
+        xmltree = etree.fromstring(xml_data)
         if 'impl' in xmltree.attrib:
             template_name = xmltree.attrib['impl']
         else:
@@ -45,13 +56,20 @@ class CustomTagModule(XModule):
                                 .format(location))
 
         params = dict(xmltree.items())
-        with self.system.filestore.open(
-                'custom_tags/{name}'.format(name=template_name)) as template:
-            self.html = Template(template.read()).render(**params)
+        with system.resources_fs.open('custom_tags/{name}'
+                                   .format(name=template_name)) as template:
+            return Template(template.read()).render(**params)
 
-    def get_html(self):
-        return self.html
 
+    def __init__(self, system, definition, **kwargs):
+        '''Render and save the template for this descriptor instance'''
+        super(CustomTagDescriptor, self).__init__(system, definition, **kwargs)
+        self.rendered_html = self.render_template(system, definition['data'])
+
+    def export_to_file(self):
+        """
+        Custom tags are special: since they're already pointers, we don't want
+        to export them in a file with yet another layer of indirection.
+        """
+        return False
 
-class CustomTagDescriptor(RawDescriptor):
-    module_class = CustomTagModule

common/lib/xmodule/xmodule/tests/__init__.py

@@ -10,12 +10,14 @@ import os
 import fs
 import json
 
+import json
 import numpy
 
 import xmodule
 import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
+from capa.util import convert_files_to_filenames
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
 from xmodule.graders import Score, aggregate_scores
@@ -32,7 +34,7 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue=None,
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
     is_staff=False,
     node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
 )
@@ -280,7 +282,6 @@ class StringResponseWithHintTest(unittest.TestCase):
 class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
-
     '''
     def test_update_score(self):
         problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
@@ -293,9 +294,14 @@ class CodeResponseTest(unittest.TestCase):
         for i in range(numAnswers):
             old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuekey=1000 + i))
 
-        # Message format inherited from ExternalResponse
-        correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
-        incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
+        # TODO: Message format inherited from ExternalResponse
+        #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
+        #incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
+
+        # New message format common to external graders
+        correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
+        incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
+
         xserver_msgs = {'correct': correct_score_msg,
                         'incorrect': incorrect_score_msg,
                         }
@@ -329,7 +335,18 @@ class CodeResponseTest(unittest.TestCase):
                         self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
                     else:
                         self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
-
+    
+    def test_convert_files_to_filenames(self):
+        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        fp = open(problem_file)
+        answers_with_file = {'1_2_1': 'String-based answer',
+                             '1_3_1': ['answer1', 'answer2', 'answer3'],
+                             '1_4_1': fp}
+        answers_converted = convert_files_to_filenames(answers_with_file)
+        self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
+        self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
+        self.assertEquals(answers_converted['1_4_1'], fp.name)
+        
 
 class ChoiceResponseTest(unittest.TestCase):
 
@@ -712,6 +729,6 @@ class ModuleProgressTest(unittest.TestCase):
     '''
     def test_xmodule_default(self):
         '''Make sure default get_progress exists, returns None'''
-        xm = x_module.XModule(i4xs, 'a://b/c/d/e', {})
+        xm = x_module.XModule(i4xs, 'a://b/c/d/e', None, {})
         p = xm.get_progress()
         self.assertEqual(p, None)

common/lib/xmodule/xmodule/tests/test_export.py

-from xmodule.modulestore.xml import XMLModuleStore
-from nose.tools import assert_equals
-from nose import SkipTest
-from tempfile import mkdtemp
+import unittest
+
 from fs.osfs import OSFS
+from nose.tools import assert_equals, assert_true
+from path import path
+from tempfile import mkdtemp
+from shutil import copytree
+
+from xmodule.modulestore.xml import XMLModuleStore
+
+# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/tests/
+# to   ~/mitx_all/mitx/common/test
+TEST_DIR = path(__file__).abspath().dirname()
+for i in range(4):
+    TEST_DIR = TEST_DIR.dirname()
+TEST_DIR = TEST_DIR / 'test'
+
+DATA_DIR = TEST_DIR / 'data'
+
+
+def strip_metadata(descriptor, key):
+    """
+    Recursively strips tag from all children.
+    """
+    print "strip {key} from {desc}".format(key=key, desc=descriptor.location.url())
+    descriptor.metadata.pop(key, None)
+    for d in descriptor.get_children():
+        strip_metadata(d, key)
+
+def strip_filenames(descriptor):
+    """
+    Recursively strips 'filename' from all children's definitions.
+    """
+    print "strip filename from {desc}".format(desc=descriptor.location.url())
+    descriptor.definition.pop('filename', None)
+    for d in descriptor.get_children():
+        strip_filenames(d)
+
+
+
+class RoundTripTestCase(unittest.TestCase):
+    '''Check that our test courses roundtrip properly'''
+    def check_export_roundtrip(self, data_dir, course_dir):
+
+        root_dir = path(mkdtemp())
+        print "Copying test course to temp dir {0}".format(root_dir)
+
+        data_dir = path(data_dir)
+        copytree(data_dir / course_dir, root_dir / course_dir)
+
+        print "Starting import"
+        initial_import = XMLModuleStore(root_dir, eager=True, course_dirs=[course_dir])
+
+        courses = initial_import.get_courses()
+        self.assertEquals(len(courses), 1)
+        initial_course = courses[0]
+
+        # export to the same directory--that way things like the custom_tags/ folder
+        # will still be there.
+        print "Starting export"
+        fs = OSFS(root_dir)
+        export_fs = fs.makeopendir(course_dir)
+
+        xml = initial_course.export_to_xml(export_fs)
+        with export_fs.open('course.xml', 'w') as course_xml:
+            course_xml.write(xml)
+
+        print "Starting second import"
+        second_import = XMLModuleStore(root_dir, eager=True, course_dirs=[course_dir])
+
+        courses2 = second_import.get_courses()
+        self.assertEquals(len(courses2), 1)
+        exported_course = courses2[0]
+
+        print "Checking course equality"
+        # HACK: data_dir metadata tags break equality because they
+        # aren't real metadata, and depend on paths. Remove them.
+        strip_metadata(initial_course, 'data_dir')
+        strip_metadata(exported_course, 'data_dir')
+
+        # HACK: filenames change when changing file formats
+        # during imports from old-style courses.  Ignore them.
+        strip_filenames(initial_course)
+        strip_filenames(exported_course)
 
+        self.assertEquals(initial_course, exported_course)
 
-def check_export_roundtrip(data_dir):
-    print "Starting import"
-    initial_import = XMLModuleStore('org', 'course', data_dir, eager=True)
-    initial_course = initial_import.course
+        print "Checking key equality"
+        self.assertEquals(sorted(initial_import.modules.keys()),
+                          sorted(second_import.modules.keys()))
 
-    print "Starting export"
-    export_dir = mkdtemp()
-    fs = OSFS(export_dir)
-    xml = initial_course.export_to_xml(fs)
-    with fs.open('course.xml', 'w') as course_xml:
-        course_xml.write(xml)
+        print "Checking module equality"
+        for location in initial_import.modules.keys():
+            print "Checking", location
+            if location.category == 'html':
+                print ("Skipping html modules--they can't import in"
+                       " final form without writing files...")
+                continue
+            self.assertEquals(initial_import.modules[location],
+                              second_import.modules[location])
 
-    print "Starting second import"
-    second_import = XMLModuleStore('org', 'course', export_dir, eager=True)
 
-    print "Checking key equality"
-    assert_equals(initial_import.modules.keys(), second_import.modules.keys())
+    def setUp(self):
+        self.maxDiff = None
 
-    print "Checking module equality"
-    for location in initial_import.modules.keys():
-        print "Checking", location
-        assert_equals(initial_import.modules[location], second_import.modules[location])
+    def test_toy_roundtrip(self):
+        self.check_export_roundtrip(DATA_DIR, "toy")
 
+    def test_simple_roundtrip(self):
+        self.check_export_roundtrip(DATA_DIR, "simple")
 
-def test_toy_roundtrip():
-    dir = ""
-    # TODO: add paths and make this run.
-    raise SkipTest()
-    check_export_roundtrip(dir)
+    def test_full_roundtrip(self):
+        self.check_export_roundtrip(DATA_DIR, "full")

common/lib/xmodule/xmodule/tests/test_import.py

@@ -5,10 +5,14 @@ from fs.memoryfs import MemoryFS
 from lxml import etree
 
 from xmodule.x_module import XMLParsingSystem, XModuleDescriptor
+from xmodule.xml_module import is_pointer_tag
 from xmodule.errortracker import make_error_tracker
 from xmodule.modulestore import Location
+from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 
+from .test_export import DATA_DIR
+
 ORG = 'test_org'
 COURSE = 'test_course'
 
@@ -46,22 +50,17 @@ class DummySystem(XMLParsingSystem):
             raise Exception("Shouldn't be called")
 
 
-
-
 class ImportTestCase(unittest.TestCase):
     '''Make sure module imports work properly, including for malformed inputs'''
-
-
     @staticmethod
     def get_system():
         '''Get a dummy system'''
         return DummySystem()
 
     def test_fallback(self):
-        '''Make sure that malformed xml loads as an ErrorDescriptor.'''
+        '''Check that malformed xml loads as an ErrorDescriptor.'''
 
         bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
-
         system = self.get_system()
 
         descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course',
@@ -70,6 +69,22 @@ class ImportTestCase(unittest.TestCase):
         self.assertEqual(descriptor.__class__.__name__,
                          'ErrorDescriptor')
 
+
+    def test_unique_url_names(self):
+        '''Check that each error gets its very own url_name'''
+        bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
+        bad_xml2 = '''<sequential url_name="oops"><video url="hi"></sequential>'''
+        system = self.get_system()
+
+        descriptor1 = XModuleDescriptor.load_from_xml(bad_xml, system, 'org',
+                                                      'course', None)
+
+        descriptor2 = XModuleDescriptor.load_from_xml(bad_xml2, system, 'org',
+                                                      'course', None)
+
+        self.assertNotEqual(descriptor1.location, descriptor2.location)
+
+
     def test_reimport(self):
         '''Make sure an already-exported error xml tag loads properly'''
 
@@ -111,30 +126,84 @@ class ImportTestCase(unittest.TestCase):
         xml_out = etree.fromstring(xml_str_out)
         self.assertEqual(xml_out.tag, 'sequential')
 
-    def test_metadata_inherit(self):
-        """Make sure metadata inherits properly"""
+    def test_metadata_import_export(self):
+        """Two checks:
+            - unknown metadata is preserved across import-export
+            - inherited metadata doesn't leak to children.
+        """
         system = self.get_system()
-        v = "1 hour"
-        start_xml = '''<course graceperiod="{grace}" url_name="test1" display_name="myseq">
-                      <chapter url="hi" url_name="ch" display_name="CH">
-                       <html url_name="h" display_name="H">Two houses, ...</html></chapter>
-                   </course>'''.format(grace=v)
+        v = '1 hour'
+        org = 'foo'
+        course = 'bbhh'
+        url_name = 'test1'
+        start_xml = '''
+        <course org="{org}" course="{course}"
+                graceperiod="{grace}" url_name="{url_name}" unicorn="purple">
+            <chapter url="hi" url_name="ch" display_name="CH">
+                <html url_name="h" display_name="H">Two houses, ...</html>
+            </chapter>
+        </course>'''.format(grace=v, org=org, course=course, url_name=url_name)
         descriptor = XModuleDescriptor.load_from_xml(start_xml, system,
-                                                     'org', 'course')
+                                                     org, course)
 
-        print "Errors: {0}".format(system.errorlog.errors)
         print descriptor, descriptor.metadata
         self.assertEqual(descriptor.metadata['graceperiod'], v)
+        self.assertEqual(descriptor.metadata['unicorn'], 'purple')
 
-        # Check that the child inherits correctly
+        # Check that the child inherits graceperiod correctly
         child = descriptor.get_children()[0]
         self.assertEqual(child.metadata['graceperiod'], v)
 
-        # Now export and see if the chapter tag has a graceperiod attribute
+        # check that the child does _not_ inherit any unicorns
+        self.assertTrue('unicorn' not in child.metadata)
+
+        # Now export and check things
         resource_fs = MemoryFS()
         exported_xml = descriptor.export_to_xml(resource_fs)
+
+        # Check that the exported xml is just a pointer
         print "Exported xml:", exported_xml
-        root = etree.fromstring(exported_xml)
-        chapter_tag = root[0]
-        self.assertEqual(chapter_tag.tag, 'chapter')
-        self.assertFalse('graceperiod' in chapter_tag.attrib)
+        pointer = etree.fromstring(exported_xml)
+        self.assertTrue(is_pointer_tag(pointer))
+        # but it's a special case course pointer
+        self.assertEqual(pointer.attrib['course'], course)
+        self.assertEqual(pointer.attrib['org'], org)
+
+        # Does the course still have unicorns?
+        with resource_fs.open('course/{url_name}.xml'.format(url_name=url_name)) as f:
+            course_xml = etree.fromstring(f.read())
+
+        self.assertEqual(course_xml.attrib['unicorn'], 'purple')
+
+        # the course and org tags should be _only_ in the pointer
+        self.assertTrue('course' not in course_xml.attrib)
+        self.assertTrue('org' not in course_xml.attrib)
+
+        # did we successfully strip the url_name from the definition contents?
+        self.assertTrue('url_name' not in course_xml.attrib)
+
+        # Does the chapter tag now have a graceperiod attribute?
+        # hardcoded path to child
+        with resource_fs.open('chapter/ch.xml') as f:
+            chapter_xml = etree.fromstring(f.read())
+        self.assertEqual(chapter_xml.tag, 'chapter')
+        self.assertFalse('graceperiod' in chapter_xml.attrib)
+
+    def test_metadata_inherit(self):
+        """Make sure that metadata is inherited properly"""
+
+        print "Starting import"
+        initial_import = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+
+        courses = initial_import.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+
+        def check_for_key(key, node):
+            "recursive check for presence of key"
+            print "Checking {}".format(node.location.url())
+            self.assertTrue(key in node.metadata)
+            for c in node.get_children():
+                check_for_key(key, c)
+
+        check_for_key('graceperiod', course)

common/lib/xmodule/xmodule/util/decorators.py

+
+
+def lazyproperty(fn):
+    """
+    Use this decorator for lazy generation of properties that
+    are expensive to compute. From http://stackoverflow.com/a/3013910/86828
+    
+    
+    Example:
+    class Test(object):
+
+        @lazyproperty
+        def a(self):
+            print 'generating "a"'
+            return range(5)
+    
+    Interactive Session:
+    >>> t = Test()
+    >>> t.__dict__
+    {}
+    >>> t.a
+    generating "a"
+    [0, 1, 2, 3, 4]
+    >>> t.__dict__
+    {'_lazy_a': [0, 1, 2, 3, 4]}
+    >>> t.a
+    [0, 1, 2, 3, 4]
+    """
+    
+    attr_name = '_lazy_' + fn.__name__
+    @property
+    def _lazyprop(self):
+        if not hasattr(self, attr_name):
+            setattr(self, attr_name, fn(self))
+        return getattr(self, attr_name)
+    return _lazyprop
\ No newline at end of file

common/lib/xmodule/xmodule/vertical_module.py

@@ -10,8 +10,8 @@ class_priority = ['video', 'problem']
 class VerticalModule(XModule):
     ''' Layout module for laying out submodules vertically.'''
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, descriptor, instance_state, shared_state, **kwargs)
         self.contents = None
 
     def get_html(self):

common/lib/xmodule/xmodule/video_module.py

@@ -23,9 +23,9 @@ class VideoModule(XModule):
     css = {'scss': [resource_string(__name__, 'css/video/display.scss')]}
     js_module_name = "Video"
 
-    def __init__(self, system, location, definition,
+    def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition,
+        XModule.__init__(self, system, location, definition, descriptor,
                          instance_state, shared_state, **kwargs)
         xmltree = etree.fromstring(self.definition['data'])
         self.youtube = xmltree.get('youtube')
@@ -80,3 +80,5 @@ class VideoModule(XModule):
 
 class VideoDescriptor(RawDescriptor):
     module_class = VideoModule
+    
+    stores_state = True

common/lib/xmodule/xmodule/x_module.py

@@ -6,6 +6,7 @@ from fs.errors import ResourceNotFoundError
 from functools import partial
 from lxml import etree
 from lxml.etree import XMLSyntaxError
+from pprint import pprint
 
 from xmodule.modulestore import Location
 from xmodule.errortracker import exc_info_to_str
@@ -142,7 +143,7 @@ class XModule(HTMLSnippet):
     # in the module
     icon_class = 'other'
 
-    def __init__(self, system, location, definition,
+    def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
         '''
         Construct a new xmodule
@@ -165,6 +166,10 @@ class XModule(HTMLSnippet):
             'children': is a list of Location-like values for child modules that
                 this module depends on
 
+        descriptor: the XModuleDescriptor that this module is an instance of.
+            TODO (vshnayder): remove the definition parameter and location--they
+            can come from the descriptor.
+
         instance_state: A string of serialized json that contains the state of
                 this module for current student accessing the system, or None if
                 no state has been saved
@@ -188,6 +193,7 @@ class XModule(HTMLSnippet):
         self.system = system
         self.location = Location(location)
         self.definition = definition
+        self.descriptor = descriptor
         self.instance_state = instance_state
         self.shared_state = shared_state
         self.id = self.location.url()
@@ -303,10 +309,18 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
     entry_point = "xmodule.v1"
     module_class = XModule
 
+    # Attributes for inpsection of the descriptor
+    stores_state = False # Indicates whether the xmodule state should be
+    # stored in a database (independent of shared state)
+    has_score = False # This indicates whether the xmodule is a problem-type.
+    # It should respond to max_score() and grade(). It can be graded or ungraded
+    # (like a practice problem).
+
     # A list of metadata that this module can inherit from its parent module
     inheritable_metadata = (
         'graded', 'start', 'due', 'graceperiod', 'showanswer', 'rerandomize',
-
+        # TODO (ichuang): used for Fall 2012 xqa server access
+        'xqa_key',
         # TODO: This is used by the XMLModuleStore to provide for locations for
         # static files, and will need to be removed when that code is removed
         'data_dir'
@@ -390,6 +404,18 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
         return dict((k,v) for k,v in self.metadata.items()
                     if k not in self._inherited_metadata)
 
+    @staticmethod
+    def compute_inherited_metadata(node):
+        """Given a descriptor, traverse all of its descendants and do metadata
+        inheritance.  Should be called on a CourseDescriptor after importing a
+        course.
+
+        NOTE: This means that there is no such thing as lazy loading at the
+        moment--this accesses all the children."""
+        for c in node.get_children():
+            c.inherit_metadata(node.metadata)
+            XModuleDescriptor.compute_inherited_metadata(c)
+
     def inherit_metadata(self, metadata):
         """
         Updates this module with metadata inherited from a containing module.
@@ -410,6 +436,9 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
             self._child_instances = []
             for child_loc in self.definition.get('children', []):
                 child = self.system.load_item(child_loc)
+                # TODO (vshnayder): this should go away once we have
+                # proper inheritance support in mongo.  The xml
+                # datastore does all inheritance on course load.
                 child.inherit_metadata(self.metadata)
                 self._child_instances.append(child)
 
@@ -426,6 +455,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
             system,
             self.location,
             self.definition,
+            self,
             metadata=self.metadata
         )
 
@@ -493,7 +523,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
             # Put import here to avoid circular import errors
             from xmodule.error_module import ErrorDescriptor
             msg = "Error loading from xml."
-            log.exception(msg)
+            log.warning(msg + " " + str(err))
             system.error_tracker(msg)
             err_msg = msg + "\n" + exc_info_to_str(sys.exc_info())
             descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course,
@@ -550,9 +580,9 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
 
         if not eq:
             for attr in self.equality_attributes:
-                print(getattr(self, attr, None),
-                      getattr(other, attr, None),
-                      getattr(self, attr, None) == getattr(other, attr, None))
+                pprint((getattr(self, attr, None),
+                       getattr(other, attr, None),
+                       getattr(self, attr, None) == getattr(other, attr, None)))
 
         return eq
 
@@ -586,9 +616,10 @@ class DescriptorSystem(object):
                try:
                   x = access_some_resource()
                   check_some_format(x)
-               except SomeProblem:
-                  msg = 'Grommet {0} is broken'.format(x)
-                  log.exception(msg) # don't rely on handler to log
+               except SomeProblem as err:
+                  msg = 'Grommet {0} is broken: {1}'.format(x, str(err))
+                  log.warning(msg)  # don't rely on tracker to log
+                        # NOTE: we generally don't want content errors logged as errors
                   self.system.error_tracker(msg)
                   # work around
                   return 'Oops, couldn't load grommet'
@@ -643,7 +674,7 @@ class ModuleSystem(object):
                  user=None,
                  filestore=None,
                  debug=False,
-                 xqueue = None,
+                 xqueue=None,
                  is_staff=False,
                  node_path=""):
         '''
@@ -668,7 +699,7 @@ class ModuleSystem(object):
         filestore - A filestore ojbect.  Defaults to an instance of OSFS based
                          at settings.DATA_DIR.
 
-        xqueue - Dict containing XqueueInterface object, as well as parameters 
+        xqueue - Dict containing XqueueInterface object, as well as parameters
                     for the specific StudentModule
 
         replace_urls - TEMPORARY - A function like static_replace.replace_urls

common/test/data/simple/course.xml

@@ -2,7 +2,7 @@
   <chapter name="Overview">
     <video name="Welcome" youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8"/>
     <videosequence format="Lecture Sequence" name="A simple sequence">
-      <html id="toylab" filename="toylab"/>
+      <html name="toylab" filename="toylab"/>
       <video name="S0V1: Video Resources" youtube="0.75:EuzkdzfR0i8,1.0:1bK-WdDi6Qw,1.25:0v1VzoDVUTM,1.50:Bxk_-ZJb240"/>
     </videosequence>
     <section name="Lecture 2">
@@ -15,7 +15,7 @@
   <chapter name="Chapter 2">
     <section name="Problem Set 1">
       <sequential>
-        <problem type="lecture" showanswer="attempted" rerandomize="true" title="A simple coding problem" name="Simple coding problem" filename="ps01-simple"/>
+        <problem type="lecture" showanswer="attempted" rerandomize="true" display_name="A simple coding problem" name="Simple coding problem" filename="ps01-simple"/>
       </sequential>
     </section>
     <video name="Lost Video" youtube="1.0:TBvX7HzxexQ"/>

doc/development.md

@@ -28,3 +28,40 @@ Check out the course data directories that you want to work with into the
 Replace `../data` with your `GITHUB_REPO_ROOT` if it's not the default value.
 
 This will import all courses in your data directory into mongodb
+
+## Unit tests
+
+This runs all the tests (long, uses collectstatic):
+
+    rake test
+
+If if you aren't changing static files, can run `rake test` once, then run
+
+    rake fasttest_{lms,cms}
+
+xmodule can be tested independently, with this:
+
+    rake test_common/lib/xmodule
+
+To see all available rake commands, do this:
+
+    rake -T
+
+To run a single django test class:
+
+    django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/courseware/tests/tests.py:TestViewAuth
+
+To run a single django test:
+
+    django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/courseware/tests/tests.py:TestViewAuth.test_dark_launch
+
+
+To run a single nose test file:
+
+    nosetests common/lib/xmodule/xmodule/tests/test_stringify.py
+
+To run a single nose test:
+
+    nosetests common/lib/xmodule/xmodule/tests/test_stringify.py:test_stringify
+
+

lms/djangoapps/certificates/views.py

@@ -52,7 +52,7 @@ def certificate_request(request):
             return return_error(survey_response['error'])
 
         grade = None
-        student_gradesheet = grades.grade_sheet(request.user)
+        student_gradesheet = grades.grade(request.user, request, course)
         grade = student_gradesheet['grade']
 
         if not grade:
@@ -65,7 +65,7 @@ def certificate_request(request):
     else:
         #This is not a POST, we should render the page with the form
 
-        grade_sheet = grades.grade_sheet(request.user)
+        student_gradesheet = grades.grade(request.user, request, course)
         certificate_state = certificate_state_for_student(request.user, grade_sheet['grade'])
 
         if certificate_state['state'] != "requestable":

lms/djangoapps/courseware/management/commands/check_course.py

@@ -78,8 +78,8 @@ class Command(BaseCommand):
 
         # TODO (cpennington): Get coursename in a legitimate way
         course_location = 'i4x://edx/6002xs12/course/6.002_Spring_2012'
-        student_module_cache = StudentModuleCache(sample_user, modulestore().get_item(course_location))
-        (course, _, _, _) = get_module(sample_user, None, course_location, student_module_cache)
+        student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(sample_user, modulestore().get_item(course_location))
+        course = get_module(sample_user, None, course_location, student_module_cache)
 
         to_run = [
             #TODO (vshnayder) : make check_rendering work (use module_render.py),

lms/djangoapps/courseware/models.py

@@ -67,17 +67,19 @@ class StudentModuleCache(object):
     """
     A cache of StudentModules for a specific student
     """
-    def __init__(self, user, descriptor, depth=None):
+    def __init__(self, user, descriptors):
         '''
-        Find any StudentModule objects that are needed by any child modules of the
-        supplied descriptor. Avoids making multiple queries to the database
-
-        descriptor: An XModuleDescriptor
-        depth is the number of levels of descendent modules to load StudentModules for, in addition to
-            the supplied descriptor. If depth is None, load all descendent StudentModules
+        Find any StudentModule objects that are needed by any descriptor
+        in descriptors. Avoids making multiple queries to the database.
+        Note: Only modules that have store_state = True or have shared
+        state will have a StudentModule.
+        
+        Arguments
+        user: The user for which to fetch maching StudentModules
+        descriptors: An array of XModuleDescriptors.
         '''
         if user.is_authenticated():
-            module_ids = self._get_module_state_keys(descriptor, depth)
+            module_ids = self._get_module_state_keys(descriptors) 
 
             # This works around a limitation in sqlite3 on the number of parameters
             # that can be put into a single query
@@ -91,27 +93,53 @@ class StudentModuleCache(object):
 
         else:
             self.cache = []
-
-    def _get_module_state_keys(self, descriptor, depth):
-        '''
-        Get a list of the state_keys needed for StudentModules
-        required for this module descriptor
-
+    
+    
+    @classmethod
+    def cache_for_descriptor_descendents(cls, user, descriptor, depth=None, descriptor_filter=lambda descriptor: True):
+        """
         descriptor: An XModuleDescriptor
         depth is the number of levels of descendent modules to load StudentModules for, in addition to
             the supplied descriptor. If depth is None, load all descendent StudentModules
+        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule 
+            should be cached
+        """
+        
+        def get_child_descriptors(descriptor, depth, descriptor_filter):
+            if descriptor_filter(descriptor):
+                descriptors = [descriptor]
+            else:
+                descriptors = []
+            
+            if depth is None or depth > 0:
+                new_depth = depth - 1 if depth is not None else depth
+
+                for child in descriptor.get_children():
+                    descriptors.extend(get_child_descriptors(child, new_depth, descriptor_filter))
+                
+            return descriptors
+        
+        
+        descriptors = get_child_descriptors(descriptor, depth, descriptor_filter)
+        
+        return StudentModuleCache(user, descriptors)    
+    
+    def _get_module_state_keys(self, descriptors):
         '''
-        keys = [descriptor.location.url()]
-
-        shared_state_key = getattr(descriptor, 'shared_state_key', None)
-        if shared_state_key is not None:
-            keys.append(shared_state_key)
-
-        if depth is None or depth > 0:
-            new_depth = depth - 1 if depth is not None else depth
-
-            for child in descriptor.get_children():
-                keys.extend(self._get_module_state_keys(child, new_depth))
+        Get a list of the state_keys needed for StudentModules
+        required for this module descriptor
+        
+        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule 
+            should be cached
+        '''
+        keys = []
+        for descriptor in descriptors:
+            if descriptor.stores_state:
+                keys.append(descriptor.location.url())
+            
+            shared_state_key = getattr(descriptor, 'shared_state_key', None)
+            if shared_state_key is not None:
+                keys.append(shared_state_key)
 
         return keys
 

lms/djangoapps/heartbeat/urls.py

-from django.conf.urls.defaults import *
+from django.conf.urls import *
 
 urlpatterns = patterns('',
     url(r'^$', 'heartbeat.views.heartbeat', name='heartbeat'),

lms/djangoapps/simplewiki/urls.py

-from django.conf.urls.defaults import patterns, url
+from django.conf.urls import patterns, url
 
 namespace_regex = r"[a-zA-Z\d._-]+"
 article_slug = r'/(?P<article_path>' + namespace_regex + r'/[a-zA-Z\d_-]*)'

lms/djangoapps/simplewiki/views.py

@@ -51,7 +51,7 @@ def update_template_dictionary(dictionary, request=None, course=None, article=No
 
 
 def view(request, article_path, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, article_path, course)
     if err:
@@ -67,7 +67,7 @@ def view(request, article_path, course_id=None):
 
 
 def view_revision(request, revision_number, article_path, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, article_path, course)
     if err:
@@ -91,7 +91,7 @@ def view_revision(request, revision_number, article_path, course_id=None):
 
 
 def root_redirect(request, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     #TODO: Add a default namespace to settings.
     namespace = course.wiki_namespace if course else "edX"
@@ -109,7 +109,7 @@ def root_redirect(request, course_id=None):
 
 
 def create(request, article_path, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     article_path_components = article_path.split('/')
 
@@ -170,7 +170,7 @@ def create(request, article_path, course_id=None):
 
 
 def edit(request, article_path, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, article_path, course)
     if err:
@@ -218,7 +218,7 @@ def edit(request, article_path, course_id=None):
 
 
 def history(request, article_path, page=1, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, article_path, course)
     if err:
@@ -300,7 +300,7 @@ def history(request, article_path, page=1, course_id=None):
 
 
 def revision_feed(request, page=1, namespace=None, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     page_size = 10
 
@@ -333,7 +333,7 @@ def revision_feed(request, page=1, namespace=None, course_id=None):
 
 
 def search_articles(request, namespace=None, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     # blampe: We should check for the presence of other popular django search
     # apps and use those if possible. Only fall back on this as a last resort.
@@ -382,7 +382,7 @@ def search_articles(request, namespace=None, course_id=None):
 
 
 def search_add_related(request, course_id, slug, namespace):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, slug, namespace if namespace else course_id)
     if err:
@@ -415,7 +415,7 @@ def search_add_related(request, course_id, slug, namespace):
 
 
 def add_related(request, course_id, slug, namespace):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, slug, namespace if namespace else course_id)
     if err:
@@ -439,7 +439,7 @@ def add_related(request, course_id, slug, namespace):
 
 
 def remove_related(request, course_id, namespace, slug, related_id):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     (article, err) = get_article(request, slug, namespace if namespace else course_id)
 
@@ -462,7 +462,7 @@ def remove_related(request, course_id, namespace, slug, related_id):
 
 
 def random_article(request, course_id=None):
-    course = check_course(course_id, course_required=False)
+    course = check_course(request.user, course_id, course_required=False)
 
     from random import randint
     num_arts = Article.objects.count()

lms/djangoapps/staticbook/views.py

@@ -2,12 +2,14 @@ from django.contrib.auth.decorators import login_required
 from mitxmako.shortcuts import render_to_response
 
 from courseware.courses import check_course
-
+from lxml import etree
 
 @login_required
 def index(request, course_id, page=0):
-    course = check_course(course_id)
-    return render_to_response('staticbook.html', {'page': int(page), 'course': course})
+    course = check_course(request.user, course_id)
+    raw_table_of_contents = open('lms/templates/book_toc.xml', 'r') # TODO: This will need to come from S3
+    table_of_contents = etree.parse(raw_table_of_contents).getroot()
+    return render_to_response('staticbook.html', {'page': int(page), 'course': course, 'table_of_contents': table_of_contents})
 
 
 def index_shifted(request, course_id, page):

lms/envs/common.py

@@ -48,6 +48,7 @@ MITX_FEATURES = {
     ## DO NOT SET TO True IN THIS FILE
     ## Doing so will cause all courses to be released on production
     'DISABLE_START_DATES': False,  # When True, all courses will be active, regardless of start date
+    'DARK_LAUNCH': False,  # When True, courses will be active for staff only
 
     'ENABLE_TEXTBOOK' : True,
     'ENABLE_DISCUSSION' : True,
@@ -55,6 +56,8 @@ MITX_FEATURES = {
     'ENABLE_SQL_TRACKING_LOGS': False,
     'ENABLE_LMS_MIGRATION': False,
 
+    'DISABLE_LOGIN_BUTTON': False,	# used in systems where login is automatic, eg MIT SSL
+
     # extrernal access methods
     'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
     'AUTH_USE_OPENID': False,
@@ -121,7 +124,7 @@ TEMPLATE_CONTEXT_PROCESSORS = (
     'django.contrib.messages.context_processors.messages',
     #'django.core.context_processors.i18n',
     'askbot.user_messages.context_processors.user_messages',#must be before auth
-    'django.core.context_processors.auth', #this is required for admin
+    'django.contrib.auth.context_processors.auth', #this is required for admin
     'django.core.context_processors.csrf', #necessary for csrf protection
 )
 
@@ -169,6 +172,9 @@ COURSE_SETTINGS =  {'6.002x_Fall_2012': {'number' : '6.002x',
                                           }
                     }
 
+# IP addresses that are allowed to reload the course, etc.
+# TODO (vshnayder): Will probably need to change as we get real access control in.
+LMS_MIGRATION_ALLOWED_IPS = []
 
 ############################### XModule Store ##################################
 MODULESTORE = {
@@ -182,6 +188,9 @@ MODULESTORE = {
     }
 }
 
+############################ SIGNAL HANDLERS ################################
+# This is imported to register the exception signal handling that logs exceptions
+import monitoring.exceptions  # noqa
 
 ############################### DJANGO BUILT-INS ###############################
 # Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
@@ -294,7 +303,6 @@ TEMPLATE_LOADERS = (
 )
 
 MIDDLEWARE_CLASSES = (
-    'util.middleware.ExceptionLoggingMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',

lms/envs/dev.py

@@ -62,6 +62,7 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 ################################ LMS Migration #################################
 MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
+MITX_FEATURES['USE_XQA_SERVER'] = 'http://xqa:server@content-qa.mitx.mit.edu/xqa'
 
 LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 

lms/envs/dev_ike.py

@@ -10,9 +10,27 @@ sessions. Assumes structure:
 from .common import *
 from .logsettings import get_logger_config
 from .dev import *
+import socket
 
 WIKI_ENABLED = False
 MITX_FEATURES['ENABLE_TEXTBOOK'] = False
 MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 
+myhost = socket.gethostname()
+if ('edxvm' in myhost) or ('ocw' in myhost):
+    MITX_FEATURES['DISABLE_LOGIN_BUTTON'] = True	# auto-login with MIT certificate
+    MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa'	# needs to be ssl or browser blocks it
+
+if ('domU' in myhost):
+    EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+    MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu'	# nonempty string = address for all activation emails
+
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')	# django 1.4 for nginx ssl proxy
+
+#-----------------------------------------------------------------------------
+# disable django debug toolbars
+
+INSTALLED_APPS = tuple([ app for app in INSTALLED_APPS if not app.startswith('debug_toolbar') ])
+MIDDLEWARE_CLASSES = tuple([ mcl for mcl in MIDDLEWARE_CLASSES if not mcl.startswith('debug_toolbar') ])
+TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])

lms/envs/devgroups/courses.py

+from ..dev import *
+
+CLASSES_TO_DBS = {
+    'BerkeleyX/CS169.1x/2012_Fall' : "cs169.db",
+    'BerkeleyX/CS188.1x/2012_Fall' : "cs188_1.db",
+    'HarvardX/CS50x/2012' : "cs50.db",
+    'HarvardX/PH207x/2012_Fall' : "ph207.db",
+    'MITx/3.091x/2012_Fall' : "3091.db",
+    'MITx/6.002x/2012_Fall' : "6002.db",
+    'MITx/6.00x/2012_Fall' : "600.db",
+}
+
+
+CACHES = {
+   'default': {
+       'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+       'LOCATION': '127.0.0.1:11211',
+       'KEY_FUNCTION': 'util.memcache.safe_key',
+   },
+   'general': {
+       'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+       'LOCATION': '127.0.0.1:11211',
+       'KEY_PREFIX' : 'general',
+       'VERSION' : 5,
+       'KEY_FUNCTION': 'util.memcache.safe_key',
+   }
+}
+
+SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
+
+
+def path_for_db(db_name):
+    return ENV_ROOT / "db" / db_name
+
+def course_db_for(course_id):
+    db_name = CLASSES_TO_DBS[course_id]
+    return {
+               'default' : {
+                    'ENGINE' : 'django.db.backends.sqlite3',
+                    'NAME' : path_for_db(db_name)
+                }
+            }
+

lms/envs/devgroups/h_cs50.py

+from .courses import *
+
+DATABASES = course_db_for('HarvardX/CS50x/2012')
\ No newline at end of file

lms/envs/devgroups/m_6002.py

+from .courses import *
+
+DATABASES = course_db_for('MITx/6.002x/2012_Fall')
\ No newline at end of file

lms/envs/devgroups/portal.py

+"""
+Note that for this to work at all, you must have memcached running (or you won't
+get shared sessions)
+"""
+from courses import *
+
+# Move this to a shared file later:
+for class_id, db_name in CLASSES_TO_DBS.items():
+    DATABASES[class_id] = {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': path_for_db(db_name)
+    }
+

lms/envs/test.py

@@ -25,6 +25,7 @@ SOUTH_TESTS_MIGRATE = False # To disable migrations and use syncdb instead
 # Nose Test Runner
 INSTALLED_APPS += ('django_nose',)
 NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html',
+             # '-v', '--pdb',     # When really stuck, uncomment to start debugger on error
              '--cover-inclusive', '--cover-html-dir',
              os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
 for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
@@ -66,6 +67,17 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': PROJECT_ROOT / "db" / "mitx.db",
+    },
+
+    # The following are for testing purposes...
+    'edX/toy/2012_Fall': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "course1.db",
+    },
+    
+    'edx/full/6.002_Spring_2012': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': ENV_ROOT / "db" / "course2.db",
     }
 }
 

lms/static/admin/css/base.css

@@ -189,6 +189,10 @@ p.mini {
     color: #999;
 }
 
+img.help-tooltip {
+    cursor: help;
+}
+
 p img, h1 img, h2 img, h3 img, h4 img, td img {
     vertical-align: middle;
 }
@@ -259,7 +263,7 @@ tfoot td {
     color: #666;
     padding: 2px 5px;
     font-size: 11px;
-    background: #e1e1e1 url(../img/admin/nav-bg.gif) top left repeat-x;
+    background: #e1e1e1 url(../img/nav-bg.gif) top left repeat-x;
     border-left: 1px solid #ddd;
     border-bottom: 1px solid #ddd;
 }
@@ -305,25 +309,84 @@ tr.alt {
 
 /* SORTABLE TABLES */
 
+thead th {
+    padding: 2px 5px;
+    line-height: normal;
+}
+
 thead th a:link, thead th a:visited {
     color: #666;
+}
+
+thead th.sorted {
+    background: #c5c5c5 url(../img/nav-bg-selected.gif) top left repeat-x;
+}
+
+table thead th .text span {
+    padding: 2px 5px;
+    display:block;
+}
+
+table thead th .text a {
+    display: block;
+    cursor: pointer;
+    padding: 2px 5px;
+}
+
+table thead th.sortable:hover {
+    background: white url(../img/nav-bg-reverse.gif) 0 -5px repeat-x;
+}
+
+thead th.sorted a.sortremove {
+    visibility: hidden;
+}
+
+table thead th.sorted:hover a.sortremove {
+    visibility: visible;
+}
+
+table thead th.sorted .sortoptions {
     display: block;
+    padding: 4px 5px 0 5px;
+    float: right;
+    text-align: right;
+}
+
+table thead th.sorted .sortpriority {
+    font-size: .8em;
+    min-width: 12px;
+    text-align: center;
+    vertical-align: top;
+}
+
+table thead th.sorted .sortoptions a {
+    width: 14px;
+    height: 12px;
+    display: inline-block;
+}
+
+table thead th.sorted .sortoptions a.sortremove {
+    background: url(../img/sorting-icons.gif) -4px -5px no-repeat;
+}
+
+table thead th.sorted .sortoptions a.sortremove:hover {
+    background: url(../img/sorting-icons.gif) -4px -27px no-repeat;
 }
 
-table thead th.sorted {
-    background-position: bottom left !important;
+table thead th.sorted .sortoptions a.ascending {
+    background: url(../img/sorting-icons.gif) -5px -50px no-repeat;
 }
 
-table thead th.sorted a {
-    padding-right: 13px;
+table thead th.sorted .sortoptions a.ascending:hover {
+    background: url(../img/sorting-icons.gif) -5px -72px no-repeat;
 }
 
-table thead th.ascending a {
-    background: url(../img/admin/arrow-up.gif) right .4em no-repeat;
+table thead th.sorted .sortoptions a.descending {
+    background: url(../img/sorting-icons.gif) -5px -94px no-repeat;
 }
 
-table thead th.descending a {
-    background: url(../img/admin/arrow-down.gif) right .4em no-repeat;
+table thead th.sorted .sortoptions a.descending:hover {
+    background: url(../img/sorting-icons.gif) -5px -115px no-repeat;
 }
 
 /* ORDERABLE TABLES */
@@ -334,7 +397,7 @@ table.orderable tbody tr td:hover {
 
 table.orderable tbody tr td:first-child {
     padding-left: 14px;
-    background-image: url(../img/admin/nav-bg-grabber.gif);
+    background-image: url(../img/nav-bg-grabber.gif);
     background-repeat: repeat-y;
 }
 
@@ -364,7 +427,7 @@ input[type=text], input[type=password], textarea, select, .vTextField {
 /* FORM BUTTONS */
 
 .button, input[type=submit], input[type=button], .submit-row input {
-    background: white url(../img/admin/nav-bg.gif) bottom repeat-x;
+    background: white url(../img/nav-bg.gif) bottom repeat-x;
     padding: 3px 5px;
     color: black;
     border: 1px solid #bbb;
@@ -372,31 +435,31 @@ input[type=text], input[type=password], textarea, select, .vTextField {
 }
 
 .button:active, input[type=submit]:active, input[type=button]:active {
-    background-image: url(../img/admin/nav-bg-reverse.gif);
+    background-image: url(../img/nav-bg-reverse.gif);
     background-position: top;
 }
 
 .button[disabled], input[type=submit][disabled], input[type=button][disabled] {
-	background-image: url(../img/admin/nav-bg.gif);
+	background-image: url(../img/nav-bg.gif);
 	background-position: bottom;
 	opacity: 0.4;
 }
 
 .button.default, input[type=submit].default, .submit-row input.default {
     border: 2px solid #5b80b2;
-    background: #7CA0C7 url(../img/admin/default-bg.gif) bottom repeat-x;
+    background: #7CA0C7 url(../img/default-bg.gif) bottom repeat-x;
     font-weight: bold;
     color: white;
     float: right;
 }
 
 .button.default:active, input[type=submit].default:active {
-    background-image: url(../img/admin/default-bg-reverse.gif);
+    background-image: url(../img/default-bg-reverse.gif);
     background-position: top;
 }
 
 .button[disabled].default, input[type=submit][disabled].default, input[type=button][disabled].default {
-	background-image: url(../img/admin/default-bg.gif);
+	background-image: url(../img/default-bg.gif);
 	background-position: bottom;
 	opacity: 0.4;
 }
@@ -433,7 +496,7 @@ input[type=text], input[type=password], textarea, select, .vTextField {
     font-size: 11px;
     text-align: left;
     font-weight: bold;
-    background: #7CA0C7 url(../img/admin/default-bg.gif) top left repeat-x;
+    background: #7CA0C7 url(../img/default-bg.gif) top left repeat-x;
     color: white;
 }
 
@@ -455,15 +518,15 @@ ul.messagelist li {
     margin: 0 0 3px 0;
     border-bottom: 1px solid #ddd;
     color: #666;
-    background: #ffc url(../img/admin/icon_success.gif) 5px .3em no-repeat;
+    background: #ffc url(../img/icon_success.gif) 5px .3em no-repeat;
 }
 
 ul.messagelist li.warning{
-    background-image: url(../img/admin/icon_alert.gif);
+    background-image: url(../img/icon_alert.gif);
 }
 
 ul.messagelist li.error{
-    background-image: url(../img/admin/icon_error.gif);
+    background-image: url(../img/icon_error.gif);
 }
 
 .errornote {
@@ -473,7 +536,7 @@ ul.messagelist li.error{
     margin: 0 0 3px 0;
     border: 1px solid red;
     color: red;
-    background: #ffc url(../img/admin/icon_error.gif) 5px .3em no-repeat;
+    background: #ffc url(../img/icon_error.gif) 5px .3em no-repeat;
 }
 
 ul.errorlist {
@@ -488,7 +551,7 @@ ul.errorlist {
     margin: 0 0 3px 0;
     border: 1px solid red;
     color: white;
-    background: red url(../img/admin/icon_alert.gif) 5px .3em no-repeat;
+    background: red url(../img/icon_alert.gif) 5px .3em no-repeat;
 }
 
 .errorlist li a {
@@ -524,7 +587,7 @@ div.system-message p.system-message-title {
     padding: 4px 5px 4px 25px;
     margin: 0;
     color: red;
-    background: #ffc url(../img/admin/icon_error.gif) 5px .3em no-repeat;
+    background: #ffc url(../img/icon_error.gif) 5px .3em no-repeat;
 }
 
 .description {
@@ -535,7 +598,7 @@ div.system-message p.system-message-title {
 /* BREADCRUMBS */
 
 div.breadcrumbs {
-    background: white url(../img/admin/nav-bg-reverse.gif) 0 -10px repeat-x;
+    background: white url(../img/nav-bg-reverse.gif) 0 -10px repeat-x;
     padding: 2px 8px 3px 8px;
     font-size: 11px;
     color: #999;
@@ -548,17 +611,17 @@ div.breadcrumbs {
 
 .addlink {
     padding-left: 12px;
-    background: url(../img/admin/icon_addlink.gif) 0 .2em no-repeat;
+    background: url(../img/icon_addlink.gif) 0 .2em no-repeat;
 }
 
 .changelink {
     padding-left: 12px;
-    background: url(../img/admin/icon_changelink.gif) 0 .2em no-repeat;
+    background: url(../img/icon_changelink.gif) 0 .2em no-repeat;
 }
 
 .deletelink {
     padding-left: 12px;
-    background: url(../img/admin/icon_deletelink.gif) 0 .25em no-repeat;
+    background: url(../img/icon_deletelink.gif) 0 .25em no-repeat;
 }
 
 a.deletelink:link, a.deletelink:visited {
@@ -593,14 +656,14 @@ a.deletelink:hover {
 .object-tools li {
     display: block;
     float: left;
-    background: url(../img/admin/tool-left.gif) 0 0 no-repeat;
+    background: url(../img/tool-left.gif) 0 0 no-repeat;
     padding: 0 0 0 8px;
     margin-left: 2px;
     height: 16px;
 }
 
 .object-tools li:hover {
-    background: url(../img/admin/tool-left_over.gif) 0 0 no-repeat;
+    background: url(../img/tool-left_over.gif) 0 0 no-repeat;
 }
 
 .object-tools a:link, .object-tools a:visited {
@@ -609,29 +672,29 @@ a.deletelink:hover {
     color: white;
     padding: .1em 14px .1em 8px;
     height: 14px;
-    background: #999 url(../img/admin/tool-right.gif) 100% 0 no-repeat;
+    background: #999 url(../img/tool-right.gif) 100% 0 no-repeat;
 }
 
 .object-tools a:hover, .object-tools li:hover a {
-    background: #5b80b2 url(../img/admin/tool-right_over.gif) 100% 0 no-repeat;
+    background: #5b80b2 url(../img/tool-right_over.gif) 100% 0 no-repeat;
 }
 
 .object-tools a.viewsitelink, .object-tools a.golink {
-    background: #999 url(../img/admin/tooltag-arrowright.gif) top right no-repeat;
+    background: #999 url(../img/tooltag-arrowright.gif) top right no-repeat;
     padding-right: 28px;
 }
 
 .object-tools a.viewsitelink:hover, .object-tools a.golink:hover {
-    background: #5b80b2 url(../img/admin/tooltag-arrowright_over.gif) top right no-repeat;
+    background: #5b80b2 url(../img/tooltag-arrowright_over.gif) top right no-repeat;
 }
 
 .object-tools a.addlink {
-    background: #999 url(../img/admin/tooltag-add.gif) top right no-repeat;
+    background: #999 url(../img/tooltag-add.gif) top right no-repeat;
     padding-right: 28px;
 }
 
 .object-tools a.addlink:hover {
-    background: #5b80b2 url(../img/admin/tooltag-add_over.gif) top right no-repeat;
+    background: #5b80b2 url(../img/tooltag-add_over.gif) top right no-repeat;
 }
 
 /* OBJECT HISTORY */
@@ -766,7 +829,7 @@ table#change-history tbody th {
 }
 
 #content-related .module h2 {
-    background: #eee url(../img/admin/nav-bg.gif) bottom left repeat-x;
+    background: #eee url(../img/nav-bg.gif) bottom left repeat-x;
     color: #666;
 }
 

lms/static/admin/css/changelists.css

@@ -20,7 +20,7 @@
 }
 
 .change-list .filtered {
-    background: white url(../img/admin/changelist-bg.gif) top right repeat-y !important;
+    background: white url(../img/changelist-bg.gif) top right repeat-y !important;
 }
 
 .change-list .filtered .results, .change-list .filtered .paginator, .filtered #toolbar, .filtered div.xfull {
@@ -40,7 +40,7 @@
     color: #666;
     border-top: 1px solid #eee;
     border-bottom: 1px solid #eee;
-    background: white url(../img/admin/nav-bg.gif) 0 180% repeat-x;
+    background: white url(../img/nav-bg.gif) 0 180% repeat-x;
     overflow: hidden;
 }
 
@@ -51,6 +51,7 @@
 /* CHANGELIST TABLES */
 
 #changelist table thead th {
+    padding: 0;
     white-space: nowrap;
     vertical-align: middle;
 }
@@ -82,7 +83,7 @@
 #changelist #toolbar {
     padding: 3px;
     border-bottom: 1px solid #ddd;
-    background: #e1e1e1 url(../img/admin/nav-bg.gif) top left repeat-x;
+    background: #e1e1e1 url(../img/nav-bg.gif) top left repeat-x;
     color: #666;
 }
 
@@ -156,7 +157,7 @@
 
 .change-list ul.toplinks {
     display: block;
-    background: white url(../img/admin/nav-bg-reverse.gif) 0 -10px repeat-x;
+    background: white url(../img/nav-bg-reverse.gif) 0 -10px repeat-x;
     border-top: 1px solid white;
     float: left;
     padding: 0 !important;
@@ -165,11 +166,10 @@
 }
 
 .change-list ul.toplinks li {
-    float: left;
-    width: 9em;
     padding: 3px 6px;
     font-weight: bold;
     list-style-type: none;
+    display: inline-block;
 }
 
 .change-list ul.toplinks .date-back a {
@@ -246,7 +246,7 @@
     padding: 3px;
     border-top: 1px solid #fff;
     border-bottom: 1px solid #ddd;
-    background: white url(../img/admin/nav-bg-reverse.gif) 0 -10px repeat-x;
+    background: white url(../img/nav-bg-reverse.gif) 0 -10px repeat-x;
 }
 
 #changelist .actions.selected {