Merge branch 'master' into feature/halogenandtoast/calculator

cms/djangoapps/contentstore/views.py

@@ -215,6 +215,7 @@ def preview_module_system(request, preview_id, descriptor):
         render_template=render_from_lms,
         debug=True,
         replace_urls=replace_urls,
+        user=request.user,
     )
 
 

cms/djangoapps/github_sync/tests/__init__.py

@@ -61,7 +61,7 @@ class GithubSyncTestCase(TestCase):
         self.assertIn(
             Location('i4x://edX/toy/chapter/Overview'),
             [child.location for child in self.import_course.get_children()])
-        self.assertEquals(1, len(self.import_course.get_children()))
+        self.assertEquals(2, len(self.import_course.get_children()))
 
     @patch('github_sync.sync_with_github')
     def test_sync_all_with_github(self, sync_with_github):

common/djangoapps/external_auth/views.py

@@ -4,19 +4,18 @@ import logging
 import random
 import re
 import string
+import fnmatch
 
 from external_auth.models import ExternalAuthMap
 
 from django.conf import settings
 from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login
-from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
+from student.models import UserProfile
 
-from django.core.urlresolvers import reverse
 from django.http import HttpResponse, HttpResponseRedirect
-from django.shortcuts import render_to_response
+from django.utils.http import urlquote
 from django.shortcuts import redirect
-from django.template import RequestContext
 from mitxmako.shortcuts import render_to_response, render_to_string
 try:
     from django.views.decorators.csrf import csrf_exempt
@@ -25,41 +24,64 @@ except ImportError:
 from django_future.csrf import ensure_csrf_cookie
 from util.cache import cache_if_anonymous
 
-from django_openid_auth import auth as openid_auth
-from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE)
 import django_openid_auth.views as openid_views
+from django_openid_auth import auth as openid_auth
+from openid.consumer.consumer import SUCCESS
+
+from openid.server.server import Server
+from openid.server.trustroot import TrustRoot
+from openid.store.filestore import FileOpenIDStore
+from openid.extensions import ax, sreg
 
 import student.views as student_views
 
 log = logging.getLogger("mitx.external_auth")
 
+
+# -----------------------------------------------------------------------------
+# OpenID Common
+# -----------------------------------------------------------------------------
+
+
 @csrf_exempt
-def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None):
-    """Render an Openid error page to the user."""
-    message = "In openid_failure " + message
-    log.debug(message)
-    data = render_to_string( template_name, dict(message=message, exception=exception))
+def default_render_failure(request,
+                           message,
+                           status=403,
+                           template_name='extauth_failure.html',
+                           exception=None):
+    """Render an Openid error page to the user"""
+
+    log.debug("In openid_failure " + message)
+
+    data = render_to_string(template_name,
+                            dict(message=message, exception=exception))
+
     return HttpResponse(data, status=status)
 
-#-----------------------------------------------------------------------------
-# Openid
 
-def edXauth_generate_password(length=12, chars=string.letters + string.digits):
+# -----------------------------------------------------------------------------
+# OpenID Authentication
+# -----------------------------------------------------------------------------
+
+
+def generate_password(length=12, chars=string.letters + string.digits):
     """Generate internal password for externally authenticated user"""
-    return ''.join([random.choice(chars) for i in range(length)])
+    choice = random.SystemRandom().choice
+    return ''.join([choice(chars) for i in range(length)])
+
 
 @csrf_exempt
-def edXauth_openid_login_complete(request,  redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None):
+def openid_login_complete(request,
+                          redirect_field_name=REDIRECT_FIELD_NAME,
+                          render_failure=None):
     """Complete the openid login process"""
 
-    redirect_to = request.REQUEST.get(redirect_field_name, '')
-    render_failure = render_failure or \
-                     getattr(settings, 'OPENID_RENDER_FAILURE', None) or \
-                     default_render_failure
+    render_failure = (render_failure or default_render_failure)
 
     openid_response = openid_views.parse_openid_response(request)
     if not openid_response:
-        return render_failure(request, 'This is an OpenID relying party endpoint.')
+        return render_failure(request,
+                              'This is an OpenID relying party endpoint.')
 
     if openid_response.status == SUCCESS:
         external_id = openid_response.identity_url
@@ -68,55 +90,64 @@ def edXauth_openid_login_complete(request,  redirect_field_name=REDIRECT_FIELD_N
 
         log.debug('openid success, details=%s' % details)
 
-        return edXauth_external_login_or_signup(request,
+        url = getattr(settings, 'OPENID_SSO_SERVER_URL', None)
+        external_domain = "openid:%s" % url
+        fullname = '%s %s' % (details.get('first_name', ''),
+                              details.get('last_name', ''))
+
+        return external_login_or_signup(request,
                                         external_id,
-                                                "openid:%s" % settings.OPENID_SSO_SERVER_URL,
+                                        external_domain,
                                         details,
-                                                details.get('email',''),
-                                                '%s %s' % (details.get('first_name',''),details.get('last_name',''))
-                                                )
+                                        details.get('email', ''),
+                                        fullname)
 
     return render_failure(request, 'Openid failure')
 
-#-----------------------------------------------------------------------------
-# generic external auth login or signup
 
-def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname,
+def external_login_or_signup(request,
+                             external_id,
+                             external_domain,
+                             credentials,
+                             email,
+                             fullname,
                              retfun=None):
+    """Generic external auth login or signup"""
+
     # see if we have a map from this external_id to an edX username
     try:
-        eamap = ExternalAuthMap.objects.get(external_id = external_id,
-                                            external_domain = external_domain,
-                                            )
+        eamap = ExternalAuthMap.objects.get(external_id=external_id,
+                                            external_domain=external_domain)
         log.debug('Found eamap=%s' % eamap)
     except ExternalAuthMap.DoesNotExist:
         # go render form for creating edX user
-        eamap = ExternalAuthMap(external_id = external_id,
-                                external_domain = external_domain,
-                                external_credentials = json.dumps(credentials),
-                                )
+        eamap = ExternalAuthMap(external_id=external_id,
+                                external_domain=external_domain,
+                                external_credentials=json.dumps(credentials))
         eamap.external_email = email
         eamap.external_name = fullname
-        eamap.internal_password = edXauth_generate_password()
-        log.debug('created eamap=%s' % eamap)
+        eamap.internal_password = generate_password()
+        log.debug('Created eamap=%s' % eamap)
 
         eamap.save()
 
     internal_user = eamap.user
     if internal_user is None:
-        log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email)
-        return edXauth_signup(request, eamap)
+        log.debug('No user for %s yet, doing signup' % eamap.external_email)
+        return signup(request, eamap)
 
     uname = internal_user.username
     user = authenticate(username=uname, password=eamap.internal_password)
     if user is None:
-        log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password))
-        return edXauth_signup(request, eamap)
+        log.warning("External Auth Login failed for %s / %s" %
+                    (uname, eamap.internal_password))
+        return signup(request, eamap)
 
     if not user.is_active:
-        log.warning("External Auth: user %s is not active" % (uname))
+        log.warning("User %s is not active" % (uname))
         # TODO: improve error page
-        return render_failure(request, 'Account not yet activated: please look for link in your email')
+        msg = 'Account not yet activated: please look for link in your email'
+        return default_render_failure(request, msg)
 
     login(request, user)
     request.session.set_expiry(0)
@@ -127,12 +158,9 @@ def edXauth_external_login_or_signup(request, external_id, external_domain, cred
     return retfun()
 
 
-#-----------------------------------------------------------------------------
-# generic external auth signup
-
 @ensure_csrf_cookie
 @cache_if_anonymous
-def edXauth_signup(request, eamap=None):
+def signup(request, eamap=None):
     """
     Present form to complete for signup via external authentication.
     Even though the user has external credentials, he/she still needs
@@ -146,28 +174,35 @@ def edXauth_signup(request, eamap=None):
     if eamap is None:
         pass
 
-    request.session['ExternalAuthMap'] = eamap	# save this for use by student.views.create_account
+    # save this for use by student.views.create_account
+    request.session['ExternalAuthMap'] = eamap
+
+    # default conjoin name, no spaces
+    username = eamap.external_name.replace(' ', '')
 
     context = {'has_extauth_info': True,
-               'show_signup_immediately' : True,
+               'show_signup_immediately': True,
                'extauth_email': eamap.external_email,
-               'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces
+               'extauth_username': username,
                'extauth_name': eamap.external_name,
                }
 
-    log.debug('ExtAuth: doing signup for %s' % eamap.external_email)
+    log.debug('Doing signup for %s' % eamap.external_email)
 
     return student_views.index(request, extra_context=context)
 
-#-----------------------------------------------------------------------------
+
+# -----------------------------------------------------------------------------
 # MIT SSL
+# -----------------------------------------------------------------------------
+
 
 def ssl_dn_extract_info(dn):
-    '''
-    Extract username, email address (may be anyuser@anydomain.com) and full name
-    from the SSL DN string.  Return (user,email,fullname) if successful, and None
-    otherwise.
-    '''
+    """
+    Extract username, email address (may be anyuser@anydomain.com) and
+    full name from the SSL DN string.  Return (user,email,fullname) if
+    successful, and None otherwise.
+    """
     ss = re.search('/emailAddress=(.*)@([^/]+)', dn)
     if ss:
         user = ss.group(1)
@@ -181,40 +216,333 @@ def ssl_dn_extract_info(dn):
         return None
     return (user, email, fullname)
 
+
 @csrf_exempt
-def edXauth_ssl_login(request):
+def ssl_login(request):
     """
-    This is called by student.views.index when MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
+    This is called by student.views.index when
+    MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
 
-    Used for MIT user authentication.  This presumes the web server (nginx) has been configured 
-    to require specific client certificates.
+    Used for MIT user authentication.  This presumes the web server
+    (nginx) has been configured to require specific client
+    certificates.
 
-    If the incoming protocol is HTTPS (SSL) then authenticate via client certificate.  
-    The certificate provides user email and fullname; this populates the ExternalAuthMap.
-    The user is nevertheless still asked to complete the edX signup.  
+    If the incoming protocol is HTTPS (SSL) then authenticate via
+    client certificate.  The certificate provides user email and
+    fullname; this populates the ExternalAuthMap.  The user is
+    nevertheless still asked to complete the edX signup.
 
     Else continues on with student.views.index, and no authentication.
     """
     certkey = "SSL_CLIENT_S_DN"  # specify the request.META field to use
 
-    cert = request.META.get(certkey,'')
+    cert = request.META.get(certkey, '')
     if not cert:
-        cert = request.META.get('HTTP_'+certkey,'')
+        cert = request.META.get('HTTP_' + certkey, '')
     if not cert:
         try:
-            cert = request._req.subprocess_env.get(certkey,'')	 # try the direct apache2 SSL key
-        except Exception as err:
-            pass
+            # try the direct apache2 SSL key
+            cert = request._req.subprocess_env.get(certkey, '')
+        except Exception:
+            cert = None
+
     if not cert:
         # no certificate information - go onward to main index
         return student_views.index(request)
 
     (user, email, fullname) = ssl_dn_extract_info(cert)
 
-    return edXauth_external_login_or_signup(request, 
+    retfun = functools.partial(student_views.index, request)
+    return external_login_or_signup(request,
                                     external_id=email,
                                     external_domain="ssl:MIT",
                                     credentials=cert,
                                     email=email,
                                     fullname=fullname,
-                                            retfun = functools.partial(student_views.index, request))
+                                    retfun=retfun)
+
+
+# -----------------------------------------------------------------------------
+# OpenID Provider
+# -----------------------------------------------------------------------------
+
+
+def get_xrds_url(resource, request):
+    """
+    Return the XRDS url for a resource
+    """
+    host = request.META['HTTP_HOST']
+
+    if not host.endswith('edx.org'):
+        return None
+
+    location = host + '/openid/provider/' + resource + '/'
+
+    if request.is_secure():
+        return 'https://' + location
+    else:
+        return 'http://' + location
+
+
+def add_openid_simple_registration(request, response, data):
+    sreg_data = {}
+    sreg_request = sreg.SRegRequest.fromOpenIDRequest(request)
+    sreg_fields = sreg_request.allRequestedFields()
+
+    # if consumer requested simple registration fields, add them
+    if sreg_fields:
+        for field in sreg_fields:
+            if field == 'email' and 'email' in data:
+                sreg_data['email'] = data['email']
+            elif field == 'fullname' and 'fullname' in data:
+                sreg_data['fullname'] = data['fullname']
+
+        # construct sreg response
+        sreg_response = sreg.SRegResponse.extractResponse(sreg_request,
+                                                          sreg_data)
+        sreg_response.toMessage(response.fields)
+
+
+def add_openid_attribute_exchange(request, response, data):
+    try:
+        ax_request = ax.FetchRequest.fromOpenIDRequest(request)
+    except ax.AXError:
+        #  not using OpenID attribute exchange extension
+        pass
+    else:
+        ax_response = ax.FetchResponse()
+
+        # if consumer requested attribute exchange fields, add them
+        if ax_request and ax_request.requested_attributes:
+            for type_uri in ax_request.requested_attributes.iterkeys():
+                email_schema = 'http://axschema.org/contact/email'
+                name_schema = 'http://axschema.org/namePerson'
+                if type_uri == email_schema and 'email' in data:
+                    ax_response.addValue(email_schema, data['email'])
+                elif type_uri == name_schema and 'fullname' in data:
+                    ax_response.addValue(name_schema, data['fullname'])
+
+            # construct ax response
+            ax_response.toMessage(response.fields)
+
+
+def provider_respond(server, request, response, data):
+    """
+    Respond to an OpenID request
+    """
+    # get and add extensions
+    add_openid_simple_registration(request, response, data)
+    add_openid_attribute_exchange(request, response, data)
+
+    # create http response from OpenID response
+    webresponse = server.encodeResponse(response)
+    http_response = HttpResponse(webresponse.body)
+    http_response.status_code = webresponse.code
+
+    # add OpenID headers to response
+    for k, v in webresponse.headers.iteritems():
+        http_response[k] = v
+
+    return http_response
+
+
+def validate_trust_root(openid_request):
+    """
+    Only allow OpenID requests from valid trust roots
+    """
+
+    trusted_roots = getattr(settings, 'OPENID_PROVIDER_TRUSTED_ROOT', None)
+
+    if not trusted_roots:
+        # not using trusted roots
+        return True
+
+    # don't allow empty trust roots
+    if (not hasattr(openid_request, 'trust_root') or
+            not openid_request.trust_root):
+        log.error('no trust_root')
+        return False
+
+    # ensure trust root parses cleanly (one wildcard, of form *.foo.com, etc.)
+    trust_root = TrustRoot.parse(openid_request.trust_root)
+    if not trust_root:
+        log.error('invalid trust_root')
+        return False
+
+    # don't allow empty return tos
+    if (not hasattr(openid_request, 'return_to') or
+            not openid_request.return_to):
+        log.error('empty return_to')
+        return False
+
+    # ensure return to is within trust root
+    if not trust_root.validateURL(openid_request.return_to):
+        log.error('invalid return_to')
+        return False
+
+    # check that the root matches the ones we trust
+    if not any(r for r in trusted_roots if fnmatch.fnmatch(trust_root, r)):
+        log.error('non-trusted root')
+        return False
+
+    return True
+
+
+@csrf_exempt
+def provider_login(request):
+    """
+    OpenID login endpoint
+    """
+
+    # make and validate endpoint
+    endpoint = get_xrds_url('login', request)
+    if not endpoint:
+        return default_render_failure(request, "Invalid OpenID request")
+
+    # initialize store and server
+    store = FileOpenIDStore('/tmp/openid_provider')
+    server = Server(store, endpoint)
+
+    # handle OpenID request
+    querydict = dict(request.REQUEST.items())
+    error = False
+    if 'openid.mode' in request.GET or 'openid.mode' in request.POST:
+        # decode request
+        openid_request = server.decodeRequest(querydict)
+
+        if not openid_request:
+            return default_render_failure(request, "Invalid OpenID request")
+
+        # don't allow invalid and non-trusted trust roots
+        if not validate_trust_root(openid_request):
+            return default_render_failure(request, "Invalid OpenID trust root")
+
+        # checkid_immediate not supported, require user interaction
+        if openid_request.mode == 'checkid_immediate':
+            return provider_respond(server, openid_request,
+                                    openid_request.answer(False), {})
+
+        # checkid_setup, so display login page
+        elif openid_request.mode == 'checkid_setup':
+            if openid_request.idSelect():
+                # remember request and original path
+                request.session['openid_setup'] = {
+                    'request': openid_request,
+                    'url': request.get_full_path()
+                }
+
+                # user failed login on previous attempt
+                if 'openid_error' in request.session:
+                    error = True
+                    del request.session['openid_error']
+
+        # OpenID response
+        else:
+            return provider_respond(server, openid_request,
+                                    server.handleRequest(openid_request), {})
+
+    # handle login
+    if request.method == 'POST' and 'openid_setup' in request.session:
+        # get OpenID request from session
+        openid_setup = request.session['openid_setup']
+        openid_request = openid_setup['request']
+        openid_request_url = openid_setup['url']
+        del request.session['openid_setup']
+
+        # don't allow invalid trust roots
+        if not validate_trust_root(openid_request):
+            return default_render_failure(request, "Invalid OpenID trust root")
+
+        # check if user with given email exists
+        email = request.POST.get('email', None)
+        try:
+            user = User.objects.get(email=email)
+        except User.DoesNotExist:
+            request.session['openid_error'] = True
+            msg = "OpenID login failed - Unknown user email: {0}".format(email)
+            log.warning(msg)
+            return HttpResponseRedirect(openid_request_url)
+
+        # attempt to authenticate user
+        username = user.username
+        password = request.POST.get('password', None)
+        user = authenticate(username=username, password=password)
+        if user is None:
+            request.session['openid_error'] = True
+            msg = "OpenID login failed - password for {0} is invalid"
+            msg = msg.format(email)
+            log.warning(msg)
+            return HttpResponseRedirect(openid_request_url)
+
+        # authentication succeeded, so log user in
+        if user is not None and user.is_active:
+            # remove error from session since login succeeded
+            if 'openid_error' in request.session:
+                del request.session['openid_error']
+
+            # fullname field comes from user profile
+            profile = UserProfile.objects.get(user=user)
+            log.info("OpenID login success - {0} ({1})".format(user.username,
+                                                               user.email))
+
+            # redirect user to return_to location
+            url = endpoint + urlquote(user.username)
+            response = openid_request.answer(True, None, url)
+
+            return provider_respond(server,
+                                    openid_request,
+                                    response,
+                                    {
+                                        'fullname': profile.name,
+                                        'email': user.email
+                                    })
+
+        request.session['openid_error'] = True
+        msg = "Login failed - Account not active for user {0}".format(username)
+        log.warning(msg)
+        return HttpResponseRedirect(openid_request_url)
+
+    # determine consumer domain if applicable
+    return_to = ''
+    if 'openid.return_to' in request.REQUEST:
+        return_to = request.REQUEST['openid.return_to']
+        matches = re.match(r'\w+:\/\/([\w\.-]+)', return_to)
+        return_to = matches.group(1)
+
+    # display login page
+    response = render_to_response('provider_login.html', {
+        'error': error,
+        'return_to': return_to
+    })
+
+    # custom XRDS header necessary for discovery process
+    response['X-XRDS-Location'] = get_xrds_url('xrds', request)
+    return response
+
+
+def provider_identity(request):
+    """
+    XRDS for identity discovery
+    """
+
+    response = render_to_response('identity.xml',
+                                  {'url': get_xrds_url('login', request)},
+                                  mimetype='text/xml')
+
+    # custom XRDS header necessary for discovery process
+    response['X-XRDS-Location'] = get_xrds_url('identity', request)
+    return response
+
+
+def provider_xrds(request):
+    """
+    XRDS for endpoint discovery
+    """
+
+    response = render_to_response('xrds.xml',
+                                  {'url': get_xrds_url('login', request)},
+                                  mimetype='text/xml')
+
+    # custom XRDS header necessary for discovery process
+    response['X-XRDS-Location'] = get_xrds_url('xrds', request)
+    return response

common/djangoapps/student/models.py

@@ -61,6 +61,7 @@ from xmodule.modulestore.django import modulestore
 
 log = logging.getLogger(__name__)
 
+
 class UserProfile(models.Model):
     """This is where we store all the user demographic fields. We have a 
     separate table for this rather than extending the built-in Django auth_user.
@@ -175,6 +176,7 @@ class PendingEmailChange(models.Model):
     new_email = models.CharField(blank=True, max_length=255, db_index=True)
     activation_key = models.CharField(('activation key'), max_length=32, unique=True, db_index=True)
 
+
 class CourseEnrollment(models.Model):
     user = models.ForeignKey(User)
     course_id = models.CharField(max_length=255, db_index=True)
@@ -184,6 +186,10 @@ class CourseEnrollment(models.Model):
     class Meta:
         unique_together = (('user', 'course_id'), )
 
+    def __unicode__(self):
+        return "[CourseEnrollment] %s: %s (%s)" % (self.user, self.course_id, self.created)
+
+
 @receiver(post_save, sender=CourseEnrollment)
 def assign_default_role(sender, instance, **kwargs):
     if instance.user.is_staff:
@@ -273,6 +279,7 @@ def add_user_to_default_group(user, group):
     utg.users.add(User.objects.get(username=user))
     utg.save()
 
+
 @receiver(post_save, sender=User)
 def update_user_information(sender, instance, created, **kwargs):
     try:
@@ -283,6 +290,7 @@ def update_user_information(sender, instance, created, **kwargs):
         log.error(unicode(e))
         log.error("update user info to discussion failed for user with id: " + str(instance.id))
 
+
 ########################## REPLICATION SIGNALS #################################
 # @receiver(post_save, sender=User)
 def replicate_user_save(sender, **kwargs):
@@ -292,6 +300,7 @@ def replicate_user_save(sender, **kwargs):
     for course_db_name in db_names_to_replicate_to(user_obj.id):
         replicate_user(user_obj, course_db_name)
 
+
 # @receiver(post_save, sender=CourseEnrollment)
 def replicate_enrollment_save(sender, **kwargs):
     """This is called when a Student enrolls in a course. It has to do the 
@@ -318,11 +327,13 @@ def replicate_enrollment_save(sender, **kwargs):
     user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
     replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
 
+
 # @receiver(post_delete, sender=CourseEnrollment)
 def replicate_enrollment_delete(sender, **kwargs):
     enrollment_obj = kwargs['instance']
     return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
 
+
 # @receiver(post_save, sender=UserProfile)
 def replicate_userprofile_save(sender, **kwargs):
     """We just updated the UserProfile (say an update to the name), so push that
@@ -336,6 +347,7 @@ USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
                        "password", "is_staff", "is_active", "is_superuser",
                        "last_login", "date_joined"]
 
+
 def replicate_user(portal_user, course_db_name):
     """Replicate a User to the correct Course DB. This is more complicated than
     it should be because Askbot extends the auth_user table and adds its own 
@@ -359,6 +371,7 @@ def replicate_user(portal_user, course_db_name):
     course_user.save(using=course_db_name)
     unmark(course_user)
 
+
 def replicate_model(model_method, instance, user_id):
     """
     model_method is the model action that we want replicated. For instance,
@@ -376,8 +389,10 @@ def replicate_model(model_method, instance, user_id):
         model_method(instance, using=db_name)
     unmark(instance)
 
+
 ######### Replication Helpers #########
 
+
 def is_valid_course_id(course_id):
     """Right now, the only database that's not a course database is 'default'.
     I had nicer checking in here originally -- it would scan the courses that
@@ -387,23 +402,27 @@ def is_valid_course_id(course_id):
     """
     return course_id != 'default'
 
+
 def is_portal():
     """Are we in the portal pool? Only Portal servers are allowed to replicate
     their changes. For now, only Portal servers see multiple DBs, so we use
     that to decide."""
     return len(settings.DATABASES) > 1
 
+
 def db_names_to_replicate_to(user_id):
     """Return a list of DB names that this user_id is enrolled in."""
     return [c.course_id
             for c in CourseEnrollment.objects.filter(user_id=user_id)
             if is_valid_course_id(c.course_id)]
 
+
 def marked_handled(instance):
     """Have we marked this instance as being handled to avoid infinite loops
     caused by saving models in post_save hooks for the same models?"""
     return hasattr(instance, '_do_not_copy_to_course_db') and instance._do_not_copy_to_course_db
 
+
 def mark_handled(instance):
     """You have to mark your instance with this function or else we'll go into
     an infinite loop since we're putting listeners on Model saves/deletes and
@@ -415,11 +434,13 @@ def mark_handled(instance):
     """
     instance._do_not_copy_to_course_db = True
 
+
 def unmark(instance):
     """If we don't unmark a model after we do replication, then consecutive
     save() calls won't be properly replicated."""
     instance._do_not_copy_to_course_db = False
 
+
 def should_replicate(instance):
     """Should this instance be replicated? We need to be a Portal server and
     the instance has to not have been marked_handled."""

common/djangoapps/student/views.py

@@ -75,8 +75,11 @@ def index(request, extra_context={}, user=None):
         entry.summary = soup.getText()
 
     # The course selection work is done in courseware.courses.
+    domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')	# normally False
+    if domain==False:				# do explicit check, because domain=None is valid
+        domain = request.META.get('HTTP_HOST')
     universities = get_courses_by_university(None,
-                                             domain=request.META.get('HTTP_HOST'))
+                                             domain=domain)
     context = {'universities': universities, 'entries': entries}
     context.update(extra_context)
     return render_to_response('index.html', context)
@@ -131,10 +134,14 @@ def dashboard(request):
         staff_access = True
         errored_courses = modulestore().get_errored_courses()
 
+    show_courseware_links_for = frozenset(course.id for course in courses
+                                          if has_access(request.user, course, 'load'))
+
     context = {'courses': courses,
                'message': message,
                'staff_access': staff_access,
-               'errored_courses': errored_courses,}
+               'errored_courses': errored_courses,
+               'show_courseware_links_for' : show_courseware_links_for}
 
     return render_to_response('dashboard.html', context)
 

common/djangoapps/xmodule_modifiers.py

 import re
 import json
 import logging
+import time
 
 from django.conf import settings
 from functools import wraps
@@ -75,7 +76,7 @@ def grade_histogram(module_id):
 
     grades = list(cursor.fetchall())
     grades.sort(key=lambda x: x[0])          # Add ORDER BY to sql query?
-    if len(grades) == 1 and grades[0][0] is None:
+    if len(grades) >= 1 and grades[0][0] is None:
         return []
     return grades
 
@@ -117,6 +118,14 @@ def add_histogram(get_html, module, user):
             data_dir = ""
         source_file = module.metadata.get('source_file','')	# source used to generate the problem XML, eg latex or word
 
+        # useful to indicate to staff if problem has been released or not
+        # TODO (ichuang): use _has_access_descriptor.can_load in lms.courseware.access, instead of now>mstart comparison here
+        now = time.gmtime()
+        is_released = "unknown"
+        mstart = getattr(module.descriptor,'start')
+        if mstart is not None:
+            is_released = "<font color='red'>Yes!</font>" if (now > mstart) else "<font color='green'>Not yet</font>"
+        
         staff_context = {'definition': module.definition.get('data'),
                          'metadata': json.dumps(module.metadata, indent=4),
                          'location': module.location,
@@ -130,7 +139,9 @@ def add_histogram(get_html, module, user):
                          'xqa_server' : settings.MITX_FEATURES.get('USE_XQA_SERVER','http://xqa:server@content-qa.mitx.mit.edu/xqa'),
                          'histogram': json.dumps(histogram),
                          'render_histogram': render_histogram,
-                         'module_content': get_html()}
+                         'module_content': get_html(),
+                         'is_released': is_released,
+                         }
         return render_to_string("staff_problem_info.html", staff_context)
 
     return _get_html

common/lib/capa/capa/capa_problem.py

@@ -14,6 +14,8 @@ This is used by capa_module.
 
 from __future__ import division
 
+from datetime import datetime
+import json
 import logging
 import math
 import numpy
@@ -32,6 +34,7 @@ from correctmap import CorrectMap
 import eia
 import inputtypes
 from util import contextualize_text, convert_files_to_filenames
+import xqueue_interface
 
 # to be replaced with auto-registering
 import responsetypes
@@ -202,11 +205,24 @@ class LoncapaProblem(object):
         '''
         Returns True if any part of the problem has been submitted to an external queue
         '''
-        queued = False
-        for answer_id in self.correct_map:
-            if self.correct_map.is_queued(answer_id):
-                queued = True
-        return queued
+        return any(self.correct_map.is_queued(answer_id) for answer_id in self.correct_map)
+
+
+    def get_recentmost_queuetime(self):
+        '''
+        Returns a DateTime object that represents the timestamp of the most recent queueing request, or None if not queued
+        '''
+        if not self.is_queued():
+            return None
+
+        # Get a list of timestamps of all queueing requests, then convert it to a DateTime object
+        queuetime_strs = [self.correct_map.get_queuetime_str(answer_id)
+                          for answer_id in self.correct_map 
+                          if self.correct_map.is_queued(answer_id)]
+        queuetimes = [datetime.strptime(qt_str, xqueue_interface.dateformat) for qt_str in queuetime_strs]
+
+        return max(queuetimes)
+
 
     def grade_answers(self, answers):
         '''

common/lib/capa/capa/correctmap.py

@@ -15,7 +15,8 @@ class CorrectMap(object):
     - msg         : string (may have HTML) giving extra message response (displayed below textline or textbox)
     - hint        : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg)
     - hintmode    : one of (None,'on_request','always') criteria for displaying hint
-    - queuekey    : a random integer for xqueue_callback verification
+    - queuestate  : Dict {key:'', time:''} where key is a secret string, and time is a string dump
+                    of a DateTime object in the format '%Y%m%d%H%M%S'. Is None when not queued
 
     Behaves as a dict.
     '''
@@ -31,14 +32,15 @@ class CorrectMap(object):
     def __iter__(self):
         return self.cmap.__iter__()
 
-    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuekey=None):
+    # See the documentation for 'set_dict' for the use of kwargs
+    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None, **kwargs):
         if answer_id is not None:
             self.cmap[answer_id] = {'correctness': correctness,
                                     'npoints': npoints,
                                     'msg': msg,
                                     'hint': hint,
                                     'hintmode': hintmode,
-                                    'queuekey': queuekey,
+                                    'queuestate': queuestate,
                                     }
 
     def __repr__(self):
@@ -52,25 +54,39 @@ class CorrectMap(object):
 
     def set_dict(self, correct_map):
         '''
-        set internal dict to provided correct_map dict
-        for graceful migration, if correct_map is a one-level dict, then convert it to the new
-        dict of dicts format.
+        Set internal dict of CorrectMap to provided correct_map dict
+
+        correct_map is saved by LMS as a plaintext JSON dump of the correctmap dict. This means that
+        when the definition of CorrectMap (e.g. its properties) are altered, existing correct_map dict
+        not coincide with the newest CorrectMap format as defined by self.set.
+
+        For graceful migration, feed the contents of each correct map to self.set, rather than
+        making a direct copy of the given correct_map dict. This way, the common keys between 
+        the incoming correct_map dict and the new CorrectMap instance will be written, while
+        mismatched keys will be gracefully ignored.
+
+        Special migration case:
+            If correct_map is a one-level dict, then convert it to the new dict of dicts format.
         '''
         if correct_map and not (type(correct_map[correct_map.keys()[0]]) == dict):
             self.__init__()	                                  # empty current dict
             for k in correct_map: self.set(k, correct_map[k]) # create new dict entries
         else:
-            self.cmap = correct_map
+            self.__init__()
+            for k in correct_map: self.set(k, **correct_map[k])
 
     def is_correct(self, answer_id):
         if answer_id in self.cmap: return self.cmap[answer_id]['correctness'] == 'correct'
         return None
 
     def is_queued(self, answer_id):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] is not None
+        return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
 
     def is_right_queuekey(self, answer_id, test_key):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] == test_key
+        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
+
+    def get_queuetime_str(self, answer_id):
+        return self.cmap[answer_id]['queuestate']['time']
 
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')

common/lib/capa/capa/inputtypes.py

@@ -351,7 +351,7 @@ def filesubmission(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     context = { 'id': eid, 'state': status, 'msg': msg, 'value': value,
             'queue_len': queue_len, 'allowed_files': allowed_files,
@@ -384,7 +384,7 @@ def textbox(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     # For CodeMirror
     mode = element.get('mode','python')

common/lib/capa/capa/responsetypes.py

@@ -8,6 +8,7 @@ Used by capa_problem.py
 '''
 
 # standard library imports
+import cgi
 import inspect
 import json
 import logging
@@ -26,6 +27,7 @@ import xml.sax.saxutils as saxutils
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
+from datetime import datetime
 from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
@@ -317,30 +319,37 @@ class JavascriptResponse(LoncapaResponse):
     
     def compile_display_javascript(self):
 
-        latestTimestamp = 0
-        basepath = self.system.filestore.root_path + '/js/'
-        for filename in (self.display_dependencies + [self.display]):
-            filepath = basepath + filename
-            timestamp = os.stat(filepath).st_mtime
-            if timestamp > latestTimestamp:
-                latestTimestamp = timestamp
-        
-        h = hashlib.md5()
-        h.update(self.answer_id + str(self.display_dependencies))
-        compiled_filename = 'compiled/' + h.hexdigest() + '.js'
-        compiled_filepath = basepath + compiled_filename
-
-        if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
-            outfile = open(compiled_filepath, 'w')
-            for filename in (self.display_dependencies + [self.display]):
-                filepath = basepath + filename
-                infile = open(filepath, 'r')
-                outfile.write(infile.read())
-                outfile.write(';\n')
-                infile.close()
-            outfile.close()
-
-        self.display_filename = compiled_filename
+        # TODO FIXME
+        # arjun: removing this behavior for now (and likely forever). Keeping
+        # until we decide on exactly how to solve this issue. For now, files are
+        # manually being compiled to DATA_DIR/js/compiled.
+
+        #latestTimestamp = 0
+        #basepath = self.system.filestore.root_path + '/js/'
+        #for filename in (self.display_dependencies + [self.display]):
+        #    filepath = basepath + filename
+        #    timestamp = os.stat(filepath).st_mtime
+        #    if timestamp > latestTimestamp:
+        #        latestTimestamp = timestamp
+        #
+        #h = hashlib.md5()
+        #h.update(self.answer_id + str(self.display_dependencies))
+        #compiled_filename = 'compiled/' + h.hexdigest() + '.js'
+        #compiled_filepath = basepath + compiled_filename
+
+        #if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
+        #    outfile = open(compiled_filepath, 'w')
+        #    for filename in (self.display_dependencies + [self.display]):
+        #        filepath = basepath + filename
+        #        infile = open(filepath, 'r')
+        #        outfile.write(infile.read())
+        #        outfile.write(';\n')
+        #        infile.close()
+        #    outfile.close()
+
+        # TODO this should also be fixed when the above is fixed.
+        filename = self.system.ajax_url.split('/')[-1] + '.js'
+        self.display_filename = 'compiled/' + filename
 
     def parse_xml(self):
         self.generator_xml = self.xml.xpath('//*[@id=$id]//generator',
@@ -385,18 +394,22 @@ class JavascriptResponse(LoncapaResponse):
         tmp_env["NODE_PATH"] = node_path
         return tmp_env
     
+    def call_node(self, args):
+
+        subprocess_args = ["node"]
+        subprocess_args.extend(args)
+
+        return subprocess.check_output(subprocess_args, env=self.get_node_env())
+
 
     def generate_problem_state(self):
 
         generator_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_generator.js'
-        output = subprocess.check_output(["node",
-                                          generator_file, 
+        output = self.call_node([generator_file,
                                  self.generator, 
                                  json.dumps(self.generator_dependencies),
                                  json.dumps(str(self.system.seed)), 
-                                          json.dumps(self.params)
-                                          ],
-                                          env=self.get_node_env()).strip()
+                                 json.dumps(self.params)]).strip()
 
         return json.loads(output)
 
@@ -407,7 +420,8 @@ class JavascriptResponse(LoncapaResponse):
         for param in self.xml.xpath('//*[@id=$id]//responseparam', 
                                         id=self.xml.get('id')):
 
-            params[param.get("name")] = json.loads(param.get("value"))
+            raw_param = param.get("value")
+            params[param.get("name")] = json.loads(contextualize_text(raw_param, self.context))
         
         return params
 
@@ -435,22 +449,23 @@ class JavascriptResponse(LoncapaResponse):
         (all_correct, evaluation, solution) = self.run_grader(json_submission)
         self.solution = solution
         correctness = 'correct' if all_correct else 'incorrect'
-        return CorrectMap(self.answer_id, correctness, msg=evaluation)
+        if all_correct:
+            points = self.get_max_score()
+        else:
+            points = 0
+        return CorrectMap(self.answer_id, correctness, npoints=points, msg=evaluation)
     
     def run_grader(self, submission):
         if submission is None or submission == '':
             submission = json.dumps(None)
 
         grader_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_grader.js'
-        outputs = subprocess.check_output(["node",
-                                           grader_file, 
+        outputs = self.call_node([grader_file, 
                                   self.grader, 
                                   json.dumps(self.grader_dependencies),
                                   submission, 
                                   json.dumps(self.problem_state), 
-                                           json.dumps(self.params)
-                                          ],
-                                          env=self.get_node_env()).split('\n')
+                                  json.dumps(self.params)]).split('\n')
 
         all_correct = json.loads(outputs[0].strip())
         evaluation  = outputs[1].strip()
@@ -711,7 +726,8 @@ class NumericalResponse(LoncapaResponse):
         # I think this is just pyparsing.ParseException, calc.UndefinedVariable:
         # But we'd need to confirm
         except:
-            raise StudentInputError('Invalid input -- please use a number only')
+            raise StudentInputError("Invalid input: could not interpret '%s' as a number" %\
+                                    cgi.escape(student_answer))
 
         if correct:
             return CorrectMap(self.answer_id, 'correct')
@@ -900,10 +916,12 @@ def sympy_check2():
             try:
                 exec self.code in self.context['global_context'], self.context
                 correct = self.context['correct']
+                messages = self.context['messages']
             except Exception as err:
                 print "oops in customresponse (code) error %s" % err
                 print "context = ", self.context
                 print traceback.format_exc()
+                raise StudentInputError("Error: Problem could not be evaluated with your input") # Notify student
         else:					# self.code is not a string; assume its a function
 
             # this is an interface to the Tutor2 check functions
@@ -953,7 +971,8 @@ def sympy_check2():
         # build map giving "correct"ness of the answer(s)
         correct_map = CorrectMap()
         for k in range(len(idset)):
-            correct_map.set(idset[k], correct[k], msg=messages[k])
+            correct_map.set(idset[k], correct[k], msg=messages[k],
+                            npoints=self.maxpoints[idset[k]])
         return correct_map
 
     def get_answers(self):
@@ -1005,7 +1024,7 @@ class CodeResponse(LoncapaResponse):
     '''
     Grade student code using an external queueing server, called 'xqueue'
 
-    Expects 'xqueue' dict in ModuleSystem with the following keys:
+    Expects 'xqueue' dict in ModuleSystem with the following keys that are needed by CodeResponse:
         system.xqueue = { 'interface': XqueueInterface object,
                           'callback_url': Per-StudentModule callback URL where results are posted (string),
                           'default_queuename': Default queuename to submit request (string)
@@ -1026,7 +1045,7 @@ class CodeResponse(LoncapaResponse):
         TODO: Determines whether in synchronous or asynchronous (queued) mode
         '''
         xml = self.xml
-        self.url = xml.get('url', None) # XML can override external resource (grader/queue) URL
+        self.url = xml.get('url', None) # TODO: XML can override external resource (grader/queue) URL
         self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
 
         # VS[compat]:
@@ -1121,22 +1140,34 @@ class CodeResponse(LoncapaResponse):
 
         # Prepare xqueue request
         #------------------------------------------------------------ 
+
         qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
+
+        anonymous_student_id = self.system.anonymous_student_id
 
         # Generate header
-        queuekey = xqueue_interface.make_hashkey(str(self.system.seed)+self.answer_id)
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
+                                                 anonymous_student_id +  
+                                                 self.answer_id)
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
                                                 lms_key=queuekey,
                                                 queue_name=self.queue_name)
         
         # Generate body
         if is_list_of_files(submission):
-            self.context.update({'submission': queuekey}) # For tracking. TODO: May want to record something else here
+            self.context.update({'submission': ''}) # TODO: Get S3 pointer from the Queue
         else:
             self.context.update({'submission': submission})
 
         contents = self.payload.copy() 
 
+        # Metadata related to the student submission revealed to the external grader
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                        'submission_time': qtime,
+                       }
+        contents.update({'student_info': json.dumps(student_info)})
+
         # Submit request. When successful, 'msg' is the prior length of the queue
         if is_list_of_files(submission):
             contents.update({'student_response': ''}) # TODO: Is there any information we want to send here?
@@ -1148,16 +1179,21 @@ class CodeResponse(LoncapaResponse):
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents))
 
+        # State associated with the queueing request
+        queuestate = {'key': queuekey,
+                      'time': qtime,
+                     }
+
         cmap = CorrectMap() 
         if error:
-            cmap.set(self.answer_id, queuekey=None,
+            cmap.set(self.answer_id, queuestate=None,
                      msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg)
         else:
             # Queueing mechanism flags:
-            #   1) Backend: Non-null CorrectMap['queuekey'] indicates that the problem has been queued
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that the problem has been queued
             #   2) Frontend: correctness='incomplete' eventually trickles down through inputtypes.textbox 
             #       and .filesubmission to inform the browser to poll the LMS
-            cmap.set(self.answer_id, queuekey=queuekey, correctness='incomplete', msg=msg)
+            cmap.set(self.answer_id, queuestate=queuestate, correctness='incomplete', msg=msg)
 
         return cmap
 
@@ -1165,7 +1201,7 @@ class CodeResponse(LoncapaResponse):
 
         (valid_score_msg, correct, points, msg) = self._parse_score_msg(score_msg) 
         if not valid_score_msg:
-            oldcmap.set(self.answer_id, msg='Error: Invalid grader reply.')
+            oldcmap.set(self.answer_id, msg='Invalid grader reply. Please contact the course staff.')
             return oldcmap
         
         correctness = 'correct' if correct else 'incorrect'
@@ -1180,7 +1216,7 @@ class CodeResponse(LoncapaResponse):
                 points = 0
             elif points > self.maxpoints[self.answer_id]:
                 points = self.maxpoints[self.answer_id]
-            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuekey=None)  # Queuekey is consumed
+            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)  # Queuestate is consumed
         else:
             log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id))
 
@@ -1197,26 +1233,41 @@ class CodeResponse(LoncapaResponse):
         '''
          Grader reply is a JSON-dump of the following dict
            { 'correct': True/False,
-             'score': # TODO -- Partial grading
+             'score': Numeric value (floating point is okay) to assign to answer
              'msg': grader_msg }
 
         Returns (valid_score_msg, correct, score, msg):
             valid_score_msg: Flag indicating valid score_msg format (Boolean)
             correct:         Correctness of submission (Boolean)
-            score:           # TODO: Implement partial grading
+            score:           Points to be assigned (numeric, can be float)
             msg:             Message from grader to display to student (string)
         '''
-        fail = (False, False, -1, '')
+        fail = (False, False, 0, '')
         try:
             score_result = json.loads(score_msg)
         except (TypeError, ValueError):
+            log.error("External grader message should be a JSON-serialized dict. Received score_msg = %s" % score_msg)
             return fail
         if not isinstance(score_result, dict):
+            log.error("External grader message should be a JSON-serialized dict. Received score_result = %s" % score_result)
             return fail
         for tag in ['correct', 'score', 'msg']:
-            if not score_result.has_key(tag):
+            if tag not in score_result:
+                log.error("External grader message is missing one or more required tags: 'correct', 'score', 'msg'")
                 return fail
-        return (True, score_result['correct'], score_result['score'], score_result['msg'])
+
+        # Next, we need to check that the contents of the external grader message 
+        #   is safe for the LMS.
+        # 1) Make sure that the message is valid XML (proper opening/closing tags)
+        # 2) TODO: Is the message actually HTML?
+        msg = score_result['msg']
+        try:
+            etree.fromstring(msg)
+        except etree.XMLSyntaxError as err:
+            log.error("Unable to parse external grader message as valid XML: score_msg['msg']=%s" % msg)
+            return fail
+            
+        return (True, score_result['correct'], score_result['score'], msg)
         
 
 #-----------------------------------------------------------------------------
@@ -1471,11 +1522,12 @@ class FormulaResponse(LoncapaResponse):
                                            cs=self.case_sensitive)
             except UndefinedVariable as uv:
                 log.debug('formularesponse: undefined variable in given=%s' % given)
-                raise StudentInputError(uv.message + " not permitted in answer")
+                raise StudentInputError("Invalid input: " + uv.message + " not permitted in answer")
             except Exception as err:
                 #traceback.print_exc()
                 log.debug('formularesponse: error %s in formula' % err)
-                raise StudentInputError("Error in formula")
+                raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %\
+                                        cgi.escape(given))
             if numpy.isnan(student_result) or numpy.isinf(student_result):
                 return "incorrect"
             if not compare_with_tolerance(student_result, instructor_result, self.tolerance):

common/lib/capa/capa/templates/choicegroup.html

-<form class="choicegroup">
+<form class="choicegroup capa_inputtype" id="inputtype_${id}">
   
   % for choice_id, choice_description in choices:
   <label for="input_${id}_${choice_id}">  <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}" value="${choice_id}"

common/lib/capa/capa/templates/filesubmission.html

 <section id="filesubmission_${id}" class="filesubmission">
-    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" data-required_files="${required_files}" data-allowed_files="${allowed_files}"/><br />
+  <div class="grader-status file">
     % if state == 'unsubmitted':
-    <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+    <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
     % elif state == 'correct':
-    <span class="correct" id="status_${id}"></span>
+    <span class="correct" id="status_${id}">Correct</span>
     % elif state == 'incorrect':
-    <span class="incorrect" id="status_${id}"></span>
+    <span class="incorrect" id="status_${id}">Incorrect</span>
     % elif state == 'queued':
-    <span class="processing" id="status_${id}"></span>
+    <span class="processing" id="status_${id}">Queued</span>
     <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
-    <span class="debug">(${state})</span>
-    <br/>
-    <span class="message">${msg|n}</span>
-    <br/>
+    <p class="debug">${state}</p>
+
+    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" data-required_files="${required_files}" data-allowed_files="${allowed_files}"/>
+  </div>
+    <div class="message">${msg|n}</div>
 </section>

common/lib/capa/capa/templates/javascriptinput.html

-<form class="javascriptinput capa_inputtype">
+<form class="javascriptinput capa_inputtype" id="inputtype_${id}">
   <input type="hidden" name="input_${id}" id="input_${id}" class="javascriptinput_input"/>
     <div class="javascriptinput_data" data-display_class="${display_class}" 
         data-problem_state="${problem_state}" data-params="${params}" 

common/lib/capa/capa/templates/textbox.html

@@ -7,26 +7,28 @@
 
   <span id="answer_${id}"></span>
 
+  <div class="grader-status">
     % if state == 'unsubmitted':
-  <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
     % elif state == 'correct':
-  <span class="correct" id="status_${id}"></span>
+      <span class="correct" id="status_${id}">Correct</span>
     % elif state == 'incorrect':
-  <span class="incorrect" id="status_${id}"></span>
+      <span class="incorrect" id="status_${id}">Incorrect</span>
     % elif state == 'queued':
-  <span class="processing" id="status_${id}"></span>
+      <span class="processing" id="status_${id}">Queued</span>
       <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
+
     % if hidden:
       <div style="display:none;" name="${hidden}" inputid="input_${id}" />
     % endif
-  <br/>
-  <span class="debug">(${state})</span>
-  <br/>
-  <span class="message">${msg|n}</span>
-  <br/>
 
-  <br/>
+    <p class="debug">${state}</p>
+  </div>
+
+  <div class="external-grader-message">
+    ${msg|n}
+  </div>
 
   <script>
     // Note: We need to make the area follow the CodeMirror for this to work.
@@ -45,12 +47,4 @@
       });
     });
   </script>
-  <style type="text/css">
-    .CodeMirror {
-        border: 1px solid black;
-        font-size: 14px;
-        line-height: 18px;
-        resize: both;
-    }
-  </style>
 </section>

common/lib/capa/capa/xqueue_interface.py

@@ -5,20 +5,17 @@ import hashlib
 import json
 import logging
 import requests
-import time
 
 
 log = logging.getLogger('mitx.' + __name__)
+dateformat = '%Y%m%d%H%M%S'
 
-
-def make_hashkey(seed=None):
+def make_hashkey(seed):
     '''
     Generate a string key by hashing 
     '''
     h = hashlib.md5()
-    if seed is not None:
     h.update(str(seed))
-    h.update(str(time.time()))
     return h.hexdigest()
 
 

common/lib/xmodule/xmodule/capa_module.py

@@ -347,6 +347,10 @@ class CapaModule(XModule):
         if self.show_answer == "never":
             return False
 
+        # Admins can see the answer, unless the problem explicitly prevents it
+        if self.system.user_is_staff:
+            return True
+
         if self.show_answer == 'attempted':
             return self.attempts > 0
 
@@ -462,6 +466,15 @@ class CapaModule(XModule):
             self.system.track_function('save_problem_check_fail', event_info)
             raise NotFoundError('Problem must be reset before it can be checked again')
 
+        # Problem queued. Students must wait a specified waittime before they are allowed to submit
+        if self.lcp.is_queued():
+            current_time = datetime.datetime.now()
+            prev_submit_time = self.lcp.get_recentmost_queuetime() 
+            waittime_between_requests = self.system.xqueue['waittime']
+            if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
+                msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
+                return {'success': msg, 'html': ''} # Prompts a modal dialog in ajax callback 
+
         try:
             old_state = self.lcp.get_state()
             lcp_id = self.lcp.problem_id

common/lib/xmodule/xmodule/course_module.py

@@ -237,6 +237,22 @@ class CourseDescriptor(SequenceDescriptor):
         return self.metadata.get('info_sidebar_name', 'Course Handouts')
 
     @property
+    def discussion_link(self):
+        """TODO: This is a quick kludge to allow CS50 (and other courses) to
+        specify their own discussion forums as external links by specifying a
+        "discussion_link" in their policy JSON file. This should later get
+        folded in with Syllabus, Course Info, and additional Custom tabs in a
+        more sensible framework later."""
+        return self.metadata.get('discussion_link', None)
+
+    @property
+    def hide_progress_tab(self):
+        """TODO: same as above, intended to let internal CS50 hide the progress tab
+        until we get grade integration set up."""
+        # Explicit comparison to True because we always want to return a bool.
+        return self.metadata.get('hide_progress_tab') == True
+
+    @property
     def title(self):
         return self.display_name
 

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -16,6 +16,7 @@ h2 {
   }
 }
 
+
 section.problem {
   @media print {
     display: block;
@@ -31,6 +32,13 @@ section.problem {
       display: inline;
   }
 
+  .choicegroup {
+    label.choicegroup_correct:after {
+      content: url('../images/correct-icon.png');
+    }
+  }
+
+
   div {
     p {
       &.answer {
@@ -171,8 +179,54 @@ section.problem {
         top: 6px;
       }
     }
+
+    .grader-status {
+      padding: 9px;
+      background: #F6F6F6;
+      border: 1px solid #ddd;
+      border-top: 0;
+      margin-bottom: 20px;
+      @include clearfix;
+
+      span {
+        text-indent: -9999px;
+        overflow: hidden;
+        display: block;
+        float: left;
+        margin: -7px 7px 0 0;
       }
 
+      p {
+        line-height: 20px;
+        text-transform: capitalize;
+        margin-bottom: 0;
+        float: left;
+      }
+
+      &.file {
+        background: #FFF;
+        margin-top: 20px;
+        padding: 20px 0 0 0;
+
+        border: {
+          top: 1px solid #eee;
+          right: 0;
+          bottom: 0;
+          left: 0;
+        }
+
+        p.debug {
+          display: none;
+        }
+
+        input {
+          float: left;
+        }
+      }
+    }
+  }
+
+
   ul {
     list-style: disc outside none;
     margin-bottom: lh();
@@ -246,6 +300,69 @@ section.problem {
 
   }
 
+  code {
+    margin: 0 2px;
+    padding: 0px 5px;
+    white-space: nowrap;
+    border: 1px solid #EAEAEA;
+    background-color: #F8F8F8;
+    @include border-radius(3px);
+    font-size: .9em;
+  }
+
+  pre {
+    background-color: #F8F8F8;
+    border: 1px solid #CCC;
+    font-size: .9em;
+    line-height: 1.4;
+    overflow: auto;
+    padding: 6px 10px;
+    @include border-radius(3px);
+
+    > code {
+      margin: 0;
+      padding: 0;
+      white-space: pre;
+      border: none;
+      background: transparent;
+    }
+  }
+
+  .CodeMirror {
+    border: 1px solid black;
+    font-size: 14px;
+    line-height: 18px;
+    resize: both;
+
+    pre {
+      @include border-radius(0);
+      border-radius: 0;
+      border-width: 0;
+      margin: 0;
+      padding: 0;
+      background: transparent;
+      font-family: inherit;
+      font-size: inherit;
+      white-space: pre;
+      word-wrap: normal;
+      overflow: hidden;
+      resize: none;
+
+      &.CodeMirror-cursor {
+        z-index: 10;
+        position: absolute;
+        visibility: hidden;
+        border-left: 1px solid black;
+        border-right: none;
+        width: 0;
+      }
+    }
+  }
+
+  .CodeMirror-focused pre.CodeMirror-cursor {
+    visibility: visible;
+  }
+
   hr {
     background: #ddd;
     border: none;
@@ -280,4 +397,118 @@ section.problem {
       @extend .blue-button;
     }
   }
+
+  .detailed-solution {
+    border: 1px solid #ddd;
+    padding: 9px 9px 20px;
+    margin-bottom: 10px;
+    background: #FFF;
+    position: relative;
+    @include box-shadow(inset 0 0 0 1px #eee);
+    @include border-radius(3px);
+
+    p:first-child {
+      font-size: 0.9em;
+      font-weight: bold;
+      font-style: normal;
+      text-transform: uppercase;
+      color: #AAA;
+    }
+
+    p:last-child {
+      margin-bottom: 0;
+    }
+  }
+
+  div.capa_alert {
+    padding: 8px 12px;
+    border: 1px solid #EBE8BF;
+    border-radius: 3px;
+    background: #FFFCDD;
+    font-size: 0.9em;
+    margin-top: 10px;
+  }
+
+  .hints {
+    border: 1px solid #ccc;
+
+    h3 {
+      border-bottom: 1px solid #e3e3e3;
+      text-shadow: 0 1px 0 #fff;
+      padding: 9px;
+      background: #eee;
+      font-weight: bold;
+      font-size: em(16);
+    }
+
+    div {
+      border-bottom: 1px solid #ddd;
+
+      &:last-child {
+        border-bottom: none;
+      }
+
+      p {
+        margin-bottom: 0;
+      }
+
+      header {
+        a {
+          display: block;
+          padding: 9px;
+          background: #F6F6F6;
+          @include box-shadow(inset 0 0 0 1px #fff);
+        }
+      }
+
+      section {
+        padding: 9px;
+      }
+    }
+  }
+
+  .test {
+    padding-top: 18px;
+
+    header {
+      margin-bottom: 12px;
+
+      h3 {
+        font-size: 0.9em;
+        font-weight: bold;
+        font-style: normal;
+        text-transform: uppercase;
+        color: #AAA;
+      }
+    }
+
+    > section {
+      border: 1px solid #ddd;
+      padding: 9px 9px 20px;
+      margin-bottom: 10px;
+      background: #FFF;
+      position: relative;
+      @include box-shadow(inset 0 0 0 1px #eee);
+      @include border-radius(3px);
+
+      p:last-of-type {
+        margin-bottom: 0;
+      }
+
+      .shortform {
+        margin-bottom: .6em;
+      }
+
+      a.full {
+        @include position(absolute,  0 0 1px 0px);
+        font-size: .8em;
+        padding: 4px;
+        text-align: right;
+        width: 100%;
+        display: block;
+        background: #F3F3F3;
+        @include box-sizing(border-box);
+      }
+    }
+  }
 }

common/lib/xmodule/xmodule/css/sequence/display.scss

@@ -37,7 +37,6 @@ nav.sequence-nav {
     height: 44px;
     margin: 0 30px;
     @include linear-gradient(top, #ddd, #eee);
-    overflow: hidden;
     @include box-shadow(0 1px 3px rgba(0, 0, 0, .1) inset);
   }
 

common/lib/xmodule/xmodule/discussion_module.py

 from lxml import etree
+from pkg_resources import resource_string, resource_listdir
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
@@ -6,6 +7,11 @@ from xmodule.raw_module import RawDescriptor
 import json
 
 class DiscussionModule(XModule):
+    js = {'coffee':
+            [resource_string(__name__, 'js/src/time.coffee'),
+            resource_string(__name__, 'js/src/discussion/display.coffee')]
+        }
+    js_module_name = "InlineDiscussion"
     def get_html(self):
         context = {
             'discussion_id': self.discussion_id,

common/lib/xmodule/xmodule/errortracker.py

@@ -35,6 +35,11 @@ def make_error_tracker():
         if in_exception_handler():
             exc_str = exc_info_to_str(sys.exc_info())
 
+            # don't display irrelevant gunicorn sync error
+            if (('python2.7/site-packages/gunicorn/workers/sync.py' in exc_str) and
+                ('[Errno 11] Resource temporarily unavailable' in exc_str)):
+                exc_str = ''
+
         errors.append((msg, exc_str))
 
     return ErrorLog(error_tracker, errors)

common/lib/xmodule/xmodule/html_module.py

@@ -4,9 +4,10 @@ import logging
 import os
 import sys
 from lxml import etree
+from path import path
 
 from .x_module import XModule
-from .xml_module import XmlDescriptor
+from .xml_module import XmlDescriptor, name_to_pathname
 from .editing_module import EditingDescriptor
 from .stringify import stringify_children
 from .html_checker import check_html
@@ -75,9 +76,19 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             cls.clean_metadata_from_xml(definition_xml)
             return {'data': stringify_children(definition_xml)}
         else:
-            # html is special.  cls.filename_extension is 'xml', but if 'filename' is in the definition,
-            # that means to load from .html
-            filepath = "{category}/{name}.html".format(category='html', name=filename)
+            # html is special.  cls.filename_extension is 'xml', but
+            # if 'filename' is in the definition, that means to load
+            # from .html
+            # 'filename' in html pointers is a relative path
+            # (not same as 'html/blah.html' when the pointer is in a directory itself)
+            pointer_path = "{category}/{url_path}".format(category='html',
+                                                  url_path=name_to_pathname(location.name))
+            base = path(pointer_path).dirname()
+            #log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
+            filepath = "{base}/{name}.html".format(base=base, name=filename)
+            #log.debug("looking for html file for {0} at {1}".format(location, filepath))
+
+
 
             # VS[compat]
             # TODO (cpennington): If the file doesn't exist at the right path,
@@ -128,13 +139,18 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             pass
 
         # Not proper format.  Write html to file, return an empty tag
-        filepath = u'{category}/{name}.html'.format(category=self.category,
-                                                    name=self.url_name)
+        pathname = name_to_pathname(self.url_name)
+        pathdir = path(pathname).dirname()
+        filepath = u'{category}/{pathname}.html'.format(category=self.category,
+                                                    pathname=pathname)
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
             file.write(self.definition['data'])
 
+        # write out the relative name
+        relname = path(pathname).basename()
+
         elt = etree.Element('html')
-        elt.set("filename", self.url_name)
+        elt.set("filename", relname)
         return elt

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -84,11 +84,14 @@ class @Problem
   # stuff if a div w a class is found
 
   setupInputTypes: =>
+    @inputtypeDisplays = {}
     @el.find(".capa_inputtype").each (index, inputtype) =>
       classes = $(inputtype).attr('class').split(' ')
+      id = $(inputtype).attr('id')
       for cls in classes
         setupMethod = @inputtypeSetupMethods[cls]
-        setupMethod(inputtype) if setupMethod?
+        if setupMethod?
+          @inputtypeDisplays[id] = setupMethod(inputtype)
 
   executeProblemScripts: (callback=null) ->
 
@@ -192,8 +195,11 @@ class @Problem
     if file_not_selected
       errors.push 'You did not select any files to submit'
 
-    if errors.length > 0
-        alert errors.join("\n")
+    error_html = '<ul>\n'
+    for error in errors
+      error_html += '<li>' + error + '</li>\n'
+    error_html += '</ul>'
+    @gentle_alert error_html
 
     abort_submission = file_too_large or file_not_selected or unallowed_file_submitted or required_files_not_submitted
 
@@ -208,7 +214,7 @@ class @Problem
             @render(response.contents)
             @updateProgress response
           else
-            alert(response.success)
+            @gentle_alert response.success
     
     if not abort_submission
       $.ajaxWithPrefix("#{@url}/problem_check", settings)
@@ -220,8 +226,10 @@ class @Problem
         when 'incorrect', 'correct'
           @render(response.contents)
           @updateProgress response
+          if @el.hasClass 'showed'
+            @el.removeClass 'showed'
         else
-          alert(response.success)
+          @gentle_alert response.success
 
   reset: =>
     Logger.log 'problem_reset', @answers
@@ -243,6 +251,17 @@ class @Problem
               @$("label[for='input_#{key}_#{choice}']").attr correct_answer: 'true'
           else
             @$("#answer_#{key}, #solution_#{key}").html(value)
+
+        # TODO remove the above once everything is extracted into its own
+        # inputtype functions.
+
+        @el.find(".capa_inputtype").each (index, inputtype) =>
+            classes = $(inputtype).attr('class').split(' ')
+            for cls in classes
+              display = @inputtypeDisplays[$(inputtype).attr('id')]
+              showMethod = @inputtypeShowAnswerMethods[cls]
+              showMethod(inputtype, display, answers) if showMethod?
+
         MathJax.Hub.Queue ["Typeset", MathJax.Hub]
         @$('.show').val 'Hide Answer'
         @el.addClass 'showed'
@@ -253,11 +272,26 @@ class @Problem
       @el.removeClass 'showed'
       @$('.show').val 'Show Answer'
 
+      @el.find(".capa_inputtype").each (index, inputtype) =>
+        display = @inputtypeDisplays[$(inputtype).attr('id')]
+        classes = $(inputtype).attr('class').split(' ')
+        for cls in classes
+          hideMethod = @inputtypeHideAnswerMethods[cls]
+          hideMethod(inputtype, display) if hideMethod?
+
+  gentle_alert: (msg) =>
+    if @el.find('.capa_alert').length
+      @el.find('.capa_alert').remove()
+    alert_elem = "<div class='capa_alert'>" + msg + "</div>"
+    @el.find('.action').after(alert_elem)
+    @el.find('.capa_alert').css(opacity: 0).animate(opacity: 1, 700)
+
   save: =>
     Logger.log 'problem_save', @answers
     $.postWithPrefix "#{@url}/problem_save", @answers, (response) =>
       if response.success
-        alert 'Saved'
+        saveMessage = "Your answers have been saved but not graded. Hit 'Check' to grade them."
+        @gentle_alert saveMessage
       @updateProgress response
 
   refreshMath: (event, element) =>
@@ -293,8 +327,35 @@ class @Problem
       problemState  = data.data("problem_state")
       displayClass  = window[data.data('display_class')]
 
+      if evaluation == ''
+          evaluation = null
+
       container = $(element).find(".javascriptinput_container")
       submissionField = $(element).find(".javascriptinput_input")
 
       display = new displayClass(problemState, submission, evaluation, container, submissionField, params)
       display.render()
+
+      return display
+
+  inputtypeShowAnswerMethods:
+    choicegroup: (element, display, answers) =>
+      element = $(element)
+      for key, value of answers
+        element.find('input').attr('disabled', 'disabled')
+        for choice in value
+          element.find("label[for='input_#{key}_#{choice}']").addClass 'choicegroup_correct'
+
+    javascriptinput: (element, display, answers) =>
+      answer_id = $(element).attr('id').split("_")[1...].join("_")
+      answer = JSON.parse(answers[answer_id])
+      display.showAnswer(answer)
+
+  inputtypeHideAnswerMethods:
+    choicegroup: (element, display) =>
+      element = $(element)
+      element.find('input').attr('disabled', null)
+      element.find('label').removeClass('choicegroup_correct')
+
+    javascriptinput: (element, display) =>
+      display.hideAnswer()

common/lib/xmodule/xmodule/js/src/discussion/display.coffee

+class @InlineDiscussion
+  constructor: (element) ->
+    @el = $(element).find('.discussion-module')
+    @view = new DiscussionModuleView(el: @el)

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -2,6 +2,7 @@ class @Sequence
   constructor: (element) ->
     @el = $(element).find('.sequence')
     @contents = @$('.seq_contents')
+    @num_contents = @contents.length
     @id = @el.data('id')
     @modx_url = @el.data('course_modx_root')
     @initProgress()
@@ -86,13 +87,23 @@ class @Sequence
       XModule.loadModules('display', @$('#seq_content'))
 
       MathJax.Hub.Queue(["Typeset", MathJax.Hub, "seq_content"]) # NOTE: Actually redundant. Some other MathJax call also being performed
+      window.update_schematics() # For embedded circuit simulator exercises in 6.002x
+
       @position = new_position
       @toggleArrows()
       @hookUpProgressEvent()
 
+      sequence_links = @$('#seq_content a.seqnav')
+      sequence_links.click @goto
+
   goto: (event) =>
     event.preventDefault()
+    if $(event.target).hasClass 'seqnav' # Links from courseware <a class='seqnav' href='n'>...</a>
+      new_position = $(event.target).attr('href')
+    else # Tab links generated by backend template
       new_position = $(event.target).data('element')
+
+    if (1 <= new_position) and (new_position <= @num_contents)
       Logger.log "seq_goto", old: @position, new: new_position, id: @id
     
       # On Sequence chage, destroy any existing polling thread 
@@ -102,6 +113,8 @@ class @Sequence
         delete window.queuePollerID
 
       @render new_position
+    else
+      alert 'Sequence error! Cannot navigate to tab ' + new_position + 'in the current SequenceModule. Please contact the course staff.'
 
   next: (event) =>
     event.preventDefault()

common/lib/xmodule/xmodule/js/src/video/display.coffee

@@ -3,6 +3,7 @@ class @Video
     @el = $(element).find('.video')
     @id = @el.attr('id').replace(/video_/, '')
     @caption_data_dir = @el.data('caption-data-dir')
+    @show_captions = @el.data('show-captions') == "true"
     window.player = null
     @el = $("#video_#{@id}")
     @parseVideos @el.data('streams')

common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee

 class @VideoCaption extends Subview
+  initialize: ->
+    @loaded = false  
+
   bind: ->
     $(window).bind('resize', @resize)
     @$('.hide-subtitles').click @toggle
@@ -10,8 +13,12 @@ class @VideoCaption extends Subview
     "/static/#{@captionDataDir}/subs/#{@youtubeId}.srt.sjson"
 
   render: ->
+    # TODO: make it so you can have a video with no captions.
+    #@$('.video-wrapper').after """
+    #  <ol class="subtitles"><li>Attempting to load captions...</li></ol>
+    #  """
     @$('.video-wrapper').after """
-      <ol class="subtitles"><li>Attempting to load captions...</li></ol>
+      <ol class="subtitles"></ol>
       """
     @$('.video-controls .secondary-controls').append """
       <a href="#" class="hide-subtitles" title="Turn off captions">Captions</a>
@@ -24,6 +31,8 @@ class @VideoCaption extends Subview
       @captions = captions.text
       @start = captions.start
 
+      @loaded = true
+
       if onTouchBasedDevice()
         $('.subtitles li').html "Caption will be displayed when you start playing the video."
       else
@@ -47,6 +56,7 @@ class @VideoCaption extends Subview
     @rendered = true
 
   search: (time) ->
+    if @loaded
       min = 0
       max = @start.length - 1
 
@@ -56,17 +66,19 @@ class @VideoCaption extends Subview
           max = index - 1
         if time >= @start[index]
           min = index
-
       return min
 
   play: ->
+    if @loaded
       @renderCaption() unless @rendered
       @playing = true
 
   pause: ->
+    if @loaded
       @playing = false
 
   updatePlayTime: (time) ->
+    if @loaded
       # This 250ms offset is required to match the video speed
       time = Math.round(Time.convert(time, @currentSpeed, '1.0') * 1000 + 250)
       newIndex = @search time

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -13,18 +13,21 @@ from xmodule.errortracker import ErrorLog, make_error_tracker
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
+
 URL_RE = re.compile("""
     (?P<tag>[^:]+)://
     (?P<org>[^/]+)/
     (?P<course>[^/]+)/
     (?P<category>[^/]+)/
-    (?P<name>[^/]+)
-    (/(?P<revision>[^/]+))?
+    (?P<name>[^@]+)
+    (@(?P<revision>[^/]+))?
     """, re.VERBOSE)
 
 # TODO (cpennington): We should decide whether we want to expand the
 # list of valid characters in a location
 INVALID_CHARS = re.compile(r"[^\w.-]")
+# Names are allowed to have colons.
+INVALID_CHARS_NAME = re.compile(r"[^\w.:-]")
 
 _LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
 
@@ -34,7 +37,7 @@ class Location(_LocationBase):
     Encodes a location.
 
     Locations representations of URLs of the
-    form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+    form {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
     However, they can also be represented a dictionaries (specifying each component),
     tuples or list (specified in order), or as strings of the url
@@ -81,7 +84,7 @@ class Location(_LocationBase):
 
         location - Can be any of the following types:
             string: should be of the form
-                    {tag}://{org}/{course}/{category}/{name}[/{revision}]
+                    {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
             list: should be of the form [tag, org, course, category, name, revision]
 
@@ -99,10 +102,11 @@ class Location(_LocationBase):
         ommitted.
 
         Components must be composed of alphanumeric characters, or the
-        characters '_', '-', and '.'
+        characters '_', '-', and '.'.  The name component is additionally allowed to have ':',
+        which is interpreted specially for xml storage.
 
-        Components may be set to None, which may be interpreted by some contexts
-        to mean wildcard selection
+        Components may be set to None, which may be interpreted in some contexts
+        to mean wildcard selection.
         """
 
 
@@ -116,14 +120,23 @@ class Location(_LocationBase):
             return _LocationBase.__new__(_cls, *([None] * 6))
 
         def check_dict(dict_):
-            check_list(dict_.itervalues())
+            # Order matters, so flatten out into a list
+            keys = ['tag', 'org', 'course', 'category', 'name', 'revision']
+            list_ = [dict_[k] for k in keys]
+            check_list(list_)
 
         def check_list(list_):
-            for val in list_:
-                if val is not None and INVALID_CHARS.search(val) is not None:
+            def check(val, regexp):
+                if val is not None and regexp.search(val) is not None:
                     log.debug('invalid characters val="%s", list_="%s"' % (val, list_))
                     raise InvalidLocationError(location)
 
+            list_ = list(list_)
+            for val in list_[:4] + [list_[5]]:
+                check(val, INVALID_CHARS)
+            # names allow colons
+            check(list_[4], INVALID_CHARS_NAME)
+
         if isinstance(location, basestring):
             match = URL_RE.match(location)
             if match is None:
@@ -162,7 +175,7 @@ class Location(_LocationBase):
         """
         url = "{tag}://{org}/{course}/{category}/{name}".format(**self.dict())
         if self.revision:
-            url += "/" + self.revision
+            url += "@" + self.revision
         return url
 
     def html_id(self):
@@ -170,6 +183,7 @@ class Location(_LocationBase):
         Return a string with a version of the location that is safe for use in
         html id attributes
         """
+        # TODO: is ':' ok in html ids?
         return "-".join(str(v) for v in self.list()
                         if v is not None).replace('.', '_')
 

common/lib/xmodule/xmodule/modulestore/django.py

@@ -5,8 +5,8 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
 """
 
 from __future__ import absolute_import
-
 from importlib import import_module
+from os import environ
 
 from django.conf import settings
 
@@ -43,3 +43,8 @@ def modulestore(name='default'):
         )
 
     return _MODULESTORES[name]
+
+# if 'DJANGO_SETTINGS_MODULE' in environ:
+#     # Initialize the modulestores immediately
+#     for store_name in settings.MODULESTORE:
+#         modulestore(store_name)

common/lib/xmodule/xmodule/modulestore/search.py

@@ -6,15 +6,14 @@ from .exceptions import (ItemNotFoundError, NoPathToItem)
 from . import ModuleStore, Location
 
 
-def path_to_location(modulestore, location, course_name=None):
+def path_to_location(modulestore, course_id, location):
     '''
     Try to find a course_id/chapter/section[/position] path to location in
     modulestore.  The courseware insists that the first level in the course is
     chapter, but any kind of module can be a "section".
 
     location: something that can be passed to Location
-    course_name: [optional].  If not None, restrict search to paths
-        in that course.
+    course_id: Search for paths in this course.
 
     raise ItemNotFoundError if the location doesn't exist.
 
@@ -27,7 +26,7 @@ def path_to_location(modulestore, location, course_name=None):
     A location may be accessible via many paths. This method may
     return any valid path.
 
-    If the section is a sequence, position will be the position
+    If the section is a sequential or vertical, position will be the position
     of this location in that sequence.  Otherwise, position will
     be None. TODO (vshnayder): Not true yet.
     '''
@@ -41,7 +40,7 @@ def path_to_location(modulestore, location, course_name=None):
             xs = xs[1]
         return p
 
-    def find_path_to_course(location, course_name=None):
+    def find_path_to_course():
         '''Find a path up the location graph to a node with the
         specified category.
 
@@ -69,7 +68,8 @@ def path_to_location(modulestore, location, course_name=None):
 
             # print 'Processing loc={0}, path={1}'.format(loc, path)
             if loc.category == "course":
-                if course_name is None or course_name == loc.name:
+                # confirm that this is the right course
+                if course_id == CourseDescriptor.location_to_id(loc):
                     # Found it!
                     path = (loc, path)
                     return flatten(path)
@@ -81,17 +81,34 @@ def path_to_location(modulestore, location, course_name=None):
         # If we're here, there is no path
         return None
 
-    path = find_path_to_course(location, course_name)
+    path = find_path_to_course()
     if path is None:
-        raise(NoPathToItem(location))
+        raise NoPathToItem(location)
 
     n = len(path)
     course_id = CourseDescriptor.location_to_id(path[0])
     # pull out the location names
     chapter = path[1].name if n > 1 else None
     section = path[2].name if n > 2 else None
-
-    # TODO (vshnayder): not handling position at all yet...
+    # Figure out the position
     position = None
 
+    # This block of code will find the position of a module within a nested tree
+    # of modules. If a problem is on tab 2 of a sequence that's on tab 3 of a
+    # sequence, the resulting position is 3_2. However, no positional modules
+    # (e.g. sequential and videosequence) currently deal with this form of
+    # representing nested positions. This needs to happen before jumping to a
+    # module nested in more than one positional module will work.
+    if n > 3:
+        position_list = []
+        for path_index in range(2, n-1):
+            category = path[path_index].category
+            if  category == 'sequential' or category == 'videosequence':
+                section_desc = modulestore.get_instance(course_id, path[path_index])
+                child_locs = [c.location for c in section_desc.get_children()]
+                # positions are 1-indexed, and should be strings to be consistent with
+                # url parsing.
+                position_list.append(str(child_locs.index(path[path_index+1]) + 1))
+        position = "_".join(position_list)
+
     return (course_id, chapter, section, position)

common/lib/xmodule/xmodule/modulestore/tests/__init__.py

+from path import path
+
+# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
+# to   ~/mitx_all/mitx/common/test
+TEST_DIR = path(__file__).abspath().dirname()
+for i in range(5):
+    TEST_DIR = TEST_DIR.dirname()
+TEST_DIR = TEST_DIR / 'test'
+
+DATA_DIR = TEST_DIR / 'data'
+
+

common/lib/xmodule/xmodule/modulestore/tests/test_location.py

@@ -10,7 +10,7 @@ def check_string_roundtrip(url):
 
 def test_string_roundtrip():
     check_string_roundtrip("tag://org/course/category/name")
-    check_string_roundtrip("tag://org/course/category/name/revision")
+    check_string_roundtrip("tag://org/course/category/name@revision")
 
 
 input_dict = {
@@ -21,18 +21,28 @@ input_dict = {
     'org': 'org'
 }
 
+
+also_valid_dict = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name:more_name',
+    'org': 'org'
+}
+
+
 input_list = ['tag', 'org', 'course', 'category', 'name']
 
 input_str = "tag://org/course/category/name"
-input_str_rev = "tag://org/course/category/name/revision"
+input_str_rev = "tag://org/course/category/name@revision"
 
-valid = (input_list, input_dict, input_str, input_str_rev)
+valid = (input_list, input_dict, input_str, input_str_rev, also_valid_dict)
 
 invalid_dict = {
     'tag': 'tag',
     'course': 'course',
     'category': 'category',
-    'name': 'name/more_name',
+    'name': 'name@more_name',
     'org': 'org'
 }
 
@@ -45,8 +55,9 @@ invalid_dict2 = {
 }
 
 invalid = ("foo", ["foo"], ["foo", "bar"],
-           ["foo", "bar", "baz", "blat", "foo/bar"],
-           "tag://org/course/category/name with spaces/revision",
+           ["foo", "bar", "baz", "blat:blat", "foo:bar"],  # ':' ok in name, not in category
+           "tag://org/course/category/name with spaces@revision",
+           "tag://org/course/category/name/with/slashes@revision",
            invalid_dict,
            invalid_dict2)
 
@@ -62,16 +73,15 @@ def test_dict():
     assert_equals(dict(revision=None, **input_dict), Location(input_dict).dict())
 
     input_dict['revision'] = 'revision'
-    assert_equals("tag://org/course/category/name/revision", Location(input_dict).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_dict).url())
     assert_equals(input_dict, Location(input_dict).dict())
 
-
 def test_list():
     assert_equals("tag://org/course/category/name", Location(input_list).url())
     assert_equals(input_list + [None], Location(input_list).list())
 
     input_list.append('revision')
-    assert_equals("tag://org/course/category/name/revision", Location(input_list).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_list).url())
     assert_equals(input_list, Location(input_list).list())
 
 
@@ -87,8 +97,10 @@ def test_none():
 def test_invalid_locations():
     assert_raises(InvalidLocationError, Location, "foo")
     assert_raises(InvalidLocationError, Location, ["foo", "bar"])
+    assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat/blat", "foo"])
     assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat", "foo/bar"])
-    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces/revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces@revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name/revision")
 
 
 def test_equality():

common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py

+from nose.tools import assert_equals, assert_raises, assert_not_equals, with_setup
+
+from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
+from xmodule.modulestore.search import path_to_location
+
+def check_path_to_location(modulestore):
+    '''Make sure that path_to_location works: should be passed a modulestore
+    with the toy and simple courses loaded.'''
+    should_work = (
+        ("i4x://edX/toy/video/Welcome",
+         ("edX/toy/2012_Fall", "Overview", "Welcome", None)),
+        ("i4x://edX/toy/chapter/Overview",
+         ("edX/toy/2012_Fall", "Overview", None, None)),
+        )
+    course_id = "edX/toy/2012_Fall"
+
+    for location, expected in should_work:
+        assert_equals(path_to_location(modulestore, course_id, location), expected)
+
+    not_found = (
+        "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
+        )
+    for location in not_found:
+        assert_raises(ItemNotFoundError, path_to_location, modulestore, course_id, location)
+
+    # Since our test files are valid, there shouldn't be any
+    # elements with no path to them.  But we can look for them in
+    # another course.
+    no_path = (
+        "i4x://edX/simple/video/Lost_Video",
+        )
+    for location in no_path:
+        assert_raises(NoPathToItem, path_to_location, modulestore, course_id, location)
+

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

 import pymongo
 
 from nose.tools import assert_equals, assert_raises, assert_not_equals, with_setup
-from path import path
 from pprint import pprint
 
 from xmodule.modulestore import Location
-from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
 from xmodule.modulestore.mongo import MongoModuleStore
 from xmodule.modulestore.xml_importer import import_from_xml
-from xmodule.modulestore.search import path_to_location
 
-# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
-# to   ~/mitx_all/mitx/common/test
-TEST_DIR = path(__file__).abspath().dirname()
-for i in range(5):
-    TEST_DIR = TEST_DIR.dirname()
-TEST_DIR = TEST_DIR / 'test'
-
-DATA_DIR = TEST_DIR / 'data'
+from .test_modulestore import check_path_to_location
+from . import DATA_DIR
 
 
 HOST = 'localhost'
@@ -110,27 +101,5 @@ class TestMongoModuleStore(object):
 
     def test_path_to_location(self):
         '''Make sure that path_to_location works'''
-        should_work = (
-            ("i4x://edX/toy/video/Welcome",
-             ("edX/toy/2012_Fall", "Overview", "Welcome", None)),
-            ("i4x://edX/toy/chapter/Overview",
-             ("edX/toy/2012_Fall", "Overview", None, None)),
-            )
-        for location, expected in should_work:
-            assert_equals(path_to_location(self.store, location), expected)
-
-        not_found = (
-            "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
-            )
-        for location in not_found:
-            assert_raises(ItemNotFoundError, path_to_location, self.store, location)
-
-        # Since our test files are valid, there shouldn't be any
-        # elements with no path to them.  But we can look for them in
-        # another course.
-        no_path = (
-            "i4x://edX/simple/video/Lost_Video",
-            )
-        for location in no_path:
-            assert_raises(NoPathToItem, path_to_location, self.store, location, "toy")
+        check_path_to_location(self.store)
 

common/lib/xmodule/xmodule/modulestore/tests/test_xml.py

+from xmodule.modulestore import Location
+from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.modulestore.xml_importer import import_from_xml
+
+from .test_modulestore import check_path_to_location
+from . import DATA_DIR
+
+class TestXMLModuleStore(object):
+    def test_path_to_location(self):
+        """Make sure that path_to_location works properly"""
+
+        print "Starting import"
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
+        print "finished import"
+        
+        check_path_to_location(modulestore)

common/lib/xmodule/xmodule/modulestore/xml.py

+import hashlib
 import json
 import logging
 import os
@@ -36,21 +37,102 @@ def clean_out_mako_templating(xml_string):
 
 class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
     def __init__(self, xmlstore, course_id, course_dir,
-                 policy, error_tracker, **kwargs):
+                 policy, error_tracker, parent_tracker, **kwargs):
         """
         A class that handles loading from xml.  Does some munging to ensure that
         all elements have unique slugs.
 
         xmlstore: the XMLModuleStore to store the loaded modules in
         """
-        self.unnamed_modules = 0
-        self.used_slugs = set()
+        self.unnamed = defaultdict(int)     # category -> num of new url_names for that category
+        self.used_names = defaultdict(set)  # category -> set of used url_names
         self.org, self.course, self.url_name = course_id.split('/')
 
         def process_xml(xml):
             """Takes an xml string, and returns a XModuleDescriptor created from
             that xml.
             """
+
+            def make_name_unique(xml_data):
+                """
+                Make sure that the url_name of xml_data is unique.  If a previously loaded
+                unnamed descriptor stole this element's url_name, create a new one.
+
+                Removes 'slug' attribute if present, and adds or overwrites the 'url_name' attribute.
+                """
+                # VS[compat]. Take this out once course conversion is done (perhaps leave the uniqueness check)
+
+                # tags that really need unique names--they store (or should store) state.
+                need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
+
+                attr = xml_data.attrib
+                tag = xml_data.tag
+                id = lambda x: x
+                # Things to try to get a name, in order  (key, cleaning function, remove key after reading?)
+                lookups = [('url_name', id, False),
+                           ('slug', id, True),
+                           ('name', Location.clean, False),
+                           ('display_name', Location.clean, False)]
+
+                url_name = None
+                for key, clean, remove in lookups:
+                    if key in attr:
+                        url_name = clean(attr[key])
+                        if remove:
+                            del attr[key]
+                        break
+
+                def fallback_name(orig_name=None):
+                    """Return the fallback name for this module.  This is a function instead of a variable
+                    because we want it to be lazy."""
+                    # append the hash of the content--the first 12 bytes should be plenty.
+                    orig_name = "_" + orig_name if orig_name is not None else ""
+                    return tag + orig_name + "_" + hashlib.sha1(xml).hexdigest()[:12]
+
+                def looks_like_fallback(tag, url_name):
+                    """Does this look like something that came from fallback_name()?"""
+                    return url_name.startswith(tag) and re.search('[0-9a-fA-F]{12}$', url_name)
+
+                # Fallback if there was nothing we could use:
+                if url_name is None or url_name == "":
+                    url_name = fallback_name()
+                    # Don't log a warning--we don't need this in the log.  Do
+                    # put it in the error tracker--content folks need to see it.
+
+                    if tag in need_uniq_names:
+                        error_tracker("PROBLEM: no name of any kind specified for {tag}.  Student "
+                                      "state will not be properly tracked for this module.  Problem xml:"
+                                      " '{xml}...'".format(tag=tag, xml=xml[:100]))
+                    else:
+                        # TODO (vshnayder): We may want to enable this once course repos are cleaned up.
+                        # (or we may want to give up on the requirement for non-state-relevant issues...)
+                        #error_tracker("WARNING: no name specified for module. xml='{0}...'".format(xml[:100]))
+                        pass
+
+                # Make sure everything is unique
+                if url_name in self.used_names[tag]:
+                    # Always complain about modules that store state.  If it
+                    # doesn't store state, don't complain about things that are
+                    # hashed.
+                    if tag in need_uniq_names or not looks_like_fallback(tag, url_name):
+                        msg = ("Non-unique url_name in xml.  This may break state tracking for content."
+                               "  url_name={0}.  Content={1}".format(url_name, xml[:100]))
+                        error_tracker("PROBLEM: " + msg)
+                        log.warning(msg)
+                        # Just set name to fallback_name--if there are multiple things with the same fallback name,
+                        # they are actually identical, so it's fragile, but not immediately broken.
+
+                        # TODO (vshnayder): if the tag is a pointer tag, this will
+                        # break the content because we won't have the right link.
+                        # That's also a legitimate attempt to reuse the same content
+                        # from multiple places.  Once we actually allow that, we'll
+                        # need to update this to complain about non-unique names for
+                        # definitions, but allow multiple uses.
+                        url_name = fallback_name(url_name)
+
+                self.used_names[tag].add(url_name)
+                xml_data.set('url_name', url_name)
+
             try:
                 # VS[compat]
                 # TODO (cpennington): Remove this once all fall 2012 courses
@@ -62,38 +144,17 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     err=str(err), xml=xml))
                 raise
 
-            # VS[compat]. Take this out once course conversion is done
-            if xml_data.get('slug') is None and xml_data.get('url_name') is None:
-                if xml_data.get('name'):
-                    slug = Location.clean(xml_data.get('name'))
-                elif xml_data.get('display_name'):
-                    slug = Location.clean(xml_data.get('display_name'))
-                else:
-                    self.unnamed_modules += 1
-                    slug = '{tag}_{count}'.format(tag=xml_data.tag,
-                                                  count=self.unnamed_modules)
-
-                while slug in self.used_slugs:
-                    self.unnamed_modules += 1
-                    slug = '{slug}_{count}'.format(slug=slug,
-                                                   count=self.unnamed_modules)
-
-                self.used_slugs.add(slug)
-                # log.debug('-> slug=%s' % slug)
-                xml_data.set('url_name', slug)
+            make_name_unique(xml_data)
 
             descriptor = XModuleDescriptor.load_from_xml(
                 etree.tostring(xml_data), self, self.org,
                 self.course, xmlstore.default_class)
-
-            #log.debug('==> importing descriptor location %s' %
-            #          repr(descriptor.location))
             descriptor.metadata['data_dir'] = course_dir
 
             xmlstore.modules[course_id][descriptor.location] = descriptor
 
-            if xmlstore.eager:
-                descriptor.get_children()
+            for child in descriptor.get_children():
+                parent_tracker.add_parent(child.location, descriptor.location)
             return descriptor
 
         render_template = lambda: ''
@@ -109,12 +170,51 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                                   error_tracker, process_xml, policy, **kwargs)
 
 
+class ParentTracker(object):
+    """A simple class to factor out the logic for tracking location parent pointers."""
+    def __init__(self):
+        """
+        Init
+        """
+        # location -> set(parents).  Not using defaultdict because we care about the empty case.
+        self._parents = dict()
+
+    def add_parent(self, child, parent):
+        """
+        Add a parent of child location to the set of parents.  Duplicate calls have no effect.
+
+        child and parent must be something that can be passed to Location.
+        """
+        child = Location(child)
+        parent = Location(parent)
+        s = self._parents.setdefault(child, set())
+        s.add(parent)
+
+    def is_known(self, child):
+        """
+        returns True iff child has some parents.
+        """
+        child = Location(child)
+        return child in self._parents
+
+    def make_known(self, location):
+        """Tell the parent tracker about an object, without registering any
+        parents for it.  Used for the top level course descriptor locations."""
+        self._parents.setdefault(location, set())
+
+    def parents(self, child):
+        """
+        Return a list of the parents of this child.  If not is_known(child), will throw a KeyError
+        """
+        child = Location(child)
+        return list(self._parents[child])
+
+
 class XMLModuleStore(ModuleStoreBase):
     """
     An XML backed ModuleStore
     """
-    def __init__(self, data_dir, default_class=None, eager=False,
-                 course_dirs=None):
+    def __init__(self, data_dir, default_class=None, course_dirs=None):
         """
         Initialize an XMLModuleStore from data_dir
 
@@ -123,15 +223,11 @@ class XMLModuleStore(ModuleStoreBase):
         default_class: dot-separated string defining the default descriptor
             class to use if none is specified in entry_points
 
-        eager: If true, load the modules children immediately to force the
-            entire course tree to be parsed
-
         course_dirs: If specified, the list of course_dirs to load. Otherwise,
             load all course dirs
         """
         ModuleStoreBase.__init__(self)
 
-        self.eager = eager
         self.data_dir = path(data_dir)
         self.modules = defaultdict(dict)  # course_id -> dict(location -> XModuleDescriptor)
         self.courses = {}  # course_dir -> XModuleDescriptor for the course
@@ -144,10 +240,7 @@ class XMLModuleStore(ModuleStoreBase):
             class_ = getattr(import_module(module_path), class_name)
             self.default_class = class_
 
-        # TODO (cpennington): We need a better way of selecting specific sets of
-        # debug messages to enable. These were drowning out important messages
-        #log.debug('XMLModuleStore: eager=%s, data_dir = %s' % (eager, self.data_dir))
-        #log.debug('default_class = %s' % self.default_class)
+        self.parent_tracker = ParentTracker()
 
         # If we are specifically asked for missing courses, that should
         # be an error.  If we are asked for "all" courses, find the ones
@@ -179,6 +272,7 @@ class XMLModuleStore(ModuleStoreBase):
         if course_descriptor is not None:
             self.courses[course_dir] = course_descriptor
             self._location_errors[course_descriptor.location] = errorlog
+            self.parent_tracker.make_known(course_descriptor.location)
         else:
             # Didn't load course.  Instead, save the errors elsewhere.
             self.errored_courses[course_dir] = errorlog
@@ -297,7 +391,7 @@ class XMLModuleStore(ModuleStoreBase):
 
 
             course_id = CourseDescriptor.make_id(org, course, url_name)
-            system = ImportSystem(self, course_id, course_dir, policy, tracker)
+            system = ImportSystem(self, course_id, course_dir, policy, tracker, self.parent_tracker)
 
             course_descriptor = system.process_xml(etree.tostring(course_data))
 
@@ -408,3 +502,19 @@ class XMLModuleStore(ModuleStoreBase):
         metadata: A nested dictionary of module metadata
         """
         raise NotImplementedError("XMLModuleStores are read-only")
+
+    def get_parent_locations(self, location):
+        '''Find all locations that are the parents of this location.  Needed
+        for path_to_location().
+
+        If there is no data at location in this modulestore, raise
+            ItemNotFoundError.
+
+        returns an iterable of things that can be passed to Location.  This may
+        be empty if there are no parents.
+        '''
+        location = Location.ensure_fully_specified(location)
+        if not self.parent_tracker.is_known(location):
+            raise ItemNotFoundError(location)
+
+        return self.parent_tracker.parents(location)

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -6,7 +6,7 @@ from .exceptions import DuplicateItemError
 log = logging.getLogger(__name__)
 
 
-def import_from_xml(store, data_dir, course_dirs=None, eager=True,
+def import_from_xml(store, data_dir, course_dirs=None, 
                     default_class='xmodule.raw_module.RawDescriptor'):
     """
     Import the specified xml data_dir into the "store" modulestore,
@@ -19,7 +19,6 @@ def import_from_xml(store, data_dir, course_dirs=None, eager=True,
     module_store = XMLModuleStore(
         data_dir,
         default_class=default_class,
-        eager=eager,
         course_dirs=course_dirs
     )
     for course_id in module_store.modules.keys():

common/lib/xmodule/xmodule/stringify.py

@@ -12,9 +12,17 @@ def stringify_children(node):
     fixed from
     http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
     '''
-    parts = ([node.text] +
-            list(chain(*([etree.tostring(c), c.tail]
-                         for c in node.getchildren())
-                         )))
+    # Useful things to know:
+
+    # node.tostring() -- generates xml for the node, including start
+    #                 and end tags.  We'll use this for the children.
+    # node.text -- the text after the end of a start tag to the start
+    #                 of the first child
+    # node.tail -- the text after the end this tag to the start of the
+    #                 next element.
+    parts = [node.text]
+    for c in node.getchildren():
+        parts.append(etree.tostring(c, with_tail=True))
+
     # filter removes possible Nones in texts and tails
     return ''.join(filter(None, parts))

common/lib/xmodule/xmodule/tests/__init__.py

@@ -19,6 +19,8 @@ import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
+from capa.xqueue_interface import dateformat
+from datetime import datetime
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
 from xmodule.graders import Score, aggregate_scores
@@ -35,8 +37,9 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
-    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'waittime': 10},
+    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
+    anonymous_student_id = 'student'
 )
 
 
@@ -282,29 +285,65 @@ class StringResponseWithHintTest(unittest.TestCase):
 class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
+    TODO: Add tests for external grader messages
     '''
+    @staticmethod
+    def make_queuestate(key, time):
+        timestr = datetime.strftime(time, dateformat)
+        return {'key': key, 'time': timestr}
+
+    def test_is_queued(self):
+        ''' 
+        Simple test of whether LoncapaProblem knows when it's been queued
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+            
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+            test_lcp.correct_map.update(cmap)
+
+            self.assertEquals(test_lcp.is_queued(), False)
+
+            # Now we queue the LCP
+            cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids): 
+                queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
+
+            self.assertEquals(test_lcp.is_queued(), True)
+
+
     def test_update_score(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+        '''
+        Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the 'queued' state
-        old_cmap = CorrectMap()
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        numAnswers = len(answer_ids)
-        for i in range(numAnswers):
-            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuekey=1000 + i))
+            answer_ids = sorted(test_lcp.get_question_answers())
 
-        # TODO: Message format inherited from ExternalResponse
-        #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
-        #incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            old_cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids):
+                queuekey = 1000 + i
+                queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
+                old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
-        # New message format common to external graders
-        correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
-        incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
+            # Message format common to external graders
+            grader_msg = '<span>MESSAGE</span>' # Must be valid XML
+            correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg': grader_msg})
+            incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg': grader_msg})
 
             xserver_msgs = {'correct': correct_score_msg,
-                        'incorrect': incorrect_score_msg,
-                        }
+                            'incorrect': incorrect_score_msg,}
 
             # Incorrect queuekey, state should not be updated
             for correctness in ['correct', 'incorrect']:
@@ -314,32 +353,68 @@ class CodeResponseTest(unittest.TestCase):
                 test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
                 self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-            for i in range(numAnswers):
-                self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
+                for answer_id in answer_ids:
+                    self.assertTrue(test_lcp.correct_map.is_queued(answer_id))  # Should be still queued, since message undelivered
 
             # Correct queuekey, state should be updated
             for correctness in ['correct', 'incorrect']:
-            for i in range(numAnswers):  # Target specific answer_id's
+                for i, answer_id in enumerate(answer_ids):
                     test_lcp.correct_map = CorrectMap()
                     test_lcp.correct_map.update(old_cmap)
 
                     new_cmap = CorrectMap()
                     new_cmap.update(old_cmap)
                     npoints = 1 if correctness=='correct' else 0
-                new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuekey=None)
+                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg=grader_msg, queuestate=None)
 
                     test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                     self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
 
-                for j in range(numAnswers):
+                    for j, test_id in enumerate(answer_ids):
                         if j == i:
-                        self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
+                            self.assertFalse(test_lcp.correct_map.is_queued(test_id))  # Should be dequeued, message delivered
                         else:
-                        self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
+                            self.assertTrue(test_lcp.correct_map.is_queued(test_id))  # Should be queued, message undelivered
+    
+
+    def test_recentmost_queuetime(self):
+        '''
+        Test whether the LoncapaProblem knows about the time of queue requests
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+            test_lcp.correct_map.update(cmap)
+            
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids):
+                queuekey = 1000 + i
+                latest_timestamp = datetime.now()
+                queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
+
+            # Queue state only tracks up to second
+            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp, dateformat), dateformat)
+
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
 
         def test_convert_files_to_filenames(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
-        fp = open(problem_file)
+            '''
+            Test whether file objects are converted to filenames without altering other structures
+            '''
+            problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+            with open(problem_file) as fp:
                 answers_with_file = {'1_2_1': 'String-based answer',
                                      '1_3_1': ['answer1', 'answer2', 'answer3'],
                                      '1_4_1': [fp, fp]}

common/lib/xmodule/xmodule/tests/test_export.py

@@ -49,7 +49,7 @@ class RoundTripTestCase(unittest.TestCase):
         copytree(data_dir / course_dir, root_dir / course_dir)
 
         print "Starting import"
-        initial_import = XMLModuleStore(root_dir, eager=True, course_dirs=[course_dir])
+        initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
 
         courses = initial_import.get_courses()
         self.assertEquals(len(courses), 1)
@@ -66,7 +66,7 @@ class RoundTripTestCase(unittest.TestCase):
             course_xml.write(xml)
 
         print "Starting second import"
-        second_import = XMLModuleStore(root_dir, eager=True, course_dirs=[course_dir])
+        second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
 
         courses2 = second_import.get_courses()
         self.assertEquals(len(courses2), 1)

common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml

@@ -9,91 +9,23 @@
 Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 
 <text>
-Write a program to compute the cube of a number
+Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 

common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml

+<problem>
+<text>
+<h2>Code response</h2>
+
+<p>
+</p>
+
+<text>
+Write a program to compute the square of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def square(n):
+"""
+
+answer = """
+def square(n):
+    return n**2    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testSquare(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: square(%d)'%n
+    return str(square(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testSquare(0))
+   elif test == 2: f.write(testSquare(1))
+   else:  f.write(testSquare())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+<text>
+Write a program to compute the cube of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def cube(n):
+"""
+
+answer = """
+def cube(n):
+    return n**3    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testCube(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: cube(%d)'%n
+    return str(cube(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testCube(0))
+   elif test == 2: f.write(testCube(1))
+   else:  f.write(testCube())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+</text>
+</problem>

common/lib/xmodule/xmodule/tests/test_files/js/compiled/javascriptresponse.js

+// Generated by CoffeeScript 1.3.3
+(function() {
+  var MinimaxProblemDisplay, root,
+    __hasProp = {}.hasOwnProperty,
+    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
+
+  MinimaxProblemDisplay = (function(_super) {
+
+    __extends(MinimaxProblemDisplay, _super);
+
+    function MinimaxProblemDisplay(state, submission, evaluation, container, submissionField, parameters) {
+      this.state = state;
+      this.submission = submission;
+      this.evaluation = evaluation;
+      this.container = container;
+      this.submissionField = submissionField;
+      this.parameters = parameters != null ? parameters : {};
+      MinimaxProblemDisplay.__super__.constructor.call(this, this.state, this.submission, this.evaluation, this.container, this.submissionField, this.parameters);
+    }
+
+    MinimaxProblemDisplay.prototype.render = function() {};
+
+    MinimaxProblemDisplay.prototype.createSubmission = function() {
+      var id, value, _ref, _results;
+      this.newSubmission = {};
+      if (this.submission != null) {
+        _ref = this.submission;
+        _results = [];
+        for (id in _ref) {
+          value = _ref[id];
+          _results.push(this.newSubmission[id] = value);
+        }
+        return _results;
+      }
+    };
+
+    MinimaxProblemDisplay.prototype.getCurrentSubmission = function() {
+      return this.newSubmission;
+    };
+
+    return MinimaxProblemDisplay;
+
+  })(XProblemDisplay);
+
+  root = typeof exports !== "undefined" && exports !== null ? exports : this;
+
+  root.TestProblemDisplay = TestProblemDisplay;
+
+}).call(this);
+;

common/lib/xmodule/xmodule/tests/test_import.py

@@ -193,7 +193,7 @@ class ImportTestCase(unittest.TestCase):
         """Make sure that metadata is inherited properly"""
 
         print "Starting import"
-        initial_import = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+        initial_import = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
 
         courses = initial_import.get_courses()
         self.assertEquals(len(courses), 1)
@@ -216,7 +216,7 @@ class ImportTestCase(unittest.TestCase):
         def get_course(name):
             print "Importing {0}".format(name)
 
-            modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=[name])
+            modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
             courses = modulestore.get_courses()
             self.assertEquals(len(courses), 1)
             return courses[0]
@@ -236,6 +236,10 @@ class ImportTestCase(unittest.TestCase):
         # Also check that the grading policy loaded
         self.assertEqual(two_toys.grade_cutoffs['C'], 0.5999)
 
+        # Also check that keys from policy are run through the
+        # appropriate attribute maps -- 'graded' should be True, not 'true'
+        self.assertEqual(toy.metadata['graded'], True)
+
 
     def test_definition_loading(self):
         """When two courses share the same org and course name and
@@ -245,7 +249,7 @@ class ImportTestCase(unittest.TestCase):
         happen--locations should uniquely name definitions.  But in
         our imperfect XML world, it can (and likely will) happen."""
 
-        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy', 'two_toys'])
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys'])
 
         toy_id = "edX/toy/2012_Fall"
         two_toy_id = "edX/toy/TT_2012_Fall"
@@ -255,3 +259,37 @@ class ImportTestCase(unittest.TestCase):
         two_toy_video =  modulestore.get_instance(two_toy_id, location)
         self.assertEqual(toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh8")
         self.assertEqual(two_toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh9")
+
+
+    def test_colon_in_url_name(self):
+        """Ensure that colons in url_names convert to file paths properly"""
+
+        print "Starting import"
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
+
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+        course_id = course.id
+
+        print "course errors:"
+        for (msg, err) in modulestore.get_item_errors(course.location):
+            print msg
+            print err
+
+        chapters = course.get_children()
+        self.assertEquals(len(chapters), 2)
+
+        ch2 = chapters[1]
+        self.assertEquals(ch2.url_name, "secret:magic")
+
+        print "Ch2 location: ", ch2.location
+
+        also_ch2 = modulestore.get_instance(course_id, ch2.location)
+        self.assertEquals(ch2, also_ch2)
+
+        print "making sure html loaded"
+        cloc = course.location
+        loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
+        html = modulestore.get_instance(course_id, loc)
+        self.assertEquals(html.display_name, "Toy lab")

common/lib/xmodule/xmodule/tests/test_stringify.py

-from nose.tools import assert_equals
+from nose.tools import assert_equals, assert_true, assert_false
 from lxml import etree
 from xmodule.stringify import stringify_children
 
@@ -8,3 +8,32 @@ def test_stringify():
     xml = etree.fromstring(html)
     out = stringify_children(xml)
     assert_equals(out, text)
+
+def test_stringify_again():
+    html = """<html name="Voltage Source Answer" >A voltage source is non-linear!
+<div align="center">
+    <img src="/static/images/circuits/voltage-source.png"/>
+    \(V=V_C\)
+  </div>
+  But it is <a href="http://mathworld.wolfram.com/AffineFunction.html">affine</a>,
+  which means linear except for an offset.
+  </html>
+"""
+
+    html = """<html>A voltage source is non-linear!
+  <div align="center">
+
+  </div>
+  But it is <a href="http://mathworld.wolfram.com/AffineFunction.html">affine</a>,
+  which means linear except for an offset.
+  </html>
+  """
+    xml = etree.fromstring(html)
+    out = stringify_children(xml)
+
+    print "output:"
+    print out
+
+    # Tracking strange content repeating bug
+    # Should appear once
+    assert_equals(out.count("But it is "), 1)

common/lib/xmodule/xmodule/video_module.py

@@ -30,6 +30,7 @@ class VideoModule(XModule):
         xmltree = etree.fromstring(self.definition['data'])
         self.youtube = xmltree.get('youtube')
         self.position = 0
+        self.show_captions = xmltree.get('show_captions', 'true')
 
         if instance_state is not None:
             state = json.loads(instance_state)
@@ -75,6 +76,7 @@ class VideoModule(XModule):
             'display_name': self.display_name,
             # TODO (cpennington): This won't work when we move to data that isn't on the filesystem
             'data_dir': self.metadata['data_dir'],
+            'show_captions': self.show_captions
         })
 
 

common/lib/xmodule/xmodule/x_module.py

@@ -544,7 +544,13 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
             # Put import here to avoid circular import errors
             from xmodule.error_module import ErrorDescriptor
             msg = "Error loading from xml."
-            log.warning(msg + " " + str(err))
+            log.warning(msg + " " + str(err)[:200])
+
+            # Normally, we don't want lots of exception traces in our logs from common
+            # content problems.  But if you're debugging the xml loading code itself,
+            # uncomment the next line.
+            # log.exception(msg)
+
             system.error_tracker(msg)
             err_msg = msg + "\n" + exc_info_to_str(sys.exc_info())
             descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course,
@@ -717,7 +723,8 @@ class ModuleSystem(object):
                  filestore=None,
                  debug=False,
                  xqueue=None,
-                 node_path=""):
+                 node_path="",
+                 anonymous_student_id=''):
         '''
         Create a closure around the system environment.
 
@@ -742,11 +749,16 @@ class ModuleSystem(object):
                          at settings.DATA_DIR.
 
         xqueue - Dict containing XqueueInterface object, as well as parameters
-                    for the specific StudentModule
+                    for the specific StudentModule:
+                    xqueue = {'interface': XQueueInterface object,
+                              'callback_url': Callback into the LMS,
+                              'queue_name': Target queuename in Xqueue}
 
         replace_urls - TEMPORARY - A function like static_replace.replace_urls
                          that capa_module can use to fix up the static urls in
                          ajax results.
+
+        anonymous_student_id - Used for tracking modules with student id
         '''
         self.ajax_url = ajax_url
         self.xqueue = xqueue
@@ -758,6 +770,8 @@ class ModuleSystem(object):
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
         self.node_path = node_path
+        self.anonymous_student_id = anonymous_student_id
+        self.user_is_staff = user is not None and user.is_staff
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''

common/lib/xmodule/xmodule/xml_module.py

@@ -12,6 +12,15 @@ import sys
 
 log = logging.getLogger(__name__)
 
+edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
+                                 remove_comments=True, remove_blank_text=True)
+
+def name_to_pathname(name):
+    """
+    Convert a location name for use in a path: replace ':' with '/'.
+    This allows users of the xml format to organize content into directories
+    """
+    return name.replace(':', '/')
 
 def is_pointer_tag(xml_obj):
     """
@@ -90,10 +99,14 @@ class XmlDescriptor(XModuleDescriptor):
 
     # A dictionary mapping xml attribute names AttrMaps that describe how
     # to import and export them
+    # Allow json to specify either the string "true", or the bool True.  The string is preferred.
+    to_bool = lambda val: val == 'true' or val == True
+    from_bool = lambda val: str(val).lower()
+    bool_map = AttrMap(to_bool, from_bool)
     xml_attribute_map = {
         # type conversion: want True/False in python, "true"/"false" in xml
-        'graded': AttrMap(lambda val: val == 'true',
-                          lambda val: str(val).lower()),
+        'graded': bool_map,
+        'hide_progress_tab': bool_map,
     }
 
 
@@ -140,7 +153,7 @@ class XmlDescriptor(XModuleDescriptor):
 
         Returns an lxml Element
         """
-        return etree.parse(file_object).getroot()
+        return etree.parse(file_object, parser=edx_xml_parser).getroot()
 
     @classmethod
     def load_file(cls, filepath, fs, location):
@@ -226,6 +239,16 @@ class XmlDescriptor(XModuleDescriptor):
 
 
     @classmethod
+    def apply_policy(cls, metadata, policy):
+        """
+        Add the keys in policy to metadata, after processing them
+        through the attrmap.  Updates the metadata dict in place.
+        """
+        for attr in policy:
+            attr_map = cls.xml_attribute_map.get(attr, AttrMap())
+            metadata[attr] = attr_map.from_xml(policy[attr])
+
+    @classmethod
     def from_xml(cls, xml_data, system, org=None, course=None):
         """
         Creates an instance of this descriptor from the supplied xml_data.
@@ -245,8 +268,8 @@ class XmlDescriptor(XModuleDescriptor):
         # VS[compat] -- detect new-style each-in-a-file mode
         if is_pointer_tag(xml_object):
             # new style:
-            # read the actual definition file--named using url_name
-            filepath = cls._format_filepath(xml_object.tag, url_name)
+            # read the actual definition file--named using url_name.replace(':','/')
+            filepath = cls._format_filepath(xml_object.tag, name_to_pathname(url_name))
             definition_xml = cls.load_file(filepath, system.resources_fs, location)
         else:
             definition_xml = xml_object	# this is just a pointer, not the real definition content
@@ -273,7 +296,7 @@ class XmlDescriptor(XModuleDescriptor):
         # Set/override any metadata specified by policy
         k = policy_key(location)
         if k in system.policy:
-            metadata.update(system.policy[k])
+            cls.apply_policy(metadata, system.policy[k])
 
         return cls(
             system,
@@ -292,7 +315,8 @@ class XmlDescriptor(XModuleDescriptor):
         """If this returns True, write the definition of this descriptor to a separate
         file.
 
-        NOTE: Do not override this without a good reason.  It is here specifically for customtag...
+        NOTE: Do not override this without a good reason.  It is here
+        specifically for customtag...
         """
         return True
 
@@ -335,7 +359,8 @@ class XmlDescriptor(XModuleDescriptor):
 
         if self.export_to_file():
             # Write the definition to a file
-            filepath = self.__class__._format_filepath(self.category, self.url_name)
+            url_path = name_to_pathname(self.url_name)
+            filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
                 file.write(etree.tostring(xml_object, pretty_print=True))

common/test/data/full/chapter/Overview.xml

 <sequential>
-  <video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" slug="Welcome" format="Video" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Welcome"/>
+  <video url_name="welcome"/>
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
     <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>

common/test/data/full/video/welcome.xml

+<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>

common/test/data/toy/chapter/secret/magic.xml

+<chapter>
+    <video url_name="toyvideo" youtube="blahblah"/>
+</chapter>

common/test/data/toy/course/2012_Fall.xml

 <course>
   <chapter url_name="Overview">
     <videosequence url_name="Toy_Videos">
-      <html url_name="toylab"/>
+      <html url_name="secret:toylab"/>
       <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
     </videosequence>
     <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
   </chapter>
+  <chapter url_name="secret:magic"/>
 </course>

common/test/data/toy/policies/2012_Fall.json

@@ -2,7 +2,8 @@
     "course/2012_Fall": {
         "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
         "start": "2015-07-17T12:00",
-        "display_name": "Toy Course"
+        "display_name": "Toy Course",
+        "graded": "true"
     },
     "chapter/Overview": {
         "display_name": "Overview"
@@ -11,7 +12,7 @@
         "display_name": "Toy Videos",
         "format": "Lecture Sequence"
     },
-    "html/toylab": {
+    "html/secret:toylab": {
         "display_name": "Toy lab"
     },
     "video/Video_Resources": {

create-dev-env.sh

@@ -105,7 +105,7 @@ NUMPY_VER="1.6.2"
 SCIPY_VER="0.10.1"
 BREW_FILE="$BASE/mitx/brew-formulas.txt"
 LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
-APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript graphviz"
+APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript graphviz libgraphviz-dev"
 
 if [[ $EUID -eq 0 ]]; then
     error "This script should not be run using sudo or as the root user"

doc/development.md

@@ -72,3 +72,30 @@ Very handy: if you uncomment the `--pdb` argument in `NOSE_ARGS` in `lms/envs/te
 If you change course content, while running the LMS in dev mode, it is unnecessary to restart to refresh the modulestore.  
 
 Instead, hit /migrate/modules to see a list of all modules loaded, and click on links (eg /migrate/reload/edx4edx) to reload a course.
+
+### Gitreload-based workflow
+
+github (or other equivalent git-based repository systems) used for
+course content can be setup to trigger an automatic reload when changes are pushed.  Here is how:
+
+1. Each content directory in mitx_all/data should be a clone of a git repo
+
+2. The user running the mitx gunicorn process should have its ssh key registered with the git repo
+
+3. The list settings.ALLOWED_GITRELOAD_IPS should contain the IP address of the git repo originating the gitreload request.
+    By default, this list is ['207.97.227.253', '50.57.128.197', '108.171.174.178'] (the github IPs).
+    The list can be overridden in the startup file used, eg lms/envs/dev*.py
+
+4. The git post-receive-hook should POST to /gitreload with a JSON payload.  This payload should define at least
+
+   { "repository" : { "name" : reload_dir }
+
+    where reload_dir is the directory name of the content to reload (ie mitx_all/data/reload_dir should exist)
+
+    The mitx server will then do "git reset --hard HEAD; git clean -f -d; git pull origin" in that directory.  After the pull,
+    it will reload the modulestore for that course.
+
+Note that the gitreload-based workflow is not meant for deployments on AWS (or elsewhere) which use collectstatic, since collectstatic is not run by a gitreload event.
+
+Also, the gitreload feature needs MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True in the django settings.
+

doc/xml-format.md

@@ -148,7 +148,7 @@ That's basically all there is to the organizational structure.  Read the next se
 * `problem` -- a problem.  See elsewhere in edx4edx for documentation on the format.
 * `problemset` -- logically, a series of related problems.  Currently displayed vertically.  May contain explanatory html, videos, etc.
 * `sequential` -- a sequence of content, currently displayed with a horizontal list of tabs.  If possible, use a more semantically meaningful tag (currently, we only have `videosequence`).
-* `vertical` -- a sequence of content, displayed vertically.  If possible, use a more semantically meaningful tag (currently, we only have `problemset`).
+* `vertical` -- a sequence of content, displayed vertically.  Content will be accessed all at once, on the right part of the page. No navigational bar. May have to use browser scroll bars. Content split with separators.   If possible, use a more semantically meaningful tag (currently, we only have `problemset`).
 * `video`  -- a link to a video, currently expected to be hosted on youtube.
 * `videosequence` -- a sequence of videos.  This can contain various non-video content; it just signals to the system that this is logically part of an explanatory sequence of content, as opposed to say an exam sequence.
 
@@ -189,7 +189,13 @@ This video has been encoded at 4 different speeds: 0.75x, 1x, 1.25x, and 1.5x.
 
 ## More on `url_name`s
 
-Every content element (within a course) should have a unique id.  This id is formed as `{category}/{url_name}`, or automatically generated from the content if `url_name` is not specified.  Categories are the different tag types ('chapter', 'problem', 'html', 'sequential', etc).  Url_name is a string containing a-z, A-Z, dot (.) and _.  This is what appears in urls that point to this object.
+Every content element (within a course) should have a unique id.  This id is formed as `{category}/{url_name}`, or automatically generated from the content if `url_name` is not specified.  Categories are the different tag types ('chapter', 'problem', 'html', 'sequential', etc).  Url_name is a string containing a-z, A-Z, dot (.), underscore (_), and ':'.  This is what appears in urls that point to this object.
+
+Colon (':') is special--when looking for the content definition in an xml, ':' will be replaced with '/'.  This allows organizing content into folders.  For example, given the pointer tag
+
+    <problem url_name="conceptual:add_apples_and_oranges"/>
+
+we would look for the problem definition in `problem/conceptual/add_apples_and_oranges.xml`.   (There is a technical reason why we can't just allow '/' in the url_name directly.)
 
 __IMPORTANT__: A student's state for a particular content element is tied to the element id, so the automatic id generation if only ok for elements that do not need to store any student state (e.g. verticals or customtags).  For problems, sequentials, and videos, and any other element where we keep track of what the student has done and where they are at, you should specify a unique `url_name`.  Of course, any content element that is split out into a file will need a `url_name` to specify where to find the definition.  When the CMS comes online, it will use these ids to enable content reuse, so if there is a logical name for something, please do specify it.
 
@@ -217,18 +223,23 @@ Values are dictionaries of the form {"metadata-key" : "metadata-value"}.
 __Not inherited:__
 
 * `display_name` - name that will appear when this content is displayed in the courseware.  Useful for all tag types.
-*	`format` - subheading under display name -- currently only displayed for chapter sub-sections.
+*	`format` - subheading under display name -- currently only displayed for chapter sub-sections.  Also used by the the grader to know how to process students assessments that the
+    section contains. New formats can be defined as a 'type' in the GRADER variable in course_settings.json. Optional.  (TODO: double check this--what's the current behavior?)
 * `hide_from_toc` -- If set to true for a chapter or chapter subsection, will hide that element from the courseware navigation accordion.  This is useful if you'd like to link to the content directly instead (e.g. for tutorials)
 * `ispublic` -- specify whether the course is public.  You should be able to use start dates instead (?)
 
 __Inherited:__
 
 * `start` -- when this content should be shown to students.  Note that anyone with staff access to the course will always see everything.
-*	`showanswer` - only for psets, is binary (closed/open).
-*	`graded` - Tutorial vs. grade, again binary (true/false). If true, will be used in calculation of student grade.
-*	`rerandomise` - Provide different numbers/variables for problems to prevent cheating. Provide different answers from questions bank?
-*	`due` - Due date for assignment. Assignment will be closed after that. This is a very important function of a policy file.
-* `graceperiod` -
+*	`showanswer` - When to show answer. For 'attempted', will show answer after first attempt. Values: never, attempted, answered, closed. Default: closed. Optional.
+*	`graded` - Whether this section will count towards the students grade. "true" or "false". Defaults to "false".
+*	`rerandomise` - Randomize question on each attempt. Values: 'always' (students see a different version of the problem after each attempt to solve it)
+                                                            'never' (all students see the same version of the problem)
+                                                            'per_student' (individual students see the same version of the problem each time the look at it, but that version is different from what other students see)
+                                                            Default: 'always'. Optional.
+*	`due` - Due date for assignment. Assignment will be closed after that.  Values: valid date. Default: none. Optional.
+* attempts: Number of allowed attempts. Values: integer. Default: infinite. Optional.
+* `graceperiod` - A default length of time that the problem is still accessible after the due date in the format "2 days 3 hours" or "1 day 15 minutes".  Note, graceperiods are currently the easiest way to handle time zones. Due dates are all expressed in UCT.
 * `xqa_key` -- for integration with Ike's content QA server. -- should typically be specified at the course level.
 
 __Inheritance example:__
@@ -252,6 +263,7 @@ Metadata can also live in the xml files, but anything defined in the policy file
    - note, some xml attributes are not metadata.  e.g. in `<video youtube="xyz987293487293847"/>`, the `youtube` attribute specifies what video this is, and is logically part of the content, not the policy, so it should stay in the xml.
 
 Another example policy file:
+
     {
         "course/2012": {
             "graceperiod": "1 day",
@@ -303,3 +315,7 @@ before the week 1 material to make it easy to find in the file.
 * A heads up: our content management system will allow you to develop content through a web browser, but will be backed by this same xml at first.  Once that happens, every element will be in its own file to make access and updates faster.
 
 * Prefer the most "semantic" name for containers: e.g., use problemset rather than vertical for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
+
+----
+
+(Dev note: This file is generated from the mitx repo, in `doc/xml-format.md`.  Please make edits there.)

lms/djangoapps/branding/views.py

@@ -20,8 +20,8 @@ def index(request):
         return redirect(reverse('dashboard'))
 
     if settings.MITX_FEATURES.get('AUTH_USE_MIT_CERTIFICATES'):
-        from external_auth.views import edXauth_ssl_login
-        return edXauth_ssl_login(request)
+        from external_auth.views import ssl_login
+        return ssl_login(request)
 
     university = branding.get_university(request.META.get('HTTP_HOST'))
     if university is None:

lms/djangoapps/course_wiki/editors.py

@@ -4,9 +4,12 @@ from django.utils.encoding import force_unicode
 from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
+from django.template.loader import render_to_string
+
 from wiki.editors.base import BaseEditor
 from wiki.editors.markitup import MarkItUpAdminWidget
 
+
 class CodeMirrorWidget(forms.Widget):
     def __init__(self, attrs=None):
         # The 'rows' and 'cols' attributes are required for HTML correctness.
@@ -18,9 +21,15 @@ class CodeMirrorWidget(forms.Widget):
     
     def render(self, name, value, attrs=None):
         if value is None: value = ''
+
         final_attrs = self.build_attrs(attrs, name=name)
-        return mark_safe(u'<div><textarea%s>%s</textarea></div>' % (flatatt(final_attrs),
-                conditional_escape(force_unicode(value))))
+
+        # TODO use the help_text field of edit form instead of rendering a template
+
+        return render_to_string('wiki/includes/editor_widget.html',
+                                {'attrs': mark_safe(flatatt(final_attrs)),
+                                 'content': conditional_escape(force_unicode(value)),
+                                 })
 
 
 class CodeMirror(BaseEditor):
@@ -50,5 +59,6 @@ class CodeMirror(BaseEditor):
               "js/vendor/CodeMirror/xml.js",
               "js/vendor/CodeMirror/mitx_markdown.js",
               "js/wiki/CodeMirror.init.js",
+              "js/wiki/cheatsheet.js",
               )
 

lms/djangoapps/courseware/access.py

@@ -30,7 +30,7 @@ def has_access(user, obj, action):
     Things this module understands:
     - start dates for modules
     - DISABLE_START_DATES
-    - different access for staff, course staff, and students.
+    - different access for instructor, staff, course staff, and students.
 
     user: a Django user object. May be anonymous.
 
@@ -70,6 +70,20 @@ def has_access(user, obj, action):
     raise TypeError("Unknown object type in has_access(): '{0}'"
                     .format(type(obj)))
 
+def get_access_group_name(obj,action):
+    '''
+    Returns group name for user group which has "action" access to the given object.
+
+    Used in managing access lists.
+    '''
+
+    if isinstance(obj, CourseDescriptor):
+        return _get_access_group_name_course_desc(obj, action)
+
+    # Passing an unknown object here is a coding error, so rather than
+    # returning a default, complain.
+    raise TypeError("Unknown object type in get_access_group_name(): '{0}'"
+                    .format(type(obj)))
 
 # ================ Implementation helpers ================================
 
@@ -138,11 +152,19 @@ def _has_access_course_desc(user, course, action):
         'load': can_load,
         'enroll': can_enroll,
         'see_exists': see_exists,
-        'staff': lambda: _has_staff_access_to_descriptor(user, course)
+        'staff': lambda: _has_staff_access_to_descriptor(user, course),
+        'instructor': lambda: _has_instructor_access_to_descriptor(user, course),
         }
 
     return _dispatch(checkers, action, user, course)
 
+def _get_access_group_name_course_desc(course, action):
+    '''
+    Return name of group which gives staff access to course.  Only understands action = 'staff'
+    '''
+    if not action=='staff':
+        return []
+    return _course_staff_group_name(course.location)
 
 def _has_access_error_desc(user, descriptor, action):
     """
@@ -292,6 +314,17 @@ def _course_staff_group_name(location):
     """
     return 'staff_%s' % Location(location).course
 
+
+def _course_instructor_group_name(location):
+    """
+    Get the name of the instructor group for a location.  Right now, that's instructor_COURSE.
+    A course instructor has all staff privileges, but also can manage list of course staff (add, remove, list).
+
+    location: something that can passed to Location.
+    """
+    return 'instructor_%s' % Location(location).course
+
+
 def _has_global_staff_access(user):
     if user.is_staff:
         debug("Allow: user.is_staff")
@@ -301,17 +334,28 @@ def _has_global_staff_access(user):
         return False
 
 
+def _has_instructor_access_to_location(user, location):
+    return _has_access_to_location(user, location, 'instructor')
+
+
 def _has_staff_access_to_location(user, location):
+    return _has_access_to_location(user, location, 'staff')
+
+
+def _has_access_to_location(user, location, access_level):
     '''
-    Returns True if the given user has staff access to a location.  For now this
-    is equivalent to having staff access to the course location.course.
+    Returns True if the given user has access_level (= staff or
+    instructor) access to a location.  For now this is equivalent to
+    having staff / instructor access to the course location.course.
 
-    This means that user is in the staff_* group, or is an overall admin.
+    This means that user is in the staff_* group or instructor_* group, or is an overall admin.
 
     TODO (vshnayder): this needs to be changed to allow per-course_id permissions, not per-course
     (e.g. staff in 2012 is different from 2013, but maybe some people always have access)
 
     course is a string: the course field of the location being accessed.
+    location = location
+    access_level = string, either "staff" or "instructor"
     '''
     if user is None or (not user.is_authenticated()):
         debug("Deny: no user or anon user")
@@ -321,25 +365,47 @@ def _has_staff_access_to_location(user, location):
         return True
 
     # If not global staff, is the user in the Auth group for this class?
-    user_groups = [x[1] for x in user.groups.values_list()]
+    user_groups = [g.name for g in user.groups.all()]
+
+    if access_level == 'staff':
         staff_group = _course_staff_group_name(location)
         if staff_group in user_groups:
             debug("Allow: user in group %s", staff_group)
             return True
         debug("Deny: user not in group %s", staff_group)
+
+    if access_level == 'instructor' or access_level == 'staff': 	# instructors get staff privileges
+        instructor_group = _course_instructor_group_name(location)
+        if instructor_group in user_groups:
+            debug("Allow: user in group %s", instructor_group)
+            return True
+        debug("Deny: user not in group %s", instructor_group)
+
+    else:
+        log.debug("Error in access._has_access_to_location access_level=%s unknown" % access_level)
+
     return False
 
+
 def _has_staff_access_to_course_id(user, course_id):
     """Helper method that takes a course_id instead of a course name"""
     loc = CourseDescriptor.id_to_location(course_id)
     return _has_staff_access_to_location(user, loc)
 
 
+def _has_instructor_access_to_descriptor(user, descriptor):
+    """Helper method that checks whether the user has staff access to
+    the course of the location.
+
+    descriptor: something that has a location attribute
+    """
+    return _has_instructor_access_to_location(user, descriptor.location)
+
 def _has_staff_access_to_descriptor(user, descriptor):
     """Helper method that checks whether the user has staff access to
     the course of the location.
 
-    location: something that can be passed to Location
+    descriptor: something that has a location attribute
     """
     return _has_staff_access_to_location(user, descriptor.location)
 

lms/djangoapps/courseware/grades.py

@@ -17,13 +17,14 @@ log = logging.getLogger("mitx.courseware")
 
 def yield_module_descendents(module):
     stack = module.get_display_items()
+    stack.reverse()
 
     while len(stack) > 0:
         next_module = stack.pop()
         stack.extend( next_module.get_display_items() )
         yield next_module
 
-def grade(student, request, course, student_module_cache=None):
+def grade(student, request, course, student_module_cache=None, keep_raw_scores=False):
     """
     This grades a student as quickly as possible. It retuns the
     output from the course grader, augmented with the final letter
@@ -37,11 +38,13 @@ def grade(student, request, course, student_module_cache=None):
         up the grade. (For display)
     - grade_breakdown : A breakdown of the major components that
         make up the final grade. (For display)
+    - keep_raw_scores : if True, then value for key 'raw_scores' contains scores for every graded module
 
     More information on the format is in the docstring for CourseGrader.
     """
 
     grading_context = course.grading_context
+    raw_scores = []
 
     if student_module_cache == None:
         student_module_cache = StudentModuleCache(course.id, student, grading_context['all_descriptors'])
@@ -82,7 +85,7 @@ def grade(student, request, course, student_module_cache=None):
                     if correct is None and total is None:
                         continue
 
-                    if settings.GENERATE_PROFILE_SCORES:
+                    if settings.GENERATE_PROFILE_SCORES:	# for debugging!
                         if total > 1:
                             correct = random.randrange(max(total - 2, 1), total + 1)
                         else:
@@ -96,6 +99,8 @@ def grade(student, request, course, student_module_cache=None):
                     scores.append(Score(correct, total, graded, module.metadata.get('display_name')))
 
                 section_total, graded_total = graders.aggregate_scores(scores, section_name)
+                if keep_raw_scores:
+                    raw_scores += scores
             else:
                 section_total = Score(0.0, 1.0, False, section_name)
                 graded_total = Score(0.0, 1.0, True, section_name)
@@ -116,7 +121,10 @@ def grade(student, request, course, student_module_cache=None):
 
     letter_grade = grade_for_percentage(course.grade_cutoffs, grade_summary['percent'])
     grade_summary['grade'] = letter_grade
-
+    grade_summary['totaled_scores'] = totaled_scores	# make this available, eg for instructor download & debugging
+    if keep_raw_scores:
+        grade_summary['raw_scores'] = raw_scores        # way to get all RAW scores out to instructor
+                                                        # so grader can be double-checked
     return grade_summary
 
 def grade_for_percentage(grade_cutoffs, percentage):

lms/djangoapps/courseware/management/commands/clean_xml.py

@@ -57,7 +57,6 @@ def import_with_checks(course_dir, verbose=True):
     # module.
     modulestore = XMLModuleStore(data_dir,
                    default_class=None,
-                   eager=True,
                    course_dirs=course_dirs)
 
     def str_of_err(tpl):

lms/djangoapps/courseware/management/commands/metadata_to_json.py

@@ -23,7 +23,6 @@ def import_course(course_dir, verbose=True):
     # module.
     modulestore = XMLModuleStore(data_dir,
                    default_class=None,
-                   eager=True,
                    course_dirs=course_dirs)
 
     def str_of_err(tpl):

lms/djangoapps/courseware/module_render.py

+import hashlib
 import json
 import logging
 import sys
@@ -61,7 +62,7 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
 
     where SECTIONS is a list
     [ {'display_name': name, 'url_name': url_name,
-       'format': format, 'due': due, 'active' : bool}, ...]
+       'format': format, 'due': due, 'active' : bool, 'graded': bool}, ...]
 
     active is set for the section and chapter corresponding to the passed
     parameters, which are expected to be url_names of the chapter+section.
@@ -97,7 +98,9 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
                                  'url_name': section.url_name,
                                  'format': section.metadata.get('format', ''),
                                  'due': section.metadata.get('due', ''),
-                                 'active': active})
+                                 'active': active,
+                                 'graded': section.metadata.get('graded', False),
+                                 })
 
         chapters.append({'display_name': chapter.display_name,
                          'url_name': chapter.url_name,
@@ -144,8 +147,8 @@ def get_module(user, request, location, student_module_cache, course_id, positio
 
     Arguments:
       - user                  : User for whom we're getting the module
-      - request               : current django HTTPrequest -- used in particular for auth
-                                (This is important e.g. for prof impersonation of students in progress view)
+      - request               : current django HTTPrequest.  Note: request.user isn't used for anything--all auth
+                                and such works based on user.
       - location              : A Location-like object identifying the module to load
       - student_module_cache  : a StudentModuleCache
       - course_id             : the course_id in the context of which to load module
@@ -171,12 +174,16 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     descriptor = modulestore().get_instance(course_id, location)
 
     # Short circuit--if the user shouldn't have access, bail without doing any work
-    # NOTE: Do access check on request.user -- that's who actually needs access (e.g. could be prof
-    # impersonating a user)
-    if not has_access(request.user, descriptor, 'load'):
+    if not has_access(user, descriptor, 'load'):
         return None
 
-    #TODO Only check the cache if this module can possibly have state
+    # Anonymized student identifier
+    h = hashlib.md5()
+    h.update(settings.SECRET_KEY)
+    h.update(str(user.id))
+    anonymous_student_id = h.hexdigest()
+
+    # Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
     if user.is_authenticated():
@@ -190,7 +197,6 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                                                         descriptor.category,
                                                         shared_state_key)
 
-
     instance_state = instance_module.state if instance_module is not None else None
     shared_state = shared_module.state if shared_module is not None else None
 
@@ -200,6 +206,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                                    location=descriptor.location.url(),
                                    dispatch=''),
                        )
+    # Intended use is as {ajax_url}/{dispatch_command}, so get rid of the trailing slash.
+    ajax_url = ajax_url.rstrip('/')
 
     # Fully qualified callback URL for external queueing system
     xqueue_callback_url = '{proto}://{host}'.format(
@@ -220,7 +228,9 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
 
     xqueue = {'interface': xqueue_interface,
               'callback_url': xqueue_callback_url,
-              'default_queuename': xqueue_default_queuename.replace(' ', '_')}
+              'default_queuename': xqueue_default_queuename.replace(' ', '_'),
+              'waittime': settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+             }
 
     def inner_get_module(location):
         """
@@ -244,7 +254,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
-                          node_path=settings.NODE_PATH
+                          node_path=settings.NODE_PATH,
+                          anonymous_student_id=anonymous_student_id,
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
@@ -412,6 +423,10 @@ def modx_dispatch(request, dispatch, location, course_id):
     '''
     # ''' (fix emacs broken parsing)
 
+    # Check parameters and fail fast if there's a problem
+    if not Location.is_valid(location):
+        raise Http404("Invalid location")
+
     # Check for submitted files and basic file size checks
     p = request.POST.copy()
     if request.FILES:

lms/djangoapps/courseware/tests/tests.py

@@ -68,7 +68,6 @@ def xml_store_config(data_dir):
         'OPTIONS': {
             'data_dir': data_dir,
             'default_class': 'xmodule.hidden_module.HiddenDescriptor',
-            'eager': True,
         }
     }
 }
@@ -204,6 +203,7 @@ class PageLoader(ActivateLoginTestCase):
         self.assertEqual(len(courses), 1)
         course = courses[0]
         self.enroll(course)
+        course_id = course.id
         
         n = 0
         num_bad = 0
@@ -214,10 +214,11 @@ class PageLoader(ActivateLoginTestCase):
             print "Checking ", descriptor.location.url()
             #print descriptor.__class__, descriptor.location
             resp = self.client.get(reverse('jump_to',
-                                   kwargs={'location': descriptor.location.url()}))
+                                   kwargs={'course_id': course_id,
+                                           'location': descriptor.location.url()}))
             msg = str(resp.status_code)
 
-            if resp.status_code != 200:
+            if resp.status_code != 302:
                 msg = "ERROR " + msg
                 all_ok = False
                 num_bad += 1
@@ -411,8 +412,6 @@ class TestViewAuth(PageLoader):
             """list of urls that only instructors/staff should be able to see"""
             urls = reverse_urls(['instructor_dashboard','gradebook','grade_summary'],
                                 course)
-            urls.append(reverse('student_progress', kwargs={'course_id': course.id,
-                                                     'student_id': user(self.student).id}))
             return urls
 
         def check_non_staff(course):
@@ -435,6 +434,17 @@ class TestViewAuth(PageLoader):
                 print 'checking for 200 on {0}'.format(url)
                 self.check_for_get_code(200, url)
 
+            # The student progress tab is not accessible to a student
+            # before launch, so the instructor view-as-student feature should return a 404 as well.
+            # TODO (vshnayder): If this is not the behavior we want, will need
+            # to make access checking smarter and understand both the effective
+            # user (the student), and the requesting user (the prof)
+            url = reverse('student_progress', kwargs={'course_id': course.id,
+                                                     'student_id': user(self.student).id})
+            print 'checking for 404 on view-as-student: {0}'.format(url)
+            self.check_for_get_code(404, url)
+
+
         # First, try with an enrolled student
         print '=== Testing student access....'
         self.login(self.student, self.password)

lms/djangoapps/courseware/views.py

@@ -5,8 +5,6 @@ import itertools
 
 from functools import partial
 
-from functools import partial
-
 from django.conf import settings
 from django.core.context_processors import csrf
 from django.core.urlresolvers import reverse
@@ -152,7 +150,7 @@ def index(request, course_id, chapter=None, section=None,
                     course_id, request.user, section_descriptor)
                 module = get_module(request.user, request,
                                     section_descriptor.location,
-                                    student_module_cache, course_id)
+                                    student_module_cache, course_id, position)
                 if module is None:
                     # User is probably being clever and trying to access something
                     # they don't have access to.
@@ -196,7 +194,7 @@ def index(request, course_id, chapter=None, section=None,
 
 
 @ensure_csrf_cookie
-def jump_to(request, location):
+def jump_to(request, course_id, location):
     '''
     Show the page that contains a specific location.
 
@@ -213,15 +211,18 @@ def jump_to(request, location):
 
     # Complain if there's not data for this location
     try:
-        (course_id, chapter, section, position) = path_to_location(modulestore(), location)
+        (course_id, chapter, section, position) = path_to_location(modulestore(), course_id, location)
     except ItemNotFoundError:
         raise Http404("No data at this location: {0}".format(location))
     except NoPathToItem:
         raise Http404("This location is not in any class: {0}".format(location))
 
     # Rely on index to do all error handling and access control.
-    return index(request, course_id, chapter, section, position)
-
+    return redirect('courseware_position',
+                    course_id=course_id, 
+                    chapter=chapter, 
+                    section=section, 
+                    position=position)
 @ensure_csrf_cookie
 def course_info(request, course_id):
     """
@@ -263,7 +264,20 @@ def registered_for_course(course, user):
 def course_about(request, course_id):
     course = get_course_with_access(request.user, course_id, 'see_exists')
     registered = registered_for_course(course, request.user)
-    return render_to_response('portal/course_about.html', {'course': course, 'registered': registered})
+
+    if has_access(request.user, course, 'load'):
+        course_target = reverse('info', args=[course.id])
+    else:
+        course_target = reverse('about_course', args=[course.id])
+
+    show_courseware_link = (has_access(request.user, course, 'load') or
+                            settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'))
+
+    return render_to_response('portal/course_about.html',
+                              {'course': course,
+                               'registered': registered,
+                               'course_target': course_target,
+                               'show_courseware_link' : show_courseware_link})
 
 
 @ensure_csrf_cookie
@@ -328,11 +342,19 @@ def progress(request, course_id, student_id=None):
     # NOTE: To make sure impersonation by instructor works, use
     # student instead of request.user in the rest of the function.
 
+    # The pre-fetching of groups is done to make auth checks not require an 
+    # additional DB lookup (this kills the Progress page in particular).
+    student = User.objects.prefetch_related("groups").get(id=student.id)
+
     student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
         course_id, student, course)
     course_module = get_module(student, request, course.location,
                                student_module_cache, course_id)
 
+    # The course_module should be accessible, but check anyway just in case something went wrong:
+    if course_module is None:
+        raise Http404("Course does not exist")
+
     courseware_summary = grades.progress_summary(student, course_module,
                                                  course.grader, student_module_cache)
     grade_summary = grades.grade(student, request, course, student_module_cache)
@@ -348,96 +370,3 @@ def progress(request, course_id, student_id=None):
 
 
 
-# ======== Instructor views =============================================================================
-
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def gradebook(request, course_id):
-    """
-    Show the gradebook for this course:
-    - only displayed to course staff
-    - shows students who are enrolled.
-    """
-    course = get_course_with_access(request.user, course_id, 'staff')
-
-    enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
-
-    # TODO (vshnayder): implement pagination.
-    enrolled_students = enrolled_students[:1000]   # HACK!
-
-    student_info = [{'username': student.username,
-                     'id': student.id,
-                     'email': student.email,
-                     'grade_summary': grades.grade(student, request, course),
-                     'realname': UserProfile.objects.get(user=student).name
-                     }
-                     for student in enrolled_students]
-
-    return render_to_response('courseware/gradebook.html', {'students': student_info,
-                                                 'course': course,
-                                                 'course_id': course_id,
-                                                 # Checked above
-                                                 'staff_access': True,})
-
-
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def grade_summary(request, course_id):
-    """Display the grade summary for a course."""
-    course = get_course_with_access(request.user, course_id, 'staff')
-
-    # For now, just a static page
-    context = {'course': course,
-               'staff_access': True,}
-    return render_to_response('courseware/grade_summary.html', context)
-
-
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def instructor_dashboard(request, course_id):
-    """Display the instructor dashboard for a course."""
-    course = get_course_with_access(request.user, course_id, 'staff')
-
-    # For now, just a static page
-    context = {'course': course,
-               'staff_access': True,}
-
-    return render_to_response('courseware/instructor_dashboard.html', context)
-
-@ensure_csrf_cookie
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def enroll_students(request, course_id):
-    ''' Allows a staff member to enroll students in a course.
-
-    This is a short-term hack for Berkeley courses launching fall
-    2012. In the long term, we would like functionality like this, but
-    we would like both the instructor and the student to agree. Right
-    now, this allows any instructor to add students to their course,
-    which we do not want.
-
-    It is poorly written and poorly tested, but it's designed to be
-    stripped out.
-    '''
-
-    course = get_course_with_access(request.user, course_id, 'staff')
-    existing_students = [ce.user.email for ce in CourseEnrollment.objects.filter(course_id = course_id)]
-
-    if 'new_students' in request.POST:
-        new_students = request.POST['new_students'].split('\n')
-    else:
-        new_students = []
-    new_students = [s.strip() for s in new_students]
-
-    added_students = []
-    rejected_students = []
-
-    for student in new_students:
-        try:
-            nce = CourseEnrollment(user=User.objects.get(email = student), course_id = course_id)
-            nce.save()
-            added_students.append(student)
-        except:
-            rejected_students.append(student)
-
-    return render_to_response("enroll_students.html", {'course':course_id,
-                                                       'existing_students': existing_students,
-                                                       'added_students': added_students,
-                                                       'rejected_students': rejected_students,
-                                                       'debug':new_students})

lms/djangoapps/dashboard/views.py

@@ -21,11 +21,17 @@ def dashboard(request):
     if not request.user.is_staff:
         raise Http404
 
-    query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc"
+    queries=[]
+    queries.append("select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc;")
+    queries.append("select count(distinct user_id) as unique_students from student_courseenrollment;")
+    queries.append("select registrations, count(registrations) from (select count(user_id) as registrations from student_courseenrollment group by user_id) as registrations_per_user group by registrations;")
     
     from django.db import connection
     cursor = connection.cursor()
+    results =[]
+
+    for query in queries:
         cursor.execute(query)
-    results = dictfetchall(cursor)
+        results.append(dictfetchall(cursor))
 
     return HttpResponse(json.dumps(results, indent=4))

lms/djangoapps/django_comment_client/forum/views.py

 from django.contrib.auth.decorators import login_required
 from django.views.decorators.http import require_POST
-from django.http import HttpResponse
+from django.http import HttpResponse, Http404
 from django.utils import simplejson
 from django.core.context_processors import csrf
 from django.core.urlresolvers import reverse
@@ -13,14 +13,14 @@ from courseware.access import has_access
 from urllib import urlencode
 from operator import methodcaller
 from django_comment_client.permissions import check_permissions_by_view
-from django_comment_client.utils import merge_dict, extract, strip_none, strip_blank
+from django_comment_client.utils import merge_dict, extract, strip_none, strip_blank, get_courseware_context
 
 import json
 import django_comment_client.utils as utils
 import comment_client as cc
 
 
-THREADS_PER_PAGE = 5
+THREADS_PER_PAGE = 8
 PAGES_NEARBY_DELTA = 2
 
 
@@ -71,6 +71,16 @@ def render_discussion(request, course_id, threads, *args, **kwargs):
 
     annotated_content_info = reduce(merge_dict, map(infogetter, threads), {})
 
+    if discussion_type != 'inline':
+        course = get_course_with_access(request.user, course_id, 'load')
+
+        for thread in threads:
+            courseware_context = get_courseware_context(thread, course)
+            if courseware_context:
+                thread['courseware_location']  = courseware_context['courseware_location']
+                thread['courseware_title']  = courseware_context['courseware_title']
+
+
     context = {
         'threads': threads,
         'discussion_id': discussion_id,
@@ -98,11 +108,15 @@ def render_user_discussion(*args, **kwargs):
     return render_discussion(discussion_type='user', *args, **kwargs)
 
 def get_threads(request, course_id, discussion_id=None):
+    """
+    This may raise cc.utils.CommentClientError or
+    cc.utils.CommentClientUnknownError if something goes wrong.
+    """
 
     default_query_params = {
         'page': 1,
         'per_page': THREADS_PER_PAGE,
-        'sort_key': 'activity',
+        'sort_key': 'date',
         'sort_order': 'desc',
         'text': '',
         'tags': '',
@@ -110,6 +124,18 @@ def get_threads(request, course_id, discussion_id=None):
         'course_id': course_id,
     }
 
+    if not request.GET.get('sort_key'):
+        # If the user did not select a sort key, use their last used sort key
+        user = cc.User.from_django_user(request.user)
+        user.retrieve()
+        # TODO: After the comment service is updated this can just be user.default_sort_key because the service returns the default value
+        default_query_params['sort_key'] = user.get('default_sort_key') or default_query_params['sort_key']
+    else:
+        # If the user clicked a sort key, update their default sort key
+        user = cc.User.from_django_user(request.user)
+        user.default_sort_key = request.GET.get('sort_key')
+        user.save()
+
     query_params = merge_dict(default_query_params,
                               strip_none(extract(request.GET, ['page', 'sort_key', 'sort_order', 'text', 'tags'])))
 
@@ -122,7 +148,14 @@ def get_threads(request, course_id, discussion_id=None):
 
 # discussion per page is fixed for now
 def inline_discussion(request, course_id, discussion_id):
+    try:
         threads, query_params = get_threads(request, course_id, discussion_id)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        # TODO (vshnayder): since none of this code seems to be aware of the fact that
+        # sometimes things go wrong, I suspect that the js client is also not
+        # checking for errors on request.  Check and fix as needed.
+        raise Http404
+
     html = render_inline_discussion(request, course_id, threads, discussion_id=discussion_id,  \
                                                                  query_params=query_params)
 
@@ -143,6 +176,7 @@ def render_search_bar(request, course_id, discussion_id=None, text=''):
 
 def forum_form_discussion(request, course_id):
     course = get_course_with_access(request.user, course_id, 'load')
+    try:
         threads, query_params = get_threads(request, course_id)
         content = render_forum_discussion(request, course_id, threads, discussion_id=_general_discussion_id(course_id), query_params=query_params)
 
@@ -171,6 +205,9 @@ def forum_form_discussion(request, course_id):
             }
             # print "start rendering.."
             return render_to_response('discussion/index.html', context)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        raise Http404
+
 
 def render_single_thread(request, discussion_id, course_id, thread_id):
 
@@ -192,6 +229,7 @@ def render_single_thread(request, discussion_id, course_id, thread_id):
 
 def single_thread(request, course_id, discussion_id, thread_id):
 
+    try:
         if request.is_ajax():
 
             user_info = cc.User.from_django_user(request.user).to_dict()
@@ -231,10 +269,13 @@ def single_thread(request, course_id, discussion_id, thread_id):
             }
 
             return render_to_response('discussion/single_thread.html', context)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        raise Http404
 
 def user_profile(request, course_id, user_id):
 
     course = get_course_with_access(request.user, course_id, 'load')
+    try:
         profiled_user = cc.User(id=user_id, course_id=course_id)
 
         query_params = {
@@ -264,3 +305,5 @@ def user_profile(request, course_id, user_id):
             }
 
             return render_to_response('discussion/user_profile.html', context)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        raise Http404

lms/djangoapps/django_comment_client/utils.py

+from collections import defaultdict 
 from importlib import import_module
 from courseware.models import StudentModuleCache
 from courseware.module_render import get_module
@@ -18,11 +19,11 @@ import urllib
 import pystache_custom as pystache
 
 
+# TODO these should be cached via django's caching rather than in-memory globals
 _FULLMODULES = None
 _DISCUSSIONINFO = None
 
 
-
 def extract(dic, keys):
     return {k: dic.get(k) for k in keys}
 
@@ -40,80 +41,93 @@ def merge_dict(dic1, dic2):
 def get_full_modules():
     global _FULLMODULES
     if not _FULLMODULES:
-        class_path = settings.MODULESTORE['default']['ENGINE']
-        module_path, _, class_name = class_path.rpartition('.')
-        class_ = getattr(import_module(module_path), class_name)
-        modulestore = class_(**dict(settings.MODULESTORE['default']['OPTIONS'].items() + [('eager', True)]))
-        _FULLMODULES = modulestore.modules
+        _FULLMODULES = modulestore().modules
     return _FULLMODULES
 
-def get_categorized_discussion_info(request, course):
+def get_discussion_id_map(course):
     """
         return a dict of the form {category: modules}
     """
     global _DISCUSSIONINFO
     if not _DISCUSSIONINFO:
-        initialize_discussion_info(request, course)
-    return _DISCUSSIONINFO['categorized']
+        initialize_discussion_info(course)
+    return _DISCUSSIONINFO['id_map']
 
 def get_discussion_title(request, course, discussion_id):
     global _DISCUSSIONINFO
     if not _DISCUSSIONINFO:
-        initialize_discussion_info(request, course)
-    title = _DISCUSSIONINFO['by_id'].get(discussion_id, {}).get('title', '(no title)')
+        initialize_discussion_info(course)
+    title = _DISCUSSIONINFO['id_map'].get(discussion_id, {}).get('title', '(no title)')
     return title
 
-def initialize_discussion_info(request, course):
+def get_discussion_category_map(course):
 
     global _DISCUSSIONINFO
-    if _DISCUSSIONINFO:
-        return
-
-    course_id = course.id
-    _, course_name, _ = course_id.split('/')
-    user = request.user
-    url_course_id = course_id.replace('/', '_').replace('.', '_')
-
-    _is_course_discussion = lambda x: x[0].dict()['category'] == 'discussion' \
-                         and x[0].dict()['course'] == course_name
+    if not _DISCUSSIONINFO:
+        initialize_discussion_info(course)
+    return _DISCUSSIONINFO['category_map']
 
-    _get_module_descriptor = operator.itemgetter(1)
+def sort_map_entries(category_map):
+    things = []
+    for title, entry in category_map["entries"].items():
+        things.append((title, entry))
+    for title, category in category_map["subcategories"].items():
+        things.append((title, category))
+        sort_map_entries(category_map["subcategories"][title])
+    category_map["children"] = [x[0] for x in sorted(things, key=lambda x: x[1]["sort_key"])]
 
-    def _get_module(module_descriptor):
-        print module_descriptor
-        module = get_module(user, request, module_descriptor.location, student_module_cache)
-        return module
 
-    def _extract_info(module):
-        return {
-            'title': module.title,
-            'discussion_id': module.discussion_id,
-            'category': module.discussion_category,
-        }
+def initialize_discussion_info(course):
 
-    def _pack_with_id(info):
-        return (info['discussion_id'], info)
-
-    discussion_module_descriptors = map(_get_module_descriptor,
-                                        filter(_is_course_discussion,
-                                               get_full_modules().items()))
+    global _DISCUSSIONINFO
+    if _DISCUSSIONINFO:
+        return
 
-    student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(user, course)
+    course_id = course.id
+    url_course_id = course_id.replace('/', '_').replace('.', '_')
 
-    discussion_info = map(_extract_info, map(_get_module, discussion_module_descriptors))
+    all_modules = get_full_modules()[course_id]
+
+    discussion_id_map = {}
+    unexpanded_category_map = defaultdict(list)
+    for location, module in all_modules.items():
+        if location.category == 'discussion':
+            id = module.metadata['id']
+            category = module.metadata['discussion_category']
+            title = module.metadata['for']
+            sort_key = module.metadata.get('sort_key', title)
+            discussion_id_map[id] = {"location": location, "title": title}
+            category = " / ".join([x.strip() for x in category.split("/")])
+            unexpanded_category_map[category].append({"title": title, "id": id,
+                "sort_key": sort_key})
+
+    category_map = {"entries": defaultdict(dict), "subcategories": defaultdict(dict)} 
+    for category_path, entries in unexpanded_category_map.items():
+        node = category_map["subcategories"]
+        path = [x.strip() for x in category_path.split("/")]
+        for level in path[:-1]:
+            if level not in node:
+                node[level] = {"subcategories": defaultdict(dict), 
+                               "entries": defaultdict(dict),
+                               "sort_key": level} 
+            node = node[level]["subcategories"]
+
+        level = path[-1]
+        if level not in node:
+            node[level] = {"subcategories": defaultdict(dict), 
+                            "entries": defaultdict(dict), 
+                            "sort_key": level}
+        for entry in entries:
+            node[level]["entries"][entry["title"]] = {"id": entry["id"], 
+                                                      "sort_key": entry["sort_key"]}
+
+    sort_map_entries(category_map)
 
     _DISCUSSIONINFO = {}
 
-    _DISCUSSIONINFO['by_id'] = dict(map(_pack_with_id, discussion_info))
-
-    _DISCUSSIONINFO['categorized'] = dict((category, list(l)) \
-                for category, l in itertools.groupby(discussion_info, operator.itemgetter('category')))
+    _DISCUSSIONINFO['id_map'] = discussion_id_map
 
-    _DISCUSSIONINFO['categorized']['General'] = [{
-        'title': 'General discussion',
-        'discussion_id': url_course_id,
-        'category': 'General',
-    }]
+    _DISCUSSIONINFO['category_map'] = category_map
 
 class JsonResponse(HttpResponse):
     def __init__(self, data=None):
@@ -220,6 +234,17 @@ def extend_content(content):
     }
     return merge_dict(content, content_info)
 
+def get_courseware_context(content, course):
+    id_map = get_discussion_id_map(course)
+    id = content['commentable_id'] 
+    content_info = None
+    if id in id_map:
+        location = id_map[id]["location"].url()
+        title = id_map[id]["title"]
+        content_info = { "courseware_location": location, "courseware_title": title}
+    return content_info
+
+
 def safe_content(content):
     fields = [
         'id', 'title', 'body', 'course_id', 'anonymous', 'endorsed',

lms/djangoapps/instructor/tests.py

+"""
+Unit tests for instructor dashboard
+
+Based on (and depends on) unit tests for courseware.
+
+Notes for running by hand:
+
+django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/instructor
+"""
+
+import courseware.tests.tests as ct
+
+from nose import SkipTest
+from mock import patch, Mock
+from override_settings import override_settings
+
+# Need access to internal func to put users in the right group
+from courseware.access import _course_staff_group_name
+from django.contrib.auth.models import User, Group
+from django.conf import settings
+from django.core.urlresolvers import reverse
+
+import xmodule.modulestore.django
+
+from xmodule.modulestore.django import modulestore
+
+
+@override_settings(MODULESTORE=ct.TEST_DATA_XML_MODULESTORE)
+class TestInstructorDashboardGradeDownloadCSV(ct.PageLoader):
+    '''
+    Check for download of csv
+    '''
+
+    def setUp(self):
+        xmodule.modulestore.django._MODULESTORES = {}
+        courses = modulestore().get_courses()
+
+        def find_course(name):
+            """Assumes the course is present"""
+            return [c for c in courses if c.location.course==name][0]
+
+        self.full = find_course("full")
+        self.toy = find_course("toy")
+
+        # Create two accounts
+        self.student = 'view@test.com'
+        self.instructor = 'view2@test.com'
+        self.password = 'foo'
+        self.create_account('u1', self.student, self.password)
+        self.create_account('u2', self.instructor, self.password)
+        self.activate_user(self.student)
+        self.activate_user(self.instructor)
+
+        group_name = _course_staff_group_name(self.toy.location)
+        g = Group.objects.create(name=group_name)
+        g.user_set.add(ct.user(self.instructor))
+
+        self.logout()
+        self.login(self.instructor, self.password)
+        self.enroll(self.toy)
+
+
+    def test_download_grades_csv(self):
+        print "running test_download_grades_csv"
+        course = self.toy
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        msg = "url = %s\n" % url
+        response = self.client.post(url, {'action': 'Download CSV of all student grades for this course',
+                                          })
+        msg += "instructor dashboard download csv grades: response = '%s'\n" % response
+
+        self.assertEqual(response['Content-Type'],'text/csv',msg)
+
+        cdisp = response['Content-Disposition'].replace('TT_2012','2012')  # jenkins course_id is TT_2012_Fall instead of 2012_Fall?
+        msg += "cdisp = '%s'\n" % cdisp
+        self.assertEqual(cdisp,'attachment; filename=grades_edX/toy/2012_Fall.csv',msg)
+
+        body = response.content.replace('\r','')
+        msg += "body = '%s'\n" % body
+
+        expected_body = '''"ID","Username","Full Name","edX email","External email","HW 01","HW 02","HW 03","HW 04","HW 05","HW 06","HW 07","HW 08","HW 09","HW 10","HW 11","HW 12","HW Avg","Lab 01","Lab 02","Lab 03","Lab 04","Lab 05","Lab 06","Lab 07","Lab 08","Lab 09","Lab 10","Lab 11","Lab 12","Lab Avg","Midterm","Final"
+"2","u2","Fred Weasley","view2@test.com","","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0.0","0.0"
+'''
+        self.assertEqual(body, expected_body, msg)

lms/djangoapps/instructor/views.py

+# ======== Instructor views =============================================================================
+
+import csv
+import itertools
+import json
+import logging
+import os
+import urllib
+
+import track.views
+
+from functools import partial
+from collections import defaultdict
+
+from django.conf import settings
+from django.core.context_processors import csrf
+from django.core.urlresolvers import reverse
+from django.contrib.auth.models import User, Group
+from django.contrib.auth.decorators import login_required
+from django.http import Http404, HttpResponse
+from django.shortcuts import redirect
+from mitxmako.shortcuts import render_to_response, render_to_string
+#from django.views.decorators.csrf import ensure_csrf_cookie
+from django_future.csrf import ensure_csrf_cookie
+from django.views.decorators.cache import cache_control
+
+from courseware import grades
+from courseware.access import has_access, get_access_group_name
+from courseware.courses import (get_course_with_access, get_courses_by_university)
+from student.models import UserProfile
+
+from student.models import UserTestGroup, CourseEnrollment
+from util.cache import cache, cache_if_anonymous
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
+from xmodule.modulestore.search import path_to_location
+
+log = logging.getLogger("mitx.courseware")
+
+template_imports = {'urllib': urllib}
+
+
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def instructor_dashboard(request, course_id):
+    """Display the instructor dashboard for a course."""
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    instructor_access = has_access(request.user, course, 'instructor')		# an instructor can manage staff lists
+
+    msg = ''
+    # msg += ('POST=%s' % dict(request.POST)).replace('<','&lt;')
+
+    def escape(s):
+        """escape HTML special characters in string"""
+        return str(s).replace('<', '&lt;').replace('>', '&gt;')
+
+    # assemble some course statistics for output to instructor
+    datatable = {'header': ['Statistic', 'Value'],
+                 'title': 'Course Statistics At A Glance',
+                 }
+    data = [['# Enrolled', CourseEnrollment.objects.filter(course_id=course_id).count()]]
+    data += compute_course_stats(course).items()
+    if request.user.is_staff:
+        data.append(['metadata', escape(str(course.metadata))])
+    datatable['data'] = data
+
+    def return_csv(fn, datatable):
+        response = HttpResponse(mimetype='text/csv')
+        response['Content-Disposition'] = 'attachment; filename=%s' % fn
+        writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
+        writer.writerow(datatable['header'])
+        for datarow in datatable['data']:
+            writer.writerow(datarow)
+        return response
+
+    def get_staff_group(course):
+        staffgrp = get_access_group_name(course, 'staff')
+        try:
+            group = Group.objects.get(name=staffgrp)
+        except Group.DoesNotExist:
+            group = Group(name=staffgrp)		# create the group
+            group.save()
+        return group
+
+    # process actions from form POST
+    action = request.POST.get('action', '')
+
+    if 'GIT pull' in action:
+        data_dir = course.metadata['data_dir']
+        log.debug('git pull %s' % (data_dir))
+        gdir = settings.DATA_DIR / data_dir
+        if not os.path.exists(gdir):
+            msg += "====> ERROR in gitreload - no such directory %s" % gdir
+        else:
+            cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
+            msg += "git pull on %s:<p>" % data_dir
+            msg += "<pre>%s</pre></p>" % escape(os.popen(cmd).read())
+            track.views.server_track(request, 'git pull %s' % data_dir, {}, page='idashboard')
+            
+    if 'Reload course' in action:
+        log.debug('reloading %s (%s)' % (course_id, course))
+        try:
+            data_dir = course.metadata['data_dir']
+            modulestore().try_load_course(data_dir)
+            msg += "<br/><p>Course reloaded from %s</p>" % data_dir
+            track.views.server_track(request, 'reload %s' % data_dir, {}, page='idashboard')
+            course_errors = modulestore().get_item_errors(course.location)
+            msg += '<ul>'
+            for cmsg, cerr in course_errors:
+                msg += "<li>%s: <pre>%s</pre>" % (cmsg,escape(cerr))
+            msg += '</ul>'
+        except Exception as err:
+            msg += '<br/><p>Error: %s</p>' % escape(err)
+
+    elif action == 'Dump list of enrolled students':
+        log.debug(action)
+        datatable = get_student_grade_summary_data(request, course, course_id, get_grades=False)
+        datatable['title'] = 'List of students enrolled in %s' % course_id
+        track.views.server_track(request, 'list-students', {}, page='idashboard')
+
+    elif 'Dump Grades' in action:
+        log.debug(action)
+        datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True)
+        datatable['title'] = 'Summary Grades of students enrolled in %s' % course_id
+        track.views.server_track(request, 'dump-grades', {}, page='idashboard')
+
+    elif 'Dump all RAW grades' in action:
+        log.debug(action)
+        datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True,
+                                                   get_raw_scores=True)
+        datatable['title'] = 'Raw Grades of students enrolled in %s' % course_id
+        track.views.server_track(request, 'dump-grades-raw', {}, page='idashboard')
+
+    elif 'Download CSV of all student grades' in action:
+        track.views.server_track(request, 'dump-grades-csv', {}, page='idashboard')
+        return return_csv('grades_%s.csv' % course_id,
+                          get_student_grade_summary_data(request, course, course_id))
+
+    elif 'Download CSV of all RAW grades' in action:
+        track.views.server_track(request, 'dump-grades-csv-raw', {}, page='idashboard')
+        return return_csv('grades_%s_raw.csv' % course_id,
+                          get_student_grade_summary_data(request, course, course_id, get_raw_scores=True))
+
+    elif 'List course staff' in action:
+        group = get_staff_group(course)
+        msg += 'Staff group = %s' % group.name
+        log.debug('staffgrp=%s' % group.name)
+        uset = group.user_set.all()
+        datatable = {'header': ['Username', 'Full name']}
+        datatable['data'] = [[x.username, x.profile.name] for x in uset]
+        datatable['title'] = 'List of Staff in course %s' % course_id
+        track.views.server_track(request, 'list-staff', {}, page='idashboard')
+
+    elif action == 'Add course staff':
+        uname = request.POST['staffuser']
+        try:
+            user = User.objects.get(username=uname)
+        except User.DoesNotExist:
+            msg += '<font color="red">Error: unknown username "%s"</font>' % uname
+            user = None
+        if user is not None:
+            group = get_staff_group(course)
+            msg += '<font color="green">Added %s to staff group = %s</font>' % (user, group.name)
+            log.debug('staffgrp=%s' % group.name)
+            user.groups.add(group)
+            track.views.server_track(request, 'add-staff %s' % user, {}, page='idashboard')
+
+    elif action == 'Remove course staff':
+        uname = request.POST['staffuser']
+        try:
+            user = User.objects.get(username=uname)
+        except User.DoesNotExist:
+            msg += '<font color="red">Error: unknown username "%s"</font>' % uname
+            user = None
+        if user is not None:
+            group = get_staff_group(course)
+            msg += '<font color="green">Removed %s from staff group = %s</font>' % (user, group.name)
+            log.debug('staffgrp=%s' % group.name)
+            user.groups.remove(group)
+            track.views.server_track(request, 'remove-staff %s' % user, {}, page='idashboard')
+
+    # For now, mostly a static page
+    context = {'course': course,
+               'staff_access': True,
+               'admin_access': request.user.is_staff,
+               'instructor_access': instructor_access,
+               'datatable': datatable,
+               'msg': msg,
+               }
+
+    return render_to_response('courseware/instructor_dashboard.html', context)
+
+
+def get_student_grade_summary_data(request, course, course_id, get_grades=True, get_raw_scores=False):
+    '''
+    Return data arrays with student identity and grades for specified course.
+
+    course = CourseDescriptor
+    course_id = course ID
+
+    Note: both are passed in, only because instructor_dashboard already has them already.
+
+    returns datatable = dict(header=header, data=data)
+    where
+
+    header = list of strings labeling the data fields
+    data = list (one per student) of lists of data corresponding to the fields
+
+    If get_raw_scores=True, then instead of grade summaries, the raw grades for all graded modules are returned.
+
+    '''
+    enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
+
+    header = ['ID', 'Username', 'Full Name', 'edX email', 'External email']
+    if get_grades:
+        # just to construct the header
+        gradeset = grades.grade(enrolled_students[0], request, course, keep_raw_scores=get_raw_scores)
+        # log.debug('student %s gradeset %s' % (enrolled_students[0], gradeset))
+        if get_raw_scores:
+            header += [score.section for score in gradeset['raw_scores']]
+        else:
+            header += [x['label'] for x in gradeset['section_breakdown']]
+
+    datatable = {'header': header}
+    data = []
+
+    for student in enrolled_students:
+        datarow = [ student.id, student.username, student.profile.name, student.email ]
+        try:
+            datarow.append(student.externalauthmap.external_email)
+        except:	# ExternalAuthMap.DoesNotExist
+            datarow.append('')
+
+        if get_grades:
+            gradeset = grades.grade(student, request, course, keep_raw_scores=get_raw_scores)
+            # log.debug('student=%s, gradeset=%s' % (student,gradeset))
+            if get_raw_scores:
+                datarow += [score.earned for score in gradeset['raw_scores']]
+            else:
+                datarow += [x['percent'] for x in gradeset['section_breakdown']]
+
+        data.append(datarow)
+    datatable['data'] = data
+    return datatable
+
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def gradebook(request, course_id):
+    """
+    Show the gradebook for this course:
+    - only displayed to course staff
+    - shows students who are enrolled.
+    """
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
+
+    # TODO (vshnayder): implement pagination.
+    enrolled_students = enrolled_students[:1000]   # HACK!
+
+    student_info = [{'username': student.username,
+                     'id': student.id,
+                     'email': student.email,
+                     'grade_summary': grades.grade(student, request, course),
+                     'realname': student.profile.name,
+                     }
+                     for student in enrolled_students]
+
+    return render_to_response('courseware/gradebook.html', {'students': student_info,
+                                                 'course': course,
+                                                 'course_id': course_id,
+                                                 # Checked above
+                                                 'staff_access': True, })
+
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def grade_summary(request, course_id):
+    """Display the grade summary for a course."""
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    # For now, just a static page
+    context = {'course': course,
+               'staff_access': True, }
+    return render_to_response('courseware/grade_summary.html', context)
+
+
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def enroll_students(request, course_id):
+    ''' Allows a staff member to enroll students in a course.
+
+    This is a short-term hack for Berkeley courses launching fall
+    2012. In the long term, we would like functionality like this, but
+    we would like both the instructor and the student to agree. Right
+    now, this allows any instructor to add students to their course,
+    which we do not want.
+
+    It is poorly written and poorly tested, but it's designed to be
+    stripped out.
+    '''
+
+    course = get_course_with_access(request.user, course_id, 'staff')
+    existing_students = [ce.user.email for ce in CourseEnrollment.objects.filter(course_id=course_id)]
+
+    if 'new_students' in request.POST:
+        new_students = request.POST['new_students'].split('\n')
+    else:
+        new_students = []
+    new_students = [s.strip() for s in new_students]
+
+    added_students = []
+    rejected_students = []
+
+    for student in new_students:
+        try:
+            nce = CourseEnrollment(user=User.objects.get(email=student), course_id=course_id)
+            nce.save()
+            added_students.append(student)
+        except:
+            rejected_students.append(student)
+
+    return render_to_response("enroll_students.html", {'course': course_id,
+                                                       'existing_students': existing_students,
+                                                       'added_students': added_students,
+                                                       'rejected_students': rejected_students,
+                                                       'debug': new_students})
+
+#-----------------------------------------------------------------------------
+
+
+def compute_course_stats(course):
+    '''
+    Compute course statistics, including number of problems, videos, html.
+
+    course is a CourseDescriptor from the xmodule system.
+    '''
+
+    # walk the course by using get_children() until we come to the leaves; count the
+    # number of different leaf types
+
+    counts = defaultdict(int)
+
+    def walk(module):
+        children = module.get_children()
+        category = module.__class__.__name__ 	# HtmlDescriptor, CapaDescriptor, ...
+        counts[category] += 1
+        for c in children:
+            walk(c)
+
+    walk(course)
+    stats = dict(counts)	# number of each kind of module
+    return stats

lms/djangoapps/lms_migration/management/commands/create_user.py

@@ -67,7 +67,10 @@ class Command(BaseCommand):
             password = GenPasswd(12)
             
             # get name from kerberos
+            try:
                 kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+            except:
+                kname = ''
             name = raw_input('Full name: [%s] ' % kname).strip()
             if name=='':
                 name = kname

lms/djangoapps/lms_migration/management/commands/manage_course_groups.py

+#!/usr/bin/python
+#
+# File:   manage_course_groups
+#
+# interactively list and edit membership in course staff and instructor groups
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+
+#-----------------------------------------------------------------------------
+# get all staff groups
+
+class Command(BaseCommand):
+    help = "Manage course group membership, interactively."
+
+    def handle(self, *args, **options):
+
+        gset = Group.objects.all()
+        
+        print "Groups:"
+        for cnt,g in zip(range(len(gset)), gset):
+            print "%d. %s" % (cnt,g)
+        
+        gnum = int(raw_input('Choose group to manage (enter #): '))
+        
+        group = gset[gnum]
+        
+        #-----------------------------------------------------------------------------
+        # users in group
+        
+        uall = User.objects.all()
+        if uall.count()<50:
+            print "----"
+            print "List of All Users: %s" % [str(x.username) for x in uall]
+            print "----"
+        else:
+            print "----"
+            print "There are %d users, which is too many to list" % uall.count()
+            print "----"
+        
+        while True:
+        
+            print "Users in the group:"
+            
+            uset = group.user_set.all()
+            for cnt, u in zip(range(len(uset)), uset):
+                print "%d. %s" % (cnt, u)
+        
+            action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ')
+        
+            m = re.match('^[0-9]+$',action)
+            if m:
+                unum = int(action)
+                u = uset[unum]
+                print "Deleting user %s" % u
+                u.groups.remove(group)
+            
+            else:
+                for uname in action.split(','):
+                    try:
+                        user = User.objects.get(username=action)
+                    except Exception as err:
+                        print "Error %s" % err
+                        continue
+                    print "adding %s to group %s" % (user, group)
+                    user.groups.add(group)
+                
+            
+            

lms/djangoapps/lms_migration/migrate.py

@@ -139,7 +139,7 @@ def gitreload(request, reload_dir=None):
 
     ALLOWED_IPS = []	# allow none by default
     if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
-        ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
+        ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS
 
     if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
         if request.user and request.user.is_staff:

lms/envs/common.py

@@ -64,6 +64,9 @@ MITX_FEATURES = {
     # university to use for branding purposes
     'SUBDOMAIN_BRANDING': False,
 
+    'FORCE_UNIVERSITY_DOMAIN': False,	# set this to the university domain to use, as an override to HTTP_HOST
+                                        # set to None to do no university selection
+
     'ENABLE_TEXTBOOK' : True,
     'ENABLE_DISCUSSION' : False,
     'ENABLE_DISCUSSION_SERVICE': True,
@@ -77,7 +80,7 @@ MITX_FEATURES = {
     'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
     'AUTH_USE_OPENID': False,
     'AUTH_USE_MIT_CERTIFICATES' : False,
-
+    'AUTH_USE_OPENID_PROVIDER': False,
 }
 
 # Used for A/B testing
@@ -86,6 +89,9 @@ DEFAULT_GROUPS = []
 # If this is true, random scores will be generated for the purpose of debugging the profile graphs
 GENERATE_PROFILE_SCORES = False
 
+# Used with XQueue
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
+
 ############################# SET PATH INFORMATION #############################
 PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /mitx/lms
 REPO_ROOT = PROJECT_ROOT.dirname()
@@ -117,6 +123,10 @@ node_paths = [COMMON_ROOT / "static/js/vendor",
               ]
 NODE_PATH = ':'.join(node_paths)
 
+
+############################ OpenID Provider  ##################################
+OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net'] 
+
 ################################## MITXWEB #####################################
 # This is where we stick our compiled template files. Most of the app uses Mako
 # templates
@@ -216,7 +226,6 @@ MODULESTORE = {
         'OPTIONS': {
             'data_dir': DATA_DIR,
             'default_class': 'xmodule.hidden_module.HiddenDescriptor',
-            'eager': True,
         }
     }
 }
@@ -426,7 +435,7 @@ main_vendor_js = [
   'js/vendor/jquery.qtip.min.js',
 ]
 
-discussion_js = glob2.glob(PROJECT_ROOT / 'static/coffee/src/discussion/*.coffee')
+discussion_js = sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/discussion/*.coffee'))
 
 # Load javascript from all of the available xmodules, and
 # prep it for use in pipeline js
@@ -494,10 +503,10 @@ PIPELINE_JS = {
         'source_filenames': [
             pth.replace(COMMON_ROOT / 'static/', '')
             for pth
-            in glob2.glob(COMMON_ROOT / 'static/coffee/src/**/*.coffee')
+            in sorted(glob2.glob(COMMON_ROOT / 'static/coffee/src/**/*.coffee'))
         ] + [
             pth.replace(PROJECT_ROOT / 'static/', '')
-            for pth in glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee')\
+            for pth in sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee'))\
             if pth not in courseware_only_js and pth not in discussion_js
         ] + [
             'js/form.ext.js',
@@ -598,6 +607,7 @@ INSTALLED_APPS = (
     'track',
     'util',
     'certificates',
+    'instructor',
     
     #For the wiki
     'wiki', # The new django-wiki from benjaoming

lms/envs/dev.py

@@ -15,8 +15,9 @@ TEMPLATE_DEBUG = True
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
-MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = True
+MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
+MITX_FEATURES['FORCE_UNIVERSITY_DOMAIN'] = None		# show all university courses if in dev (ie don't use HTTP_HOST)
 
 WIKI_ENABLED = True
 
@@ -78,10 +79,10 @@ COURSE_LISTINGS = {
                 'MITx/3.091x/2012_Fall',
                 'MITx/6.002x/2012_Fall',
                 'MITx/6.00x/2012_Fall'],
-    'berkeley': ['BerkeleyX/CS169.1x/Cal_2012_Fall',
-                 'BerkeleyX/CS188.1x/Cal_2012_Fall'],
+    'berkeley': ['BerkeleyX/CS169/fa12',
+                 'BerkeleyX/CS188/fa12'],
     'harvard': ['HarvardX/CS50x/2012H'],
-    'mit': [],
+    'mit': ['MITx/3.091/MIT_2012_Fall'],
     'sjsu': ['MITx/6.002x-EE98/2012_Fall_SJSU'],
 }
 
@@ -105,6 +106,7 @@ LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 
 ################################ OpenID Auth #################################
 MITX_FEATURES['AUTH_USE_OPENID'] = True
+MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
 
 INSTALLED_APPS += ('external_auth',)
@@ -115,6 +117,8 @@ OPENID_UPDATE_DETAILS_FROM_SREG = True
 OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'	# TODO: accept more endpoints
 OPENID_USE_AS_ADMIN_LOGIN = False
 
+OPENID_PROVIDER_TRUSTED_ROOTS = ['*']
+
 ################################ MIT Certificates SSL Auth #################################
 MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
 

lms/envs/dev_ike.py

@@ -18,6 +18,7 @@ MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = False
+MITX_FEATURES['FORCE_UNIVERSITY_DOMAIN'] = None		# show all university courses if in dev (ie don't use HTTP_HOST)
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
 # MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
@@ -28,6 +29,9 @@ if ('edxvm' in myhost) or ('ocw' in myhost):
     MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa'	# needs to be ssl or browser blocks it
     MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
 
+if ('ocw' in myhost):
+    MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False
+
 if ('domU' in myhost):
     EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
     MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu'	# nonempty string = address for all activation emails

lms/envs/test.py

@@ -58,7 +58,7 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
-
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # TODO (cpennington): We need to figure out how envs/test.py can inject things
 # into common.py so that we don't have to repeat this sort of thing
@@ -123,6 +123,11 @@ CACHES = {
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 
+################################## OPENID ######################################
+MITX_FEATURES['AUTH_USE_OPENID'] = True
+MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
+OPENID_PROVIDER_TRUSTED_ROOTS = ['*']
+
 ############################ FILE UPLOADS (ASKBOT) #############################
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 MEDIA_ROOT = TEST_ROOT / "uploads"

lms/lib/comment_client/user.py

@@ -8,10 +8,10 @@ class User(models.Model):
     accessible_fields = ['username', 'email', 'follower_ids', 'upvoted_ids', 'downvoted_ids',
                          'id', 'external_id', 'subscribed_user_ids', 'children', 'course_id',
                          'subscribed_thread_ids', 'subscribed_commentable_ids',
-                         'threads_count', 'comments_count',
+                         'threads_count', 'comments_count', 'default_sort_key'
                         ]
 
-    updatable_fields = ['username', 'external_id', 'email']
+    updatable_fields = ['username', 'external_id', 'email', 'default_sort_key']
     initializable_fields = updatable_fields
 
     base_url = "{prefix}/users".format(prefix=settings.PREFIX)

lms/lib/comment_client/utils.py

-import requests
 import json
+import logging
+import requests
 import settings
 
+log = logging.getLogger('mitx.' + __name__)
+
 def strip_none(dic):
     return dict([(k, v) for k, v in dic.iteritems() if v is not None])
 
@@ -23,10 +26,17 @@ def perform_request(method, url, data_or_params=None, *args, **kwargs):
     if data_or_params is None:
         data_or_params = {}
     data_or_params['api_key'] = settings.API_KEY
+    try:
         if method in ['post', 'put', 'patch']:
             response = requests.request(method, url, data=data_or_params)
         else:
             response = requests.request(method, url, params=data_or_params)
+    except Exception as err:
+        log.exception("Trying to call {method} on {url} with params {params}".format(
+            method=method, url=url, params=data_or_params))
+        # Reraise with a single exception type 
+        raise CommentClientError(str(err))
+
     if 200 < response.status_code < 500:
         raise CommentClientError(response.text)
     elif response.status_code == 500:

lms/static/coffee/src/discussion/discussion.coffee

@@ -169,7 +169,7 @@ if Backbone?
       url = URI($elem.attr("action")).addSearch({text: @$(".search-input").val()})
       @reload($elem, url)
 
-    sort: ->
+    sort: (event) ->
       $elem = $(event.target)
       url = $elem.attr("sort-url")
       @reload($elem, url)

lms/static/coffee/src/discussion/main.coffee

@@ -3,9 +3,6 @@ $ ->
   window.$$contents = {}
   window.$$discussions = {}
 
-  $(".discussion-module").each (index, elem) ->
-    view = new DiscussionModuleView(el: elem)
-
   $("section.discussion").each (index, elem) ->
     discussionData = DiscussionUtil.getDiscussionData($(elem).attr("_id"))
     discussion = new Discussion()

lms/static/js/wiki/cheatsheet.js

+$(document).ready(function () {
+    $('#cheatsheetLink').click(function() {
+        $('#cheatsheetModal').modal('show');
+    });
+    
+    $('#cheatsheetModal .close-btn').click(function(e) {
+        $('#cheatsheetModal').modal('hide');
+    });
+});
\ No newline at end of file

lms/static/sass/_discussion.scss

@@ -342,6 +342,11 @@ $tag-text-color: #5b614f;
       display: inline-block;
     }
 
+    .context{
+        margin-top: 1em;
+        font-size: $comment-font-size;
+    }
+
     .info {
       @include discussion-font;
       color: gray;
@@ -501,7 +506,8 @@ $tag-text-color: #5b614f;
       font-size: inherit;
       font-weight: bold;
       margin-left: 1%;
-      padding-top: 9px;
+      padding-top: 4px;
+      padding-bottom: 2px;
       text-decoration: none;
     }
 

lms/static/sass/course/courseware/_sidebar.scss

@@ -180,6 +180,18 @@ section.course-index {
             font-weight: normal;
           }
         }
+
+        &.graded {
+         > a {
+           background-image: url('../images/graded.png');
+           background-repeat: no-repeat;
+           background-position: 97% center;
+          }
+
+          &.active > a {
+            @include background-image(url('../images/graded.png'), linear-gradient(top, #e6e6e6, #d6d6d6));
+          }
+        }
       }
     }
   }

lms/static/sass/course/courseware/courses/_cs188.scss

@@ -27,6 +27,12 @@ body.cs188 {
             margin-bottom: 1.416em;
         }
 
+        .choicegroup {
+            input[type=checkbox], input[type=radio] {
+                margin-right: 5px;
+            }
+        }
+
     }
 
 }

lms/static/sass/course/wiki/_wiki.scss

@@ -391,6 +391,18 @@ section.wiki {
     line-height: 1.4em;
   }
 
+  #div_id_content {
+    position: relative;
+  }
+
+  #hint_id_content {
+    position: absolute;
+    top: 10px;
+    right: 0%;
+    font-size: 12px;
+    text-align:right;
+  }
+
   .CodeMirror {
     background: #fafafa;
     border: 1px solid #c8c8c8;
@@ -567,11 +579,73 @@ section.wiki {
     background: #f00 !important;
   }
 
+  #cheatsheetLink {
+    text-align: right;
+    display: float;
+  }
+
+  #cheatsheetModal {
+    width: 950px;
+    margin-left: -450px;
+    margin-top: -100px;
+
+    .left-column {
+      margin-right: 10px;
+    }
+
+    .left-column,
+    .right-column {
+      float: left;
+      width: 450px;
+    }
+
+    .close-btn {
+      display: block;
+      position: absolute;
+      top: -8px;
+      right: -8px;
+      width: 30px;
+      height: 30px;
+      border-radius: 30px;
+      border: 1px solid #ccc;
+      @include linear-gradient(top, #eee, #d2d2d2);
+      font-size: 22px;
+      line-height: 28px;    
+      color: #333;
+      text-align: center;
+      @include box-shadow(0 1px 0 #fff inset, 0 1px 2px rgba(0, 0, 0, .2));
+    }
+  }
 
+  #cheatsheet-body {
+    background: #fff;
+    text-align: left;
+    padding: 20px;
+    font-size: 14px;
+    @include clearfix;
 
+    h3 {
+      font-weight: bold;
+    }
 
+    ul {
+      list-style: circle;
+      line-height: 1.4;
+      color: #333;
+    }
+  }
 
+  #cheatsheet-body section + section {
+    margin-top: 40px;
+  }
 
+  #cheatsheet-body pre{
+    color: #000;
+    text-align: left;
+    background: #eee;
+    padding: 10px;
+    font-size: 12px;
+  }
 
   /*-----------------
 

lms/static/sass/multicourse/_course_about.scss

@@ -115,6 +115,30 @@
             }
           }
 
+          a {
+            &:hover, &:visited {
+              text-decoration: none;
+            }
+          }
+
+          strong {
+            @include button(shiny, $blue);
+            @include box-sizing(border-box);
+            @include border-radius(3px);
+            display: block;
+            float: left;
+            font: normal 1.2rem/1.6rem $sans-serif;
+            letter-spacing: 1px;
+            padding: 10px 0px;
+            text-transform: uppercase;
+            text-align: center;
+            width: flex-grid(3, 8);
+
+            &:hover {
+              color: rgb(255,255,255);
+            }
+          }
+
           span.register {
             background: lighten($blue, 20%);
             border: 1px solid $blue;
@@ -125,7 +149,10 @@
             padding: 10px 0px 8px;
             text-transform: uppercase;
             text-align: center;
-            width: flex-grid(12);
+            float: left;
+            margin: 1px flex-gutter(8) 0 0;
+            @include transition();
+            width: flex-grid(5, 8);
           }
         }
       }

lms/static/sass/multicourse/_dashboard.scss

@@ -261,7 +261,7 @@
           padding: 12px 0px;
           width: 100%;
 
-          a.university {
+          .university {
             background: rgba(255,255,255, 1);
             border: 1px solid rgb(180,180,180);
             @include border-radius(3px);
@@ -269,17 +269,14 @@
             color: $lighter-base-font-color;
             display: block;
             font-style: italic;
+            font-family: $sans-serif;
+            font-size: 16px;
             font-weight: 800;
             @include inline-block;
             margin-right: 10px;
+            margin-bottom: 0;
             padding: 5px 10px;
-
             float: left;
-
-            &:hover {
-              color: $blue;
-              text-decoration: none;
-            }
           }
 
           h3 {
@@ -306,8 +303,12 @@
           background: $yellow;
           border: 1px solid rgb(200,200,200);
           @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
-          margin-top: 16px;
+          margin-top: 17px;
+          margin-right: flex-gutter();
           padding: 5px;
+          width: flex-grid(8);
+          float: left;
+          @include box-sizing(border-box);
 
           p {
             color: $lighter-base-font-color;
@@ -317,82 +318,23 @@
           }
         }
 
-        .meta {
-          @include clearfix;
-          margin-top: 22px;
-          position: relative;
-          @include transition(opacity, 0.15s, linear);
-          width: 100%;
-
-
-          .course-work-icon {
-            @include background-image(url('../images/portal-icons/pencil-icon.png'));
-            background-size: cover;
-            float: left;
-            height: 22px;
-            opacity: 0.7;
-            width: 22px;
-          }
-
-          .complete {
-            float: right;
-
-            p {
-              color: $lighter-base-font-color;
-              font-style: italic;
-              @include inline-block;
-              text-align: right;
-              text-shadow: 0 1px rgba(255,255,255, 0.6);
-
-              .completeness {
-                color: $base-font-color;
-                font-weight: 700;
-                margin-right: 5px;
-              }
-            }
-          }
-
-          .progress {
-            @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
-            left: 35px;
-            position: absolute;
-            right: 130px;
-
-            .meter {
-              background: rgb(245,245,245);
-              border: 1px solid rgb(160,160,160);
-              @include box-shadow(inset 0 0 3px 0 rgba(0,0,0, 0.15));
-              @include box-sizing(border-box);
-              @include border-radius(4px);
-              height: 22px;
-              margin: 0 auto;
-              padding: 2px;
-              width: 100%;
-
-              .meter-fill {
-                background: $blue;
-                @include background-image(linear-gradient(-45deg, rgba(255,255,255, 0.15) 25%,
-                                                                  transparent 25%,
-                                                                  transparent 50%,
-                                                                  rgba(255,255,255, 0.15) 50%,
-                                                                  rgba(255,255,255, 0.15) 75%,
-                                                                  transparent 75%));
-                background-size: 40px 40px;
-                background-repeat: repeat-x;
-                border: 1px solid rgb(115,115,115);
-                @include border-radius(4px);
+        .enter-course {
+          @include button(shiny, $blue);
           @include box-sizing(border-box);
-                content: "";
+          @include border-radius(3px);
           display: block;
-                height: 100%;
-                width: 60%;
-              }
-            }
-          }
+          float: left;
+          font: normal 1rem/1.6rem $sans-serif;
+          letter-spacing: 1px;
+          padding: 6px 0px;
+          text-transform: uppercase;
+          text-align: center;
+          margin-top: 16px;
+          width: flex-grid(4);
         }
       }
 
-      &:hover {
+      > a:hover {
         .cover {
           .shade {
             background: rgba(255,255,255, 0.1);
@@ -404,6 +346,18 @@
             opacity: 1;
           }
         }
+
+        .info {
+          background: darken(rgb(250,250,250), 5%);
+          @include background-image(linear-gradient(-90deg, darken(rgb(253,253,253), 3%), darken(rgb(240,240,240), 5%)));
+          border-color: darken(rgb(190,190,190), 10%);
+
+          .course-status {
+            background: darken($yellow, 3%);
+            border-color: darken(rgb(200,200,200), 3%);
+            @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
+          }
+        }
       }
     }
 
@@ -420,5 +374,6 @@
         color: #333;
       }
     }
+
   }
 }

lms/templates/accordion.html

@@ -7,7 +7,7 @@
 
     <ul>
       % for section in chapter['sections']:
-          <li${' class="active"' if 'active' in section and section['active'] else ''}>
+          <li class="${'active' if 'active' in section and section['active'] else ''} ${'graded'  if 'graded' in section and section['graded'] else ''}">
             <a href="${reverse('courseware_section', args=[course_id, chapter['url_name'], section['url_name']])}">
               <p>${section['display_name']}
               <span class="subtitle">

lms/templates/courseware/course_navigation.html

@@ -19,34 +19,41 @@ def url_class(url):
     <ol class="course-tabs">
       <li class="courseware"><a href="${reverse('courseware', args=[course.id])}" class="${url_class('courseware')}">Courseware</a></li>
       <li class="info"><a href="${reverse('info', args=[course.id])}" class="${url_class('info')}">Course Info</a></li>
-% if hasattr(course,'syllabus_present') and course.syllabus_present:
+    % if hasattr(course,'syllabus_present') and course.syllabus_present:
       <li class="syllabus"><a href="${reverse('syllabus', args=[course.id])}" class="${url_class('syllabus')}">Syllabus</a></li>
-% endif
-% if user.is_authenticated():
+    % endif
+    % if user.is_authenticated():
       % if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
         % for index, textbook in enumerate(course.textbooks):
             <li class="book"><a href="${reverse('book', args=[course.id, index])}" class="${url_class('book')}">${textbook.title}</a></li>
         % endfor
       % endif
-  % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
+
+      ## If they have a discussion link specified, use that even if we feature
+      ## flag discussions off. Disabling that is mostly a server safety feature
+      ## at this point, and we don't need to worry about external sites.
+      % if course.discussion_link:
+        <li class="discussion"><a href="${course.discussion_link}">Discussion</a></li>
+      % elif settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
         <li class="discussion"><a href="${reverse('django_comment_client.forum.views.forum_form_discussion', args=[course.id])}" class="${url_class('discussion')}">Discussion</a></li>
         ## <li class="news"><a href="${reverse('news', args=[course.id])}" class="${url_class('news')}">News</a></li>
       % endif
 
+      ## This is Askbot, which we should be retiring soon...
       % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
         <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
       % endif
-% endif
-% if settings.WIKI_ENABLED:
+    % endif
+
+    % if settings.WIKI_ENABLED:
       <li class="wiki"><a href="${reverse('course_wiki', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
-% endif
-% if user.is_authenticated():
+    % endif
+    % if user.is_authenticated() and not course.hide_progress_tab:
       <li class="profile"><a href="${reverse('progress', args=[course.id])}" class="${url_class('progress')}">Progress</a></li>
-% endif
-% if staff_access:
+    % endif
+    % if staff_access:
       <li class="instructor"><a href="${reverse('instructor_dashboard', args=[course.id])}" class="${url_class('instructor')}">Instructor</a></li>
-% endif
-
+    % endif
     </ol>
   </div>
 </nav>

lms/templates/courseware/instructor_dashboard.html

@@ -8,17 +8,99 @@
 
 <%include file="/courseware/course_navigation.html" args="active_page='instructor'" />
 
+<style type="text/css">
+table.stat_table {
+	font-family: verdana,arial,sans-serif;
+	font-size:11px;
+	color:#333333;
+	border-width: 1px;
+	border-color: #666666;
+	border-collapse: collapse;
+}
+table.stat_table th {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #dedede;
+}
+table.stat_table td {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #ffffff;
+}
+</style>
+
 <section class="container">
 <div class="instructor-dashboard-wrapper">
   <section class="instructor-dashboard-content">
     <h1>Instructor Dashboard</h1>
 
+    <form method="POST">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
+
     <p>
     <a href="${reverse('gradebook', kwargs=dict(course_id=course.id))}">Gradebook</a>
 
     <p>
     <a href="${reverse('grade_summary', kwargs=dict(course_id=course.id))}">Grade summary</a>
 
+    <p>
+    <input type="submit" name="action" value="Dump list of enrolled students">
+
+    <p>
+    <input type="submit" name="action" value="Dump Grades for all students in this course">
+    <input type="submit" name="action" value="Download CSV of all student grades for this course">
+
+    <p>
+    <input type="submit" name="action" value="Dump all RAW grades for all students in this course">
+    <input type="submit" name="action" value="Download CSV of all RAW grades">
+
+%if instructor_access:
+    <hr width="40%" style="align:left">
+    <p>
+    <input type="submit" name="action" value="List course staff members"> 
+    <p>
+    <input type="text" name="staffuser"> <input type="submit" name="action" value="Remove course staff"> 
+    <input type="submit" name="action" value="Add course staff"> 
+    <hr width="40%" style="align:left">
+ %endif
+ 
+%if admin_access:
+    <p>
+    <input type="submit" name="action" value="Reload course from XML files"> 
+    <input type="submit" name="action" value="GIT pull and Reload course"> 
+%endif
+
+    </form>
+
+    <br/>
+    <br/>
+    <p>
+      <hr width="100%">
+      <h2>${datatable['title']}</h2>
+      <table class="stat_table">
+        <tr>
+	%for hname in datatable['header']:
+	  <th>${hname}</th>
+	%endfor
+	</tr>
+        %for row in datatable['data']:
+          <tr>
+	    %for value in row:
+	      <td>${value}</td>
+	    %endfor
+	  </tr>
+        %endfor
+      </table>
+    </p>
+
+%if msg:
+    <p>${msg}</p>    
+%endif
+
   </section>
 </div>
 </section>

lms/templates/dashboard.html

@@ -72,30 +72,26 @@
           else:
             course_target = reverse('about_course', args=[course.id])
           %>
-          <a href="${course_target}" class="cover" style="background-image: url('${course_image_url(course)}')">
+
+          <a href="${course_target}">
+            <section class="cover" style="background-image: url('${course_image_url(course)}')">
               <div class="shade"></div>
               <div class="arrow">&#10095;</div>
-          </a>
+            </section>
+
             <section class="info">
               <hgroup>
-              <a href="${reverse('university_profile', args=[course.org])}" class="university">${get_course_about_section(course, 'university')}</a>
-              <h3><a href="${course_target}">${course.number} ${course.title}</a></h3>
+                <h2 class="university">${get_course_about_section(course, 'university')}</h2>
+                <h3>${course.number} ${course.title}</h3>
               </hgroup>
               <section class="course-status">
-              <p>Class Starts - <span>${course.start_date_text}</span></div>
-            </section>
-            <section class="meta">
-              <div class="course-work-icon"></div>
-              <div class="progress">
-                <div class="meter">
-                  <div class="meter-fill"></div>
-                </div>
-              </div>
-              <div class="complete">
-                ##<p><span class="completeness">60%</span> complete</p>
-              </div>
+                <p>Class Starts - <span>${course.start_date_text}</span></p>
               </section>
+              % if course.id in show_courseware_links_for:
+                <p class="enter-course">View Courseware</p>
+              % endif
             </section>
+          </a>
         </article>
         <a href="#unenroll-modal" class="unenroll" rel="leanModal" data-course-id="${course.id}" data-course-number="${course.number}">Unregister</a>
 

lms/templates/discussion/_sort.html

@@ -31,10 +31,10 @@
 
 <div class="discussion-sort local">
   <span class="discussion-label">Sort by:</span>
-  ${link_to_sort('activity', 'top')}
-  
   ${link_to_sort('date', 'date')}
 
+  ${link_to_sort('activity', 'top')}
+  
   ${link_to_sort('votes', 'votes')}
   
   ${link_to_sort('comments', 'comments')}

lms/templates/discussion/mustache/_content.mustache

@@ -27,6 +27,11 @@
             {{/content.tags}}
           </div>
         {{/thread}}
+        <div class="context">
+            {{#content.courseware_location}}
+               (this post is about <a href="../../jump_to/{{content.courseware_location}}">{{content.courseware_title}}</a>)
+            {{/content.courseware_location}}
+        </div>
         <div class="info">
           <div class="comment-time">
             <span class="timeago" title="{{content.updated_at}}">sometime</span> by

lms/templates/identity.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<xrds:XRDS xmlns:xrds="xri://$xrds" xmlns="xri://$xrd*($v*2.0)">
+    <XRD>
+        <Service priority="0">
+            <Type>http://specs.openid.net/auth/2.0/signon</Type>
+            <Type>http://openid.net/signon/1.1</Type>
+            <URI>${url}</URI>
+        </Service>
+    </XRD>
+</xrds:XRDS>

lms/templates/portal/course_about.html

@@ -63,19 +63,12 @@
         <div class="main-cta">
         %if user.is_authenticated():
             %if registered:
-   	        <%
-		  ## TODO: move this logic into a view
-                  if has_access(user, course, 'load'):
-                      course_target = reverse('info', args=[course.id])
-                  else:
-                      course_target = reverse('about_course', args=[course.id])
-                  show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION')
-                %>
-	        %if show_link:
+	              %if show_courseware_link:
                     <a href="${course_target}">
                 %endif
-                <span class="register disabled">You are registered for this course (${course.number}).</span>
-	        %if show_link:
+                <span class="register disabled">You are registered for this course (${course.number})</span>
+                %if show_courseware_link:
+                    <strong>View Courseware</strong>
                     </a>
                 %endif
             %else:

lms/templates/problem.html

@@ -32,3 +32,27 @@
   </section>
 </section>
 
+<%block name="js_extra">
+<script type="text/javascript" charset="utf-8">
+  $(function(){
+      // this should be brought back into problems
+      $('.longform').hide();
+      $('.shortform').append('<a href="#" class="full">See full output</a>');
+
+      $('.full').click(function() {
+        $(this).parent().siblings().slideToggle();
+        $(this).parent().parent().toggleClass('open');
+        var text = $(this).text() == 'See full output' ? 'Hide output' : 'See full output';
+        $(this).text(text);
+        return false;
+      });
+
+      $('.collapsible section').hide()
+      $('.collapsible header a').click(function() {
+        $(this).parent().siblings().slideToggle();
+        $(this).parent().parent().toggleClass('open');
+        return false
+      });
+    });
+</script>
+</%block>

lms/templates/provider_login.html

+<%inherit file="main.html" />
+<%namespace name='static' file='static_content.html'/>
+
+<%block name="headextra">
+<style type="text/css">
+.openid-login {
+    display: block;
+    position: relative;
+    left: 0;
+    margin: 100px auto;
+    top: 0;
+    z-index: 200;
+}
+
+.openid-login input[type=submit] {
+    white-space: normal;
+    height: auto !important;
+}
+
+#lean_overlay {
+  display: block;
+  position: fixed;
+  left: 0px;
+  top: 0px;
+  z-index: 100;
+  width:100%;
+  height:100%;
+}
+</style>
+</%block>
+
+<section id="login-modal" class="modal login-modal openid-login">
+  <div class="inner-wrapper">
+    <header>
+      <h2>Log In</h2>
+      <hr>
+    </header>
+    <form id="login_form" class="login_form" method="post" action="/openid/provider/login/">
+%if error:
+      <div id="login_error" class="modal-form-error" style="display: block;">Email or password is incorrect.</div>
+%endif
+      <label>E-mail</label>
+      <input type="text" name="email" placeholder="E-mail" tabindex="1" autofocus="autofocus" />
+      <label>Password</label>
+      <input type="password" name="password" placeholder="Password" tabindex="2" />
+      <div class="submit">
+        <input name="submit" type="submit" value="Return To ${return_to}" tabindex="3" />
+      </div>
+    </form>
+  </div>
+</section>
+<div id="lean_overlay"></div>

lms/templates/staff_problem_info.html

@@ -32,6 +32,7 @@ ${module_content}
       <h2>Staff Debug</h2>
     </header>
     <div class="staff_info" style="display:block">
+is_released = ${is_released}
 location = ${location | h}
 github = <a href="${edit_link}">${edit_link | h}</a>
 %if source_file:

lms/templates/video.html

-% if name is not UNDEFINED and name is not None:
-  <h1> ${display_name} </h1>
+% if display_name is not UNDEFINED and display_name is not None:
+  <h2> ${display_name} </h2>
 % endif
 
-<div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}">
+<div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}" data-show-captions="${show_captions}">
   <div class="tc-wrapper">
     <article class="video-wrapper">
       <section class="video-player">

lms/templates/wiki/base.html

@@ -39,7 +39,6 @@
     {% with mathjax_mode='wiki' %}
       {% include "mathjax_include.html" %}
     {% endwith %}
-    
   {% endaddtoblock %}
 
 {% endblock %}
@@ -69,6 +68,7 @@
 
       {% endblock %}
     </div>
+
   </section>
 
 {% endblock %}

lms/templates/wiki/create.html

@@ -42,6 +42,7 @@
             {% trans "Go back" %}
           </a>
       </div>  
+      {% include "wiki/includes/cheatsheet.html" %}
     </form>
 
   </article>

lms/templates/wiki/edit.html

@@ -40,7 +40,10 @@
       </a>
     </div>
   </div>
+  {% include "wiki/includes/cheatsheet.html" %}
 </form>
 
 {% endblock %}
 
+
+

lms/templates/wiki/includes/cheatsheet.html

+<div class="modal hide fade" id="cheatsheetModal">
+	<a href="#" class="close-btn">×</a>
+	<div id="cheatsheet-body" class="modal-body">
+		<div class="left-column">
+			<section>
+				<h2>Wiki Syntax Help</h2>
+				<p>This wiki uses <strong>Markdown</strong> for styling. There are several useful guides online. See any of the links below for in-depth details:</p>
+				<ul>
+					<li><a href="http://daringfireball.net/projects/markdown/basics" target="_blank">Markdown: Basics</a></li>
+					<li><a href="http://greg.vario.us/doc/markdown.txt" target="_blank">Quick Markdown Syntax Guide</a></li>
+					<li><a href="http://www.lowendtalk.com/discussion/6/miniature-markdown-guide" target="_blank">Miniature Markdown Guide</a></li>
+				</ul>
+				<p>To create a new wiki article, create a link to it. Clicking the link gives you the creation page.</p>
+				<pre>[Article Name](wiki:ArticleName)</pre>
+			</section>
+			<section>
+				<h3>edX Additions:</h3>
+				<pre>
+circuit-schematic:</pre>
+				<pre>
+$LaTeX Math Expression$</pre>
+			</section>
+		</div>
+
+		<div class="right-column">
+			<section>
+				<h3>Useful examples:</h3>
+				<pre>
+http://wikipedia.org
+[Wikipedia](http://wikipedia.org)
+[edX Wiki](wiki:/edx/)</pre>
+				<pre>
+Huge Header
+===========</pre>
+				<pre>
+Smaller Header
+--------------</pre>
+				<pre>
+*emphasis* or _emphasis_</pre>
+				<pre>
+**strong** or __strong__</pre>
+				<pre>
+- Unordered List
+	- Sub Item 1
+	- Sub Item 2</pre>
+				<pre>
+1. Ordered
+2. List</pre>
+				<pre>
+> Quotes</pre>
+			</section>
+		</div>
+
+		</div>
+	</div>
+</div>

lms/templates/wiki/includes/editor_widget.html

+<textarea {{ attrs }}>{{ content }}</textarea>
+<p id="hint_id_content" class="help-block">
+  Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.
+</p>

lms/templates/xrds.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<xrds:XRDS xmlns:xrds="xri://$xrds" xmlns="xri://$xrd*($v*2.0)">
+    <XRD>
+        <Service priority="0">
+            <Type>http://specs.openid.net/auth/2.0/server</Type>
+            <Type>http://openid.net/sreg/1.0</Type>
+            <Type>http://openid.net/srv/ax/1.0</Type>
+            <URI>${url}</URI>
+        </Service>
+    </XRD>
+</xrds:XRDS>

lms/urls.py

@@ -98,8 +98,9 @@ if settings.COURSEWARE_ENABLED:
     urlpatterns += (
         # Hook django-masquerade, allowing staff to view site as other users
         url(r'^masquerade/', include('masquerade.urls')),
-        url(r'^jump_to/(?P<location>.*)$', 'courseware.views.jump_to', name="jump_to"),
 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/jump_to/(?P<location>.*)$',
+            'courseware.views.jump_to', name="jump_to"),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/modx/(?P<location>.*?)/(?P<dispatch>[^/]*)$',
             'courseware.module_render.modx_dispatch',
             name='modx_dispatch'),
@@ -142,6 +143,8 @@ if settings.COURSEWARE_ENABLED:
             'courseware.views.index', name="courseware_chapter"),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/$',
             'courseware.views.index', name="courseware_section"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/(?P<position>[^/]*)/?$',
+            'courseware.views.index', name="courseware_position"),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/progress$',
             'courseware.views.progress', name="progress"),
         # Takes optional student_id for instructor use--shows profile as that student sees it.
@@ -150,14 +153,14 @@ if settings.COURSEWARE_ENABLED:
 
         # For the instructor
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/instructor$',
-            'courseware.views.instructor_dashboard', name="instructor_dashboard"),
+            'instructor.views.instructor_dashboard', name="instructor_dashboard"),
 
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/gradebook$',
-            'courseware.views.gradebook', name='gradebook'),
+            'instructor.views.gradebook', name='gradebook'),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/grade_summary$',
-            'courseware.views.grade_summary', name='grade_summary'),
+            'instructor.views.grade_summary', name='grade_summary'),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/enroll_students$',
-            'courseware.views.enroll_students', name='enroll_students'),
+            'instructor.views.enroll_students', name='enroll_students'),
     )
 
     # discussion forums live within courseware, so courseware must be enabled first
@@ -215,10 +218,18 @@ if settings.DEBUG:
 if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
     urlpatterns += (
         url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'),
-        url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'),
+        url(r'^openid/complete/$', 'external_auth.views.openid_login_complete', name='openid-complete'),
         url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
     )
 
+if settings.MITX_FEATURES.get('AUTH_USE_OPENID_PROVIDER'):
+    urlpatterns += (
+        url(r'^openid/provider/login/$', 'external_auth.views.provider_login', name='openid-provider-login'),
+        url(r'^openid/provider/login/(?:[\w%\. ]+)$', 'external_auth.views.provider_identity', name='openid-provider-login-identity'),
+        url(r'^openid/provider/identity/$', 'external_auth.views.provider_identity', name='openid-provider-identity'),
+        url(r'^openid/provider/xrds/$', 'external_auth.views.provider_xrds', name='openid-provider-xrds')
+    )
+
 if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
     urlpatterns += (
         url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),

lms/wsgi.py

+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws")
+
+# This application object is used by the development server
+# as well as any WSGI server configured to use this file.
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+from django.conf import settings
+from xmodule.modulestore.django import modulestore
+
+for store_name in settings.MODULESTORE:
+    modulestore(store_name)

requirements.txt

@@ -46,4 +46,5 @@ django-sekizai<0.7
 django-mptt>=0.5.3
 sorl-thumbnail
 networkx
+pygraphviz
 -r repo-requirements.txt