Merge master

Gemfile

 source :rubygems
-
+ruby "1.9.3"
 gem 'rake'
 gem 'sass', '3.1.15'
 gem 'bourbon', '~> 1.3.6'

cms/djangoapps/contentstore/management/commands/import.py

@@ -23,4 +23,7 @@ class Command(BaseCommand):
             course_dirs = args[1:]
         else:
             course_dirs = None
+        print "Importing.  Data_dir={data}, course_dirs={courses}".format(
+            data=data_dir,
+            courses=course_dirs)
         import_from_xml(modulestore(), data_dir, course_dirs)

cms/djangoapps/contentstore/views.py

@@ -108,7 +108,7 @@ def edit_item(request):
         'contents': item.get_html(),
         'js_module': item.js_module_name,
         'category': item.category,
-        'name': item.name,
+        'url_name': item.url_name,
         'previews': get_module_previews(request, item),
     })
 
@@ -176,7 +176,7 @@ def load_preview_state(request, preview_id, location):
 
 def save_preview_state(request, preview_id, location, instance_state, shared_state):
     """
-    Load the state of a preview module to the request
+    Save the state of a preview module to the request
 
     preview_id (str): An identifier specifying which preview this module is used for
     location: The Location of the module to dispatch to
@@ -214,7 +214,10 @@ def preview_module_system(request, preview_id, descriptor):
         get_module=partial(get_preview_module, request, preview_id),
         render_template=render_from_lms,
         debug=True,
-        replace_urls=replace_urls
+        replace_urls=replace_urls,
+        # TODO (vshnayder): All CMS users get staff view by default
+        # is that what we want?
+        is_staff=True,
     )
 
 

cms/djangoapps/github_sync/tests/test_views.py

@@ -11,33 +11,33 @@ class PostReceiveTestCase(TestCase):
     def setUp(self):
         self.client = Client()
 
-    @patch('github_sync.views.sync_with_github')
-    def test_non_branch(self, sync_with_github):
+    @patch('github_sync.views.import_from_github')
+    def test_non_branch(self, import_from_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/tags/foo'})
         })
-        self.assertFalse(sync_with_github.called)
+        self.assertFalse(import_from_github.called)
 
-    @patch('github_sync.views.sync_with_github')
-    def test_non_watched_repo(self, sync_with_github):
+    @patch('github_sync.views.import_from_github')
+    def test_non_watched_repo(self, import_from_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/heads/branch',
             'repository': {'name': 'bad_repo'}})
         })
-        self.assertFalse(sync_with_github.called)
+        self.assertFalse(import_from_github.called)
 
-    @patch('github_sync.views.sync_with_github')
-    def test_non_tracked_branch(self, sync_with_github):
+    @patch('github_sync.views.import_from_github')
+    def test_non_tracked_branch(self, import_from_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/heads/non_branch',
             'repository': {'name': 'repo'}})
         })
-        self.assertFalse(sync_with_github.called)
+        self.assertFalse(import_from_github.called)
 
-    @patch('github_sync.views.sync_with_github')
-    def test_tracked_branch(self, sync_with_github):
+    @patch('github_sync.views.import_from_github')
+    def test_tracked_branch(self, import_from_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/heads/branch',
             'repository': {'name': 'repo'}})
         })
-        sync_with_github.assert_called_with(load_repo_settings('repo'))
+        import_from_github.assert_called_with(load_repo_settings('repo'))

cms/djangoapps/github_sync/views.py

@@ -5,7 +5,7 @@ from django.http import HttpResponse
 from django.conf import settings
 from django_future.csrf import csrf_exempt
 
-from . import sync_with_github, load_repo_settings
+from . import import_from_github, load_repo_settings
 
 log = logging.getLogger()
 
@@ -46,6 +46,6 @@ def github_post_receive(request):
         log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name))
         return HttpResponse('Ignoring non-tracked branch')
 
-    sync_with_github(repo)
+    import_from_github(repo)
 
     return HttpResponse('Push received')

cms/envs/dev.py

@@ -2,13 +2,18 @@
 This config file runs the simplest dev environment"""
 
 from .common import *
+from .logsettings import get_logger_config
 
 import logging
 import sys
-logging.basicConfig(stream=sys.stdout, )
 
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
+LOGGING = get_logger_config(ENV_ROOT / "log",
+                            logging_env="dev",
+                            tracking_filename="tracking.log",
+                            debug=True)
+
 
 MODULESTORE = {
     'default': {
@@ -37,7 +42,8 @@ REPOS = {
     },
     'content-mit-6002x': {
         'branch': 'master',
-        'origin': 'git@github.com:MITx/6002x-fall-2012.git',
+        #'origin': 'git@github.com:MITx/6002x-fall-2012.git',
+        'origin': 'git@github.com:MITx/content-mit-6002x.git',
     },
     '6.00x': {
         'branch': 'master',
@@ -75,3 +81,6 @@ CACHES = {
         'KEY_FUNCTION': 'util.memcache.safe_key',
     }
 }
+
+# Make the keyedcache startup warnings go away
+CACHE_TIMEOUT = 0

cms/envs/logsettings.py

@@ -3,19 +3,19 @@ import os.path
 import platform
 import sys
 
-def get_logger_config(log_dir, 
-                      logging_env="no_env", 
+def get_logger_config(log_dir,
+                      logging_env="no_env",
                       tracking_filename=None,
                       syslog_addr=None,
                       debug=False):
     """Return the appropriate logging config dictionary. You should assign the
-    result of this to the LOGGING var in your settings. The reason it's done 
+    result of this to the LOGGING var in your settings. The reason it's done
     this way instead of registering directly is because I didn't want to worry
-    about resetting the logging state if this is called multiple times when 
+    about resetting the logging state if this is called multiple times when
     settings are extended."""
 
     # If we're given an explicit place to put tracking logs, we do that (say for
-    # debugging). However, logging is not safe for multiple processes hitting 
+    # debugging). However, logging is not safe for multiple processes hitting
     # the same file. So if it's left blank, we dynamically create the filename
     # based on the PID of this worker process.
     if tracking_filename:
@@ -33,6 +33,7 @@ def get_logger_config(log_dir,
 
     return {
         'version': 1,
+        'disable_existing_loggers': False,
         'formatters' : {
             'standard' : {
                 'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s',

cms/static/sass/_base.scss

@@ -2,6 +2,7 @@ $fg-column: 70px;
 $fg-gutter: 26px;
 $fg-max-columns: 12;
 $body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
+$sans-serif: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
 $body-font-size: 14px;
 $body-line-height: 20px;
 
@@ -12,6 +13,7 @@ $orange: #f96e5b;
 $yellow: #fff8af;
 $cream: #F6EFD4;
 $mit-red: #933;
+$border-color: #ddd;
 
 @mixin hide-text {
   background-color: transparent;

cms/static/sass/_content-types.scss

@@ -56,10 +56,10 @@
 
 .module a:first-child {
   @extend .content-type;
-  background-image: url('/static/img/content-types/module.png');
+  background-image: url('../img/content-types/module.png');
 }
 
 .module a:first-child {
   @extend .content-type;
-  background-image: url('/static/img/content-types/module.png');
+  background-image: url('../img/content-types/module.png');
 }

cms/templates/unit.html

 <section id="unit-wrapper">
   <header>
     <section>
-      <h1 class="editable">${name}</h1>
+      <h1 class="editable">${url_name}</h1>
       <p class="${category}"><a href="#">${category}</a></p>
     </section>
 

cms/templates/widgets/navigation.html

@@ -41,7 +41,7 @@
     % for week in weeks:
     <li class="week" data-id="${week.location.url()}">
       <header>
-        <h1><a href="#" class="week-edit">${week.name}</a></h1>
+        <h1><a href="#" class="week-edit">${week.url_name}</a></h1>
         <ul>
           % if 'goals' in week.metadata:
             % for goal in week.metadata['goals']:
@@ -60,7 +60,7 @@
             data-type="${module.js_module_name}"
             data-preview-type="${module.module_class.js_module_name}">
 
-            <a href="#" class="module-edit">${module.name}</a>
+            <a href="#" class="module-edit">${module.url_name}</a>
             <a href="#" class="draggable">handle</a>
         </li>
         % endfor

cms/templates/widgets/sequence-edit.html

@@ -39,7 +39,7 @@
               <a href="#" class="module-edit"
                  data-id="${child.location.url()}"
                  data-type="${child.js_module_name}"
-                 data-preview-type="${child.module_class.js_module_name}">${child.name}</a>
+                 data-preview-type="${child.module_class.js_module_name}">${child.url_name}</a>
               <a href="#" class="draggable">handle</a>
             </li>
             %endfor

common/djangoapps/external_auth/admin.py

+'''
+django admin pages for courseware model
+'''
+
+from external_auth.models import *
+from django.contrib import admin
+
+admin.site.register(ExternalAuthMap)

common/djangoapps/external_auth/models.py

+"""
+WE'RE USING MIGRATIONS!
+
+If you make changes to this model, be sure to create an appropriate migration
+file and check it in at the same time as your model changes. To do that,
+
+1. Go to the mitx dir
+2. django-admin.py schemamigration student --auto --settings=lms.envs.dev --pythonpath=. description_of_your_change
+3. Add the migration file created in mitx/common/djangoapps/external_auth/migrations/
+"""
+
+from django.db import models
+from django.contrib.auth.models import User
+
+class ExternalAuthMap(models.Model):
+    class Meta:
+        unique_together = (('external_id', 'external_domain'), )
+    external_id = models.CharField(max_length=255, db_index=True)
+    external_domain = models.CharField(max_length=255, db_index=True)
+    external_credentials = models.TextField(blank=True)  # JSON dictionary 
+    external_email = models.CharField(max_length=255, db_index=True)
+    external_name = models.CharField(blank=True,max_length=255, db_index=True)
+    user = models.OneToOneField(User, unique=True, db_index=True, null=True)
+    internal_password = models.CharField(blank=True, max_length=31)	# randomly generated
+    dtcreated = models.DateTimeField('creation date',auto_now_add=True)
+    dtsignup = models.DateTimeField('signup date',null=True)		# set after signup
+    
+    def __unicode__(self):
+        s = "[%s] = (%s / %s)" % (self.external_id, self.external_name, self.external_email)
+        return s
+    

common/djangoapps/external_auth/views.py

+import json
+import logging
+import random
+import re
+import string
+
+from external_auth.models import ExternalAuthMap
+
+from django.conf import settings
+from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login
+from django.contrib.auth.models import Group
+from django.contrib.auth.models import User
+
+from django.core.urlresolvers import reverse
+from django.http import HttpResponse, HttpResponseRedirect
+from django.shortcuts import render_to_response
+from django.shortcuts import redirect
+from django.template import RequestContext
+from mitxmako.shortcuts import render_to_response, render_to_string
+try:
+    from django.views.decorators.csrf import csrf_exempt
+except ImportError:
+    from django.contrib.csrf.middleware import csrf_exempt
+from django_future.csrf import ensure_csrf_cookie
+from util.cache import cache_if_anonymous
+    
+from django_openid_auth import auth as openid_auth
+from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE)
+import django_openid_auth.views as openid_views
+
+import student.views as student_views
+
+log = logging.getLogger("mitx.external_auth")
+
+@csrf_exempt
+def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None):
+    """Render an Openid error page to the user."""
+    message = "In openid_failure " + message
+    log.debug(message)
+    data = render_to_string( template_name, dict(message=message, exception=exception))
+    return HttpResponse(data, status=status)
+
+#-----------------------------------------------------------------------------
+# Openid
+
+def edXauth_generate_password(length=12, chars=string.letters + string.digits):
+    """Generate internal password for externally authenticated user"""
+    return ''.join([random.choice(chars) for i in range(length)])
+
+@csrf_exempt
+def edXauth_openid_login_complete(request,  redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None):
+    """Complete the openid login process"""
+
+    redirect_to = request.REQUEST.get(redirect_field_name, '')
+    render_failure = render_failure or \
+                     getattr(settings, 'OPENID_RENDER_FAILURE', None) or \
+                     default_render_failure
+                                                   
+    openid_response = openid_views.parse_openid_response(request)
+    if not openid_response:
+        return render_failure(request, 'This is an OpenID relying party endpoint.')
+
+    if openid_response.status == SUCCESS:
+        external_id = openid_response.identity_url
+        oid_backend =  openid_auth.OpenIDBackend()
+        details = oid_backend._extract_user_details(openid_response)
+
+        log.debug('openid success, details=%s' % details)
+
+        return edXauth_external_login_or_signup(request,
+                                                external_id, 
+                                                "openid:%s" % settings.OPENID_SSO_SERVER_URL,
+                                                details,
+                                                details.get('email',''),
+                                                '%s %s' % (details.get('first_name',''),details.get('last_name',''))
+                                                )
+                                 
+    return render_failure(request, 'Openid failure')
+
+#-----------------------------------------------------------------------------
+# generic external auth login or signup
+
+def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname,
+                                     retfun=None):
+    # see if we have a map from this external_id to an edX username
+    try:
+        eamap = ExternalAuthMap.objects.get(external_id = external_id,
+                                            external_domain = external_domain,
+                                            )
+        log.debug('Found eamap=%s' % eamap)
+    except ExternalAuthMap.DoesNotExist:
+        # go render form for creating edX user
+        eamap = ExternalAuthMap(external_id = external_id,
+                                external_domain = external_domain,
+                                external_credentials = json.dumps(credentials),
+                                )
+        eamap.external_email = email
+        eamap.external_name = fullname
+        eamap.internal_password = edXauth_generate_password()
+        log.debug('created eamap=%s' % eamap)
+
+        eamap.save()
+
+    internal_user = eamap.user
+    if internal_user is None:
+        log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email)
+        return edXauth_signup(request, eamap)
+    
+    uname = internal_user.username
+    user = authenticate(username=uname, password=eamap.internal_password)
+    if user is None:
+        log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password))
+        return edXauth_signup(request, eamap)
+
+    if not user.is_active:
+        log.warning("External Auth: user %s is not active" % (uname))
+        # TODO: improve error page
+        return render_failure(request, 'Account not yet activated: please look for link in your email')
+     
+    login(request, user)
+    request.session.set_expiry(0)
+    student_views.try_change_enrollment(request)
+    log.info("Login success - {0} ({1})".format(user.username, user.email))
+    if retfun is None:
+        return redirect('/')
+    return retfun()
+        
+    
+#-----------------------------------------------------------------------------
+# generic external auth signup
+
+@ensure_csrf_cookie
+@cache_if_anonymous
+def edXauth_signup(request, eamap=None):
+    """
+    Present form to complete for signup via external authentication.
+    Even though the user has external credentials, he/she still needs
+    to create an account on the edX system, and fill in the user
+    registration form.
+
+    eamap is an ExteralAuthMap object, specifying the external user
+    for which to complete the signup.
+    """
+    
+    if eamap is None:
+        pass
+
+    request.session['ExternalAuthMap'] = eamap	# save this for use by student.views.create_account
+    
+    context = {'has_extauth_info': True,
+               'show_signup_immediately' : True,
+               'extauth_email': eamap.external_email,
+               'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces
+               'extauth_name': eamap.external_name,
+               }
+    
+    log.debug('ExtAuth: doing signup for %s' % eamap.external_email)
+
+    return student_views.main_index(extra_context=context)
+
+#-----------------------------------------------------------------------------
+# MIT SSL
+
+def ssl_dn_extract_info(dn):
+    '''
+    Extract username, email address (may be anyuser@anydomain.com) and full name
+    from the SSL DN string.  Return (user,email,fullname) if successful, and None
+    otherwise.
+    '''
+    ss = re.search('/emailAddress=(.*)@([^/]+)', dn)
+    if ss:
+        user = ss.group(1)
+        email = "%s@%s" % (user, ss.group(2))
+    else:
+        return None
+    ss = re.search('/CN=([^/]+)/', dn)
+    if ss:
+        fullname = ss.group(1)
+    else:
+        return None
+    return (user, email, fullname)
+
+@csrf_exempt
+def edXauth_ssl_login(request):
+    """
+    This is called by student.views.index when MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
+
+    Used for MIT user authentication.  This presumes the web server (nginx) has been configured 
+    to require specific client certificates.
+
+    If the incoming protocol is HTTPS (SSL) then authenticate via client certificate.  
+    The certificate provides user email and fullname; this populates the ExternalAuthMap.
+    The user is nevertheless still asked to complete the edX signup.  
+
+    Else continues on with student.views.main_index, and no authentication.
+    """
+    certkey = "SSL_CLIENT_S_DN"	 # specify the request.META field to use
+    
+    cert = request.META.get(certkey,'')
+    if not cert:
+        cert = request.META.get('HTTP_'+certkey,'')
+    if not cert:
+        try:
+            cert = request._req.subprocess_env.get(certkey,'')	 # try the direct apache2 SSL key
+        except Exception as err:
+            pass
+    if not cert:
+        # no certificate information - go onward to main index
+        return student_views.main_index()
+
+    (user, email, fullname) = ssl_dn_extract_info(cert)
+    
+    return edXauth_external_login_or_signup(request, 
+                                            external_id=email, 
+                                            external_domain="ssl:MIT", 
+                                            credentials=cert, 
+                                            email=email,
+                                            fullname=fullname,
+                                            retfun = student_views.main_index)

common/djangoapps/student/views.py

@@ -8,7 +8,6 @@ import uuid
 import feedparser
 import urllib
 import itertools
-from collections import defaultdict
 
 from django.conf import settings
 from django.contrib.auth import logout, authenticate, login
@@ -23,7 +22,7 @@ from django.http import HttpResponse, Http404
 from django.shortcuts import redirect
 from mitxmako.shortcuts import render_to_response, render_to_string
 from django.core.urlresolvers import reverse
-from BeautifulSoup import BeautifulSoup
+from bs4 import BeautifulSoup
 from django.core.cache import cache
 
 from django_future.csrf import ensure_csrf_cookie
@@ -37,6 +36,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment
 from datetime import date
 from collections import namedtuple
+from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university
 
 log = logging.getLogger("mitx.student")
 Article = namedtuple('Article', 'title url author image deck publication publish_date')
@@ -60,6 +60,19 @@ def index(request):
     if settings.COURSEWARE_ENABLED and request.user.is_authenticated():
         return redirect(reverse('dashboard'))
 
+    if settings.MITX_FEATURES.get('AUTH_USE_MIT_CERTIFICATES'):
+        from external_auth.views import edXauth_ssl_login
+        return edXauth_ssl_login(request)
+
+    return main_index(user=request.user)
+
+def main_index(extra_context = {}, user=None):
+    '''
+    Render the edX main page.
+
+    extra_context is used to allow immediate display of certain modal windows, eg signup,
+    as used by external_auth.
+    '''
     feed_data = cache.get("students_index_rss_feed_data")
     if feed_data == None:
         if hasattr(settings, 'RSS_URL'):
@@ -75,13 +88,11 @@ def index(request):
         entry.image = soup.img['src'] if soup.img else None
         entry.summary = soup.getText()
 
-    universities = defaultdict(list)
-    courses = sorted(modulestore().get_courses(), key=lambda course: course.number)
-    for course in courses:
-        universities[course.org].append(course)
-
-    return render_to_response('index.html', {'universities': universities, 'entries': entries})
-
+    # The course selection work is done in courseware.courses.
+    universities = get_courses_by_university(None)
+    context = {'universities': universities, 'entries': entries}
+    context.update(extra_context)
+    return render_to_response('index.html', context)
 
 def course_from_id(id):
     course_loc = CourseDescriptor.id_to_location(id)
@@ -170,6 +181,14 @@ def change_enrollment(request):
                       .format(user.username, enrollment.course_id))
             return {'success': False, 'error': 'The course requested does not exist.'}
 
+        if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'):
+            # require that user be in the staff_* group (or be an overall admin) to be able to enroll
+            # eg staff_6.002x or staff_6.00x
+            if not has_staff_access_to_course(user,course):
+                staff_group = course_staff_group_name(course)
+                log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group))
+                return {'success': False, 'error' : '%s membership required to access course.' % staff_group}
+
         enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id)
         return {'success': True}
 
@@ -256,11 +275,26 @@ def change_setting(request):
 
 @ensure_csrf_cookie
 def create_account(request, post_override=None):
-    ''' JSON call to enroll in the course. '''
+    '''
+    JSON call to create new edX account.
+    Used by form in signup_modal.html, which is included into navigation.html
+    '''
     js = {'success': False}
 
     post_vars = post_override if post_override else request.POST
 
+    # if doing signup for an external authorization, then get email, password, name from the eamap
+    # don't use the ones from the form, since the user could have hacked those
+    DoExternalAuth = 'ExternalAuthMap' in request.session
+    if DoExternalAuth:
+        eamap = request.session['ExternalAuthMap']
+        email = eamap.external_email
+        name = eamap.external_name
+        password = eamap.internal_password
+        post_vars = dict(post_vars.items())
+        post_vars.update(dict(email=email, name=name, password=password))
+        log.debug('extauth test: post_vars = %s' % post_vars)
+
     # Confirm we have a properly formed request
     for a in ['username', 'email', 'password', 'name']:
         if a not in post_vars:
@@ -355,8 +389,9 @@ def create_account(request, post_override=None):
          'key': r.activation_key,
          }
 
+    # composes activation email
     subject = render_to_string('emails/activation_email_subject.txt', d)
-        # Email subject *must not* contain newlines
+    # Email subject *must not* contain newlines
     subject = ''.join(subject.splitlines())
     message = render_to_string('emails/activation_email.txt', d)
 
@@ -381,6 +416,17 @@ def create_account(request, post_override=None):
 
     try_change_enrollment(request)
 
+    if DoExternalAuth:
+        eamap.user = login_user
+        eamap.dtsignup = datetime.datetime.now()
+        eamap.save()
+        log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'],eamap))
+
+        if settings.MITX_FEATURES.get('BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'):
+            log.debug('bypassing activation email')
+            login_user.is_active = True
+            login_user.save()
+
     js = {'success': True}
     return HttpResponse(json.dumps(js), mimetype="application/json")
 

common/djangoapps/track/models.py

 from django.db import models
 
-# Create your models here.
+from django.db import models
+
+class TrackingLog(models.Model):
+    dtcreated = models.DateTimeField('creation date',auto_now_add=True)
+    username = models.CharField(max_length=32,blank=True)
+    ip = models.CharField(max_length=32,blank=True)
+    event_source = models.CharField(max_length=32)
+    event_type = models.CharField(max_length=32,blank=True)
+    event = models.TextField(blank=True)
+    agent = models.CharField(max_length=256,blank=True)
+    page = models.CharField(max_length=32,blank=True,null=True)
+    time = models.DateTimeField('event time')
+
+    def __unicode__(self):
+        s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source,
+                                               self.event_type, self.page, self.event)
+        return s
+
+

common/djangoapps/track/views.py

@@ -2,19 +2,32 @@ import json
 import logging
 import os
 import datetime
+import dateutil.parser
 
-# Create your views here.
+from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.http import Http404
+from django.shortcuts import redirect
 from django.conf import settings
+from mitxmako.shortcuts import render_to_response
+
+from django_future.csrf import ensure_csrf_cookie
+from track.models import TrackingLog
 
 log = logging.getLogger("tracking")
 
+LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time']
 
 def log_event(event):
     event_str = json.dumps(event)
     log.info(event_str[:settings.TRACK_MAX_EVENT])
-
+    if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
+        event['time'] = dateutil.parser.parse(event['time'])
+        tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS ))
+        try:
+            tldat.save()
+        except Exception as err:
+            log.exception(err)
 
 def user_track(request):
     try:  # TODO: Do the same for many of the optional META parameters
@@ -70,4 +83,16 @@ def server_track(request, event_type, event, page=None):
         "page": page,
         "time": datetime.datetime.utcnow().isoformat(),
         }
+
+    if event_type=="/event_logs" and request.user.is_staff:	# don't log
+        return
     log_event(event)
+
+@login_required
+@ensure_csrf_cookie
+def view_tracking_log(request):
+    if not request.user.is_staff:
+        return redirect('/')
+    record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
+    return render_to_response('tracking_log.html',{'records':record_instances})
+

common/djangoapps/xmodule_modifiers.py

+import re
 import json
+import logging
+
 from django.conf import settings
 from functools import wraps
 from static_replace import replace_urls
 from mitxmako.shortcuts import render_to_string
+from xmodule.seq_module import SequenceModule
+from xmodule.vertical_module import VerticalModule
 
+log = logging.getLogger("mitx.xmodule_modifiers")
 
 def wrap_xmodule(get_html, module, template):
     """
@@ -69,29 +75,32 @@ def add_histogram(get_html, module):
     the output of the old get_html function with additional information
     for admin users only, including a histogram of student answers and the
     definition of the xmodule
+
+    Does nothing if module is a SequenceModule
     """
     @wraps(get_html)
     def _get_html():
+
+        if type(module) in [SequenceModule, VerticalModule]:	# TODO: make this more general, eg use an XModule attribute instead
+            return get_html()
+
         module_id = module.id
         histogram = grade_histogram(module_id)
         render_histogram = len(histogram) > 0
 
-        # TODO: fixme - no filename in module.xml in general (this code block for edx4edx)
-        # the following if block is for summer 2012 edX course development; it will change when the CMS comes online
-        if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None:
-            coursename = multicourse_settings.get_coursename_from_request(request)
-            github_url = multicourse_settings.get_course_github_url(coursename)
-            fn = module_xml.get('filename')
-            if module_xml.tag=='problem': fn = 'problems/' + fn	# grrr
-            edit_link = (github_url + '/tree/master/' + fn) if github_url is not None else None
-            if module_xml.tag=='problem': edit_link += '.xml'	# grrr
+        # TODO (ichuang): Remove after fall 2012 LMS migration done
+        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+            [filepath, filename] = module.definition.get('filename','')
+            osfs = module.system.filestore
+            if osfs.exists(filename):
+                filepath = filename	# if original, unmangled filename exists then use it (github doesn't like symlinks)
+            data_dir = osfs.root_path.rsplit('/')[-1]
+            edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath)
         else:
             edit_link = False
 
-        # Cast module.definition and module.metadata to dicts so that json can dump them
-        # even though they are lazily loaded
-        staff_context = {'definition': json.dumps(dict(module.definition), indent=4),
-                         'metadata': json.dumps(dict(module.metadata), indent=4),
+        staff_context = {'definition': module.definition.get('data'),
+                         'metadata': json.dumps(module.metadata, indent=4),
                          'element_id': module.location.html_id(),
                          'edit_link': edit_link,
                          'histogram': json.dumps(histogram),
@@ -100,3 +109,4 @@ def add_histogram(get_html, module):
         return render_to_string("staff_problem_info.html", staff_context)
 
     return _get_html
+

common/lib/capa/capa/capa_problem.py

@@ -294,20 +294,30 @@ class LoncapaProblem(object):
                 try:
                     ifp = self.system.filestore.open(file)  # open using ModuleSystem OSFS filestore
                 except Exception as err:
-                    log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True)))
-                    log.error('Cannot find file %s in %s' % (file, self.system.filestore))
-                    if not self.system.get('DEBUG'):		# if debugging, don't fail - just log error
+                    log.error('Error %s in problem xml include: %s' % (
+                            err, etree.tostring(inc, pretty_print=True)))
+                    log.error('Cannot find file %s in %s' % (
+                            file, self.system.filestore))
+                    # if debugging, don't fail - just log error
+                    # TODO (vshnayder): need real error handling, display to users
+                    if not self.system.get('DEBUG'):
                         raise
-                    else: continue
+                    else:
+                        continue
                 try:
                     incxml = etree.XML(ifp.read())		# read in and convert to XML
                 except Exception as err:
-                    log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True)))
+                    log.error('Error %s in problem xml include: %s' % (
+                            err, etree.tostring(inc, pretty_print=True)))
                     log.error('Cannot parse XML in %s' % (file))
-                    if not self.system.get('DEBUG'):		# if debugging, don't fail - just log error
+                    # if debugging, don't fail - just log error
+                    # TODO (vshnayder): same as above
+                    if not self.system.get('DEBUG'):
                         raise
-                    else: continue
-                parent = inc.getparent()			# insert  new XML into tree in place of inlcude
+                    else:
+                        continue
+                # insert new XML into tree in place of inlcude
+                parent = inc.getparent()
                 parent.insert(parent.index(inc), incxml)
                 parent.remove(inc)
                 log.debug('Included %s into %s' % (file, self.problem_id))
@@ -335,7 +345,7 @@ class LoncapaProblem(object):
             # path is an absolute path or a path relative to the data dir
             dir = os.path.join(self.system.filestore.root_path, dir)
             abs_dir = os.path.normpath(dir)
-            log.debug("appending to path: %s" % abs_dir)
+            #log.debug("appending to path: %s" % abs_dir)
             path.append(abs_dir)
 
         return path

common/lib/supertrace.py

+"""
+A handy util to print a django-debug-screen-like stack trace with
+values of local variables.
+"""
+
+import sys, traceback
+from django.utils.encoding import smart_unicode
+
+
+def supertrace(max_len=160):
+    """
+    Print the usual traceback information, followed by a listing of all the
+    local variables in each frame.  Should be called from an exception handler.
+
+    if max_len is not None, will print up to max_len chars for each local variable.
+
+    (cite: modified from somewhere on stackoverflow)
+    """
+    tb = sys.exc_info()[2]
+    while True:
+        if not tb.tb_next:
+            break
+        tb = tb.tb_next
+    stack = []
+    frame = tb.tb_frame
+    while frame:
+        stack.append(f)
+        frame = frame.f_back
+    stack.reverse()
+    # First print the regular traceback
+    traceback.print_exc()
+
+    print "Locals by frame, innermost last"
+    for frame in stack:
+        print
+        print "Frame %s in %s at line %s" % (frame.f_code.co_name,
+                                             frame.f_code.co_filename,
+                                             frame.f_lineno)
+        for key, value in frame.f_locals.items():
+            print ("\t%20s = " % smart_unicode(key, errors='ignore')),
+            # We have to be careful not to cause a new error in our error
+            # printer! Calling str() on an unknown object could cause an
+            # error.
+            try:
+                s = smart_unicode(value, errors='ignore')
+                if max_len is not None:
+                    s = s[:max_len]
+                print s
+            except:
+                print "<ERROR WHILE PRINTING VALUE>"
+
+

common/lib/xmodule/progress.py

-'''
-Progress class for modules.  Represents where a student is in a module.
-
-Useful things to know:
- - Use Progress.to_js_status_str() to convert a progress into a simple
-   status string to pass to js.
- - Use Progress.to_js_detail_str() to convert a progress into a more detailed
-   string to pass to js.
-
-In particular, these functions have a canonical handing of None.
-
-For most subclassing needs, you should only need to reimplement
-frac() and __str__().
-'''
-
-from collections import namedtuple
-import numbers
-
-
-class Progress(object):
-    '''Represents a progress of a/b (a out of b done)
-
-    a and b must be numeric, but not necessarily integer, with
-    0 <= a <= b and b > 0.
-
-    Progress can only represent Progress for modules where that makes sense.  Other
-    modules (e.g. html) should return None from get_progress().
-
-    TODO: add tag for module type?  Would allow for smarter merging.
-    '''
-
-    def __init__(self, a, b):
-        '''Construct a Progress object.  a and b must be numbers, and must have
-              0 <= a <= b and b > 0
-        '''
-
-        # Want to do all checking at construction time, so explicitly check types
-        if not (isinstance(a, numbers.Number) and
-                isinstance(b, numbers.Number)):
-            raise TypeError('a and b must be numbers.  Passed {0}/{1}'.format(a, b))
-
-        if not (0 <= a <= b and b > 0):
-            raise ValueError(
-                'fraction a/b = {0}/{1} must have 0 <= a <= b and b > 0'.format(a, b))
-
-        self._a = a
-        self._b = b
-
-    def frac(self):
-        ''' Return tuple (a,b) representing progress of a/b'''
-        return (self._a, self._b)
-
-    def percent(self):
-        ''' Returns a percentage progress as a float between 0 and 100.
-
-        subclassing note: implemented in terms of frac(), assumes sanity
-        checking is done at construction time.
-        '''
-        (a, b) = self.frac()
-        return 100.0 * a / b
-
-    def started(self):
-        ''' Returns True if fractional progress is greater than 0.
-
-        subclassing note: implemented in terms of frac(), assumes sanity
-        checking is done at construction time.
-        '''
-        return self.frac()[0] > 0
-
-    def inprogress(self):
-        ''' Returns True if fractional progress is strictly between 0 and 1.
-
-        subclassing note: implemented in terms of frac(), assumes sanity
-        checking is done at construction time.
-        '''
-        (a, b) = self.frac()
-        return a > 0 and a < b
-
-    def done(self):
-        ''' Return True if this represents done.
-
-        subclassing note: implemented in terms of frac(), assumes sanity
-        checking is done at construction time.
-        '''
-        (a, b) = self.frac()
-        return a == b
-
-    def ternary_str(self):
-        ''' Return a string version of this progress: either
-           "none", "in_progress", or "done".
-
-        subclassing note: implemented in terms of frac()
-        '''
-        (a, b) = self.frac()
-        if a == 0:
-            return "none"
-        if a < b:
-            return "in_progress"
-        return "done"
-
-    def __eq__(self, other):
-        ''' Two Progress objects are equal if they have identical values.
-        Implemented in terms of frac()'''
-        if not isinstance(other, Progress):
-            return False
-        (a, b) = self.frac()
-        (a2, b2) = other.frac()
-        return a == a2 and b == b2
-
-    def __ne__(self, other):
-        ''' The opposite of equal'''
-        return not self.__eq__(other)
-
-    def __str__(self):
-        ''' Return a string representation of this string.
-
-        subclassing note: implemented in terms of frac().
-        '''
-        (a, b) = self.frac()
-        return "{0}/{1}".format(a, b)
-
-    @staticmethod
-    def add_counts(a, b):
-        '''Add two progress indicators, assuming that each represents items done:
-        (a / b) + (c / d) = (a + c) / (b + d).
-        If either is None, returns the other.
-        '''
-        if a is None:
-            return b
-        if b is None:
-            return a
-        # get numerators + denominators
-        (n, d) = a.frac()
-        (n2, d2) = b.frac()
-        return Progress(n + n2, d + d2)
-
-    @staticmethod
-    def to_js_status_str(progress):
-        '''
-        Return the "status string" version of the passed Progress
-        object that should be passed to js.  Use this function when
-        sending Progress objects to js to limit dependencies.
-        '''
-        if progress is None:
-            return "NA"
-        return progress.ternary_str()
-
-    @staticmethod
-    def to_js_detail_str(progress):
-        '''
-        Return the "detail string" version of the passed Progress
-        object that should be passed to js.  Use this function when
-        passing Progress objects to js to limit dependencies.
-        '''
-        if progress is None:
-            return "NA"
-        return str(progress)

common/lib/xmodule/setup.py

@@ -25,6 +25,7 @@ setup(
             "discuss = xmodule.backcompat_module:TranslateCustomTagDescriptor",
             "html = xmodule.html_module:HtmlDescriptor",
             "image = xmodule.backcompat_module:TranslateCustomTagDescriptor",
+            "error = xmodule.error_module:ErrorDescriptor",
             "problem = xmodule.capa_module:CapaDescriptor",
             "problemset = xmodule.vertical_module:VerticalDescriptor",
             "section = xmodule.backcompat_module:SemanticSectionDescriptor",

common/lib/xmodule/xmodule/backcompat_module.py

@@ -32,21 +32,25 @@ def process_includes(fn):
                 # read in and convert to XML
                 incxml = etree.XML(ifp.read())
 
-                # insert  new XML into tree in place of inlcude
+                # insert  new XML into tree in place of include
                 parent.insert(parent.index(next_include), incxml)
             except Exception:
-                msg = "Error in problem xml include: %s" % (etree.tostring(next_include, pretty_print=True))
-                log.exception(msg)
-                parent = next_include.getparent()
+                # Log error
+                msg = "Error in problem xml include: %s" % (
+                    etree.tostring(next_include, pretty_print=True))
+                # tell the tracker
+                system.error_tracker(msg)
 
+                # work around
+                parent = next_include.getparent()
                 errorxml = etree.Element('error')
                 messagexml = etree.SubElement(errorxml, 'message')
                 messagexml.text = msg
                 stackxml = etree.SubElement(errorxml, 'stacktrace')
                 stackxml.text = traceback.format_exc()
-
                 # insert error XML in place of include
                 parent.insert(parent.index(next_include), errorxml)
+
             parent.remove(next_include)
 
             next_include = xml_object.find('include')

common/lib/xmodule/xmodule/capa_module.py

@@ -5,6 +5,7 @@ import json
 import logging
 import traceback
 import re
+import sys
 
 from datetime import timedelta
 from lxml import etree
@@ -92,7 +93,8 @@ class CapaModule(XModule):
         display_due_date_string = self.metadata.get('due', None)
         if display_due_date_string is not None:
             self.display_due_date = dateutil.parser.parse(display_due_date_string)
-            #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date))
+            #log.debug("Parsed " + display_due_date_string +
+            #          " to " + str(self.display_due_date))
         else:
             self.display_due_date = None
 
@@ -100,7 +102,8 @@ class CapaModule(XModule):
         if grace_period_string is not None and self.display_due_date:
             self.grace_period = parse_timedelta(grace_period_string)
             self.close_date = self.display_due_date + self.grace_period
-            #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date))
+            #log.debug("Then parsed " + grace_period_string +
+            #          " to closing date" + str(self.close_date))
         else:
             self.grace_period = None
             self.close_date = self.display_due_date
@@ -139,10 +142,16 @@ class CapaModule(XModule):
         try:
             self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(),
                                       instance_state, seed=seed, system=self.system)
-        except Exception:
-            msg = 'cannot create LoncapaProblem %s' % self.location.url()
-            log.exception(msg)
+        except Exception as err:
+            msg = 'cannot create LoncapaProblem {loc}: {err}'.format(
+                loc=self.location.url(), err=err)
+            # TODO (vshnayder): do modules need error handlers too?
+            # We shouldn't be switching on DEBUG.
             if self.system.DEBUG:
+                log.error(msg)
+                # TODO (vshnayder): This logic should be general, not here--and may
+                # want to preserve the data instead of replacing it.
+                # e.g. in the CMS
                 msg = '<p>%s</p>' % msg.replace('<', '&lt;')
                 msg += '<p><pre>%s</pre></p>' % traceback.format_exc().replace('<', '&lt;')
                 # create a dummy problem with error message instead of failing
@@ -153,7 +162,8 @@ class CapaModule(XModule):
                     problem_text, self.location.html_id(),
                     instance_state, seed=seed, system=self.system)
             else:
-                raise
+                # add extra info and raise
+                raise Exception(msg), None, sys.exc_info()[2]
 
     @property
     def rerandomize(self):
@@ -192,6 +202,7 @@ class CapaModule(XModule):
             try:
                 return Progress(score, total)
             except Exception as err:
+                # TODO (vshnayder): why is this still here? still needed?
                 if self.system.DEBUG:
                     return None
                 raise
@@ -211,6 +222,7 @@ class CapaModule(XModule):
         try:
             html = self.lcp.get_html()
         except Exception, err:
+            # TODO (vshnayder): another switch on DEBUG.
             if self.system.DEBUG:
                 log.exception(err)
                 msg = (
@@ -560,6 +572,7 @@ class CapaDescriptor(RawDescriptor):
             'problems/' + path[8:],
             path[8:],
         ]
+
     @classmethod
     def split_to_file(cls, xml_object):
         '''Problems always written in their own files'''

common/lib/xmodule/xmodule/course_module.py

+from fs.errors import ResourceNotFoundError
 import time
 import dateutil.parser
 import logging
 
+from xmodule.graders import load_grading_policy
 from xmodule.modulestore import Location
 from xmodule.seq_module import SequenceDescriptor, SequenceModule
 
@@ -14,20 +16,57 @@ class CourseDescriptor(SequenceDescriptor):
 
     def __init__(self, system, definition=None, **kwargs):
         super(CourseDescriptor, self).__init__(system, definition, **kwargs)
+        
+        self._grader = None
+        self._grade_cutoffs = None
 
+        msg = None
         try:
             self.start = time.strptime(self.metadata["start"], "%Y-%m-%dT%H:%M")
         except KeyError:
             self.start = time.gmtime(0) #The epoch
-            log.critical("Course loaded without a start date. %s", self.id)
+            msg = "Course loaded without a start date. id = %s" % self.id
+            log.critical(msg)
         except ValueError as e:
             self.start = time.gmtime(0) #The epoch
-            log.critical("Course loaded with a bad start date. %s '%s'",
-                         self.id, e)
+            msg = "Course loaded with a bad start date. %s '%s'" % (self.id, e)
+            log.critical(msg)
+
+        # Don't call the tracker from the exception handler.
+        if msg is not None:
+            system.error_tracker(msg)
+
 
     def has_started(self):
         return time.gmtime() > self.start
-
+    
+    @property
+    def grader(self):
+        self.__load_grading_policy()
+        return self._grader
+    
+    @property
+    def grade_cutoffs(self):
+        self.__load_grading_policy()
+        return self._grade_cutoffs
+    
+    
+    def __load_grading_policy(self):
+        if not self._grader or not self._grade_cutoffs:
+            policy_string = ""
+            
+            try:
+                with self.system.resources_fs.open("grading_policy.json") as grading_policy_file:
+                    policy_string = grading_policy_file.read()
+            except (IOError, ResourceNotFoundError):
+                log.warning("Unable to load course settings file from grading_policy.json in course " + self.id)
+            
+            grading_policy = load_grading_policy(policy_string)
+            
+            self._grader = grading_policy['GRADER']
+            self._grade_cutoffs = grading_policy['GRADE_CUTOFFS']
+    
+    
     @staticmethod
     def id_to_location(course_id):
         '''Convert the given course_id (org/course/name) to a location object.
@@ -72,3 +111,4 @@ class CourseDescriptor(SequenceDescriptor):
     @property
     def org(self):
         return self.location.org
+

common/lib/xmodule/xmodule/css/sequence/display.scss

@@ -2,9 +2,8 @@ nav.sequence-nav {
   // TODO (cpennington): This doesn't work anymore. XModules aren't able to
   // import from external sources.
   @extend .topbar;
-
-  border-bottom: 1px solid darken($cream, 20%);
-  margin-bottom: $body-line-height;
+  border-bottom: 1px solid $border-color;
+  margin: (-(lh())) (-(lh())) lh() (-(lh()));
   position: relative;
   @include border-top-right-radius(4px);
 
@@ -12,6 +11,8 @@ nav.sequence-nav {
     @include box-sizing(border-box);
     display: table;
     height: 100%;
+    margin: 0;
+    padding-left: 3px;
     padding-right: flex-grid(1, 9);
     width: 100%;
 
@@ -20,135 +21,104 @@ nav.sequence-nav {
     }
 
     li {
-      border-left: 1px solid darken($cream, 20%);
       display: table-cell;
       min-width: 20px;
 
-      &:first-child {
-        border-left: none;
-      }
-
-      .inactive {
+      a {
+        background-position: center;
         background-repeat: no-repeat;
+        border: 1px solid transparent;
+        border-bottom: none;
+        @include border-radius(3px 3px 0 0);
+        cursor: pointer;
+        display: block;
+        height: 10px;
+        padding: 15px 0 14px;
+        position: relative;
+        @include transition();
+        width: 100%;
 
         &:hover {
-          background-color: lighten($cream, 3%);
+          background-repeat: no-repeat;
+          background-position: center;
+          background-color: #F6F6F6;
         }
-      }
 
-      .visited {
-        background-color: #DCCDA2;
-        background-repeat: no-repeat;
-        @include box-shadow(inset 0 0 3px darken(#dccda2, 10%));
+        &.visited {
+          background-color: #F6F6F6;
 
-        &:hover {
-          background-color: $cream;
-          background-position: center center;
+          &:hover {
+            background-position: center center;
+          }
         }
-      }
 
-      .active {
-        background-color: #fff;
-        background-repeat: no-repeat;
-        @include box-shadow(0 1px 0 #fff);
-
-        &:hover {
+        &.active {
+          border-color: $border-color;
+          @include box-shadow(0 2px 0 #fff);
           background-color: #fff;
-          background-position: center;
-        }
-      }
+          z-index: 9;
 
-      a {
-        background-position: center center;
-        border: none;
-        cursor: pointer;
-        display: block;
-        height: 17px;
-        padding: 15px 0 14px;
-        position: relative;
-        @include transition(all, .4s, $ease-in-out-quad);
-        width: 100%;
-
-        &.progress {
-          border-bottom-style: solid;
-          border-bottom-width: 4px;
+          &:hover {
+            background-position: center;
+            background-color: #fff;
+          }
         }
 
         &.progress-none {
-          @extend .progress;
-          border-bottom-color: red;
+          background-color: lighten(red, 50%);
         }
 
         &.progress-some {
-          @extend .progress;
-          border-bottom-color: yellow;
+          background-color: yellow;
         }
 
         &.progress-done {
-          @extend .progress;
-          border-bottom-color: green;
+          background-color: green;
         }
 
         //video
         &.seq_video {
           &.inactive {
-            @extend .inactive;
             background-image: url('../images/sequence-nav/video-icon-normal.png');
-            background-position: center;
           }
 
           &.visited {
-            @extend .visited;
             background-image: url('../images/sequence-nav/video-icon-visited.png');
-            background-position: center;
           }
 
           &.active {
             @extend .active;
             background-image: url('../images/sequence-nav/video-icon-current.png');
-            background-position: center;
           }
         }
 
         //other
         &.seq_other {
           &.inactive {
-            @extend .inactive;
             background-image: url('../images/sequence-nav/document-icon-normal.png');
-            background-position: center;
           }
 
           &.visited {
-            @extend .visited;
             background-image: url('../images/sequence-nav/document-icon-visited.png');
-            background-position: center;
           }
 
           &.active {
-            @extend .active;
             background-image: url('../images/sequence-nav/document-icon-current.png');
-            background-position: center;
           }
         }
 
         //vertical & problems
         &.seq_vertical, &.seq_problem {
           &.inactive {
-            @extend .inactive;
             background-image: url('../images/sequence-nav/list-icon-normal.png');
-            background-position: center;
           }
 
           &.visited {
-            @extend .visited;
             background-image: url('../images/sequence-nav/list-icon-visited.png');
-            background-position: center;
           }
 
           &.active {
-            @extend .active;
             background-image: url('../images/sequence-nav/list-icon-current.png');
-            background-position: center;
           }
         }
 
@@ -156,6 +126,7 @@ nav.sequence-nav {
           background: #333;
           color: #fff;
           display: none;
+          font-family: $sans-serif;
           line-height: lh();
           left: 0px;
           opacity: 0;
@@ -206,27 +177,29 @@ nav.sequence-nav {
     right: 0;
     top: 0;
     width: flex-grid(1, 9);
+    border: 1px solid $border-color;
+    border-bottom: 0;
+    @include border-radius(3px 3px 0 0);
 
     li {
       float: left;
+      margin-bottom: 0;
       width: 50%;
 
       &.prev, &.next {
 
         a {
-          background-color: darken($cream, 5%);
-          background-position: center center;
+          background-position: center;
           background-repeat: no-repeat;
-          border-left: 1px solid darken(#f6efd4, 20%);
-          @include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%));
-          @include box-sizing(border-box);
-          cursor: pointer;
           display: block;
+          height: 10px;
+          padding: 15px 0 14px;
           text-indent: -9999px;
           @include transition(all, .2s, $ease-in-out-quad);
 
           &:hover {
             opacity: .5;
+            background-color: #f4f4f4;
           }
 
           &.disabled {
@@ -239,20 +212,13 @@ nav.sequence-nav {
       &.prev {
         a {
           background-image: url('../images/sequence-nav/previous-icon.png');
-
-          &:hover {
-            background-color: $cream;
-          }
         }
       }
 
       &.next {
         a {
+          border-left: 1px solid lighten($border-color, 10%);
           background-image: url('../images/sequence-nav/next-icon.png');
-
-          &:hover {
-            background-color: $cream;
-          }
         }
       }
     }
@@ -273,11 +239,8 @@ nav.sequence-bottom {
 
   ul {
     @extend .clearfix;
-    background-color: darken(#F6EFD4, 5%);
-    background-color: darken($cream, 5%);
-    border: 1px solid darken(#f6efd4, 20%);
+    border: 1px solid $border-color;
     @include border-radius(3px);
-    @include box-shadow(inset 0 0 0 1px lighten(#f6efd4, 5%));
     @include inline-block();
 
     li {
@@ -297,14 +260,13 @@ nav.sequence-bottom {
           width: 45px;
 
           &:hover {
-            background-color: $cream;
-            color: darken($cream, 60%);
+            background-color: #ddd;
+            color: #000;
             opacity: .5;
             text-decoration: none;
           }
 
           &.disabled {
-            background-color: lighten($cream, 10%);
             opacity: .4;
           }
         }
@@ -313,7 +275,7 @@ nav.sequence-bottom {
       &.prev {
         a {
           background-image: url('../images/sequence-nav/previous-icon.png');
-          border-right: 1px solid darken(#f6efd4, 20%);
+          border-right: 1px solid lighten($border-color, 10%);
 
           &:hover {
             background-color: none;

common/lib/xmodule/xmodule/css/video/display.scss

@@ -114,14 +114,13 @@ div.video {
         @extend .dullify;
         float: left;
         list-style: none;
-        margin-right: lh();
+        margin: 0 lh() 0 0;
         padding: 0;
 
         li {
           float: left;
           margin-bottom: 0;
 
-
           a {
             border-bottom: none;
             border-right: 1px solid #000;
@@ -183,6 +182,8 @@ div.video {
             ol.video_speeds {
               display: block;
               opacity: 1;
+              padding: 0;
+              margin: 0;
             }
           }
 
@@ -210,6 +211,7 @@ div.video {
               font-weight: normal;
               letter-spacing: 1px;
               padding: 0 lh(.25) 0 lh(.5);
+              line-height: 46px;
               text-transform: uppercase;
             }
 
@@ -218,6 +220,7 @@ div.video {
               font-weight: bold;
               margin-bottom: 0;
               padding: 0 lh(.5) 0 0;
+              line-height: 46px;
             }
 
             &:hover, &:active, &:focus {
@@ -422,10 +425,12 @@ div.video {
   }
 
   ol.subtitles {
+    padding-left: 0;
     float: left;
     max-height: 460px;
     overflow: auto;
     width: flex-grid(3, 9);
+    margin: 0;
 
     li {
       border: 0;

common/lib/xmodule/xmodule/editing_module.py

+from pkg_resources import resource_string
+from lxml import etree
+from xmodule.mako_module import MakoModuleDescriptor
+import logging
+
+log = logging.getLogger(__name__)
+
+class EditingDescriptor(MakoModuleDescriptor):
+    """
+    Module that provides a raw editing view of its data and children.  It does not
+    perform any validation on its definition---just passes it along to the browser.
+
+    This class is intended to be used as a mixin.
+    """
+    mako_template = "widgets/raw-edit.html"
+
+    js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]}
+    js_module_name = "RawDescriptor"
+
+    def get_context(self):
+        return {
+            'module': self,
+            'data': self.definition.get('data', ''),
+    # TODO (vshnayder): allow children and metadata to be edited.
+    #'children' : self.definition.get('children, ''),
+
+    # TODO: show both own metadata and inherited?
+    #'metadata' : self.own_metadata,
+        }

common/lib/xmodule/xmodule/error_module.py

+import sys
+import logging
+
+from pkg_resources import resource_string
+from lxml import etree
+from xmodule.x_module import XModule
+from xmodule.mako_module import MakoModuleDescriptor
+from xmodule.xml_module import XmlDescriptor
+from xmodule.editing_module import EditingDescriptor
+from xmodule.errortracker import exc_info_to_str
+
+
+log = logging.getLogger(__name__)
+
+class ErrorModule(XModule):
+    def get_html(self):
+        '''Show an error.
+        TODO (vshnayder): proper style, divs, etc.
+        '''
+        # staff get to see all the details
+        return self.system.render_template('module-error.html', {
+            'data' : self.definition['data']['contents'],
+            'error' : self.definition['data']['error_msg'],
+            'is_staff' : self.system.is_staff,
+            })
+
+class ErrorDescriptor(EditingDescriptor):
+    """
+    Module that provides a raw editing view of broken xml.
+    """
+    module_class = ErrorModule
+
+    @classmethod
+    def from_xml(cls, xml_data, system, org=None, course=None,
+                 error_msg='Error not available'):
+        '''Create an instance of this descriptor from the supplied data.
+
+        Does not try to parse the data--just stores it.
+
+        Takes an extra, optional, parameter--the error that caused an
+        issue.  (should be a string, or convert usefully into one).
+        '''
+        # Use a nested inner dictionary because 'data' is hardcoded
+        inner = {}
+        definition = {'data': inner}
+        inner['error_msg'] = str(error_msg)
+
+        try:
+            # If this is already an error tag, don't want to re-wrap it.
+            xml_obj = etree.fromstring(xml_data)
+            if xml_obj.tag == 'error':
+                xml_data = xml_obj.text
+                error_node = xml_obj.find('error_msg')
+                if error_node is not None:
+                    inner['error_msg'] = error_node.text
+                else:
+                    inner['error_msg'] = 'Error not available'
+
+        except etree.XMLSyntaxError:
+            # Save the error to display later--overrides other problems
+            inner['error_msg'] = exc_info_to_str(sys.exc_info())
+
+        inner['contents'] = xml_data
+        # TODO (vshnayder): Do we need a unique slug here?  Just pick a random
+        # 64-bit num?
+        location = ['i4x', org, course, 'error', 'slug']
+        metadata = {}  # stays in the xml_data
+
+        return cls(system, definition, location=location, metadata=metadata)
+
+    def export_to_xml(self, resource_fs):
+        '''
+        If the definition data is invalid xml, export it wrapped in an "error"
+        tag.  If it is valid, export without the wrapper.
+
+        NOTE: There may still be problems with the valid xml--it could be
+        missing required attributes, could have the wrong tags, refer to missing
+        files, etc.  That would just get re-wrapped on import.
+        '''
+        try:
+           xml = etree.fromstring(self.definition['data']['contents'])
+           return etree.tostring(xml)
+        except etree.XMLSyntaxError:
+            # still not valid.
+            root = etree.Element('error')
+            root.text = self.definition['data']['contents']
+            err_node = etree.SubElement(root, 'error_msg')
+            err_node.text = self.definition['data']['error_msg']
+            return etree.tostring(root)

common/lib/xmodule/xmodule/errorhandlers.py

-import logging
-import sys
-
-log = logging.getLogger(__name__)
-
-def in_exception_handler():
-    '''Is there an active exception?'''
-    return sys.exc_info() != (None, None, None)
-
-def strict_error_handler(msg, exc_info=None):
-    '''
-    Do not let errors pass.  If exc_info is not None, ignore msg, and just
-    re-raise.  Otherwise, check if we are in an exception-handling context.
-    If so, re-raise.  Otherwise, raise Exception(msg).
-
-    Meant for use in validation, where any errors should trap.
-    '''
-    if exc_info is not None:
-        raise exc_info[0], exc_info[1], exc_info[2]
-
-    if in_exception_handler():
-        raise
-
-    raise Exception(msg)
-
-
-def logging_error_handler(msg, exc_info=None):
-    '''Log all errors, but otherwise let them pass, relying on the caller to
-    workaround.'''
-    if exc_info is not None:
-        log.exception(msg, exc_info=exc_info)
-        return
-
-    if in_exception_handler():
-        log.exception(msg)
-        return
-
-    log.error(msg)
-
-
-def ignore_errors_handler(msg, exc_info=None):
-    '''Ignore all errors, relying on the caller to workaround.
-    Meant for use in the LMS, where an error in one part of the course
-    shouldn't bring down the whole system'''
-    pass

common/lib/xmodule/xmodule/errortracker.py

+import logging
+import sys
+import traceback
+
+from collections import namedtuple
+
+log = logging.getLogger(__name__)
+
+ErrorLog = namedtuple('ErrorLog', 'tracker errors')
+
+def exc_info_to_str(exc_info):
+    """Given some exception info, convert it into a string using
+    the traceback.format_exception() function.
+    """
+    return ''.join(traceback.format_exception(*exc_info))
+
+def in_exception_handler():
+    '''Is there an active exception?'''
+    return sys.exc_info() != (None, None, None)
+
+
+def make_error_tracker():
+    '''Return an ErrorLog (named tuple), with fields (tracker, errors), where
+    the logger appends a tuple (message, exception_str) to the errors on every
+    call.  exception_str is in the format returned by traceback.format_exception.
+
+    error_list is a simple list.  If the caller modifies it, info
+    will be lost.
+    '''
+    errors = []
+
+    def error_tracker(msg):
+        '''Log errors'''
+        exc_str = ''
+        if in_exception_handler():
+            exc_str = exc_info_to_str(sys.exc_info())
+
+        errors.append((msg, exc_str))
+
+    return ErrorLog(error_tracker, errors)
+
+def null_error_tracker(msg):
+    '''A dummy error tracker that just ignores the messages'''
+    pass

common/lib/xmodule/xmodule/graders.py

 import abc
+import json
 import logging
 
 from collections import namedtuple
@@ -9,6 +10,69 @@ log = logging.getLogger("mitx.courseware")
 # Section either indicates the name of the problem or the name of the section
 Score = namedtuple("Score", "earned possible graded section")
 
+def load_grading_policy(course_policy_string):
+    """
+    This loads a grading policy from a string (usually read from a file),
+    which can be a JSON object or an empty string.
+    
+    The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
+    missing, it reverts to the default.
+    """
+    
+    default_policy_string = """
+    {
+        "GRADER" : [
+            {
+                "type" : "Homework",
+                "min_count" : 12,
+                "drop_count" : 2,
+                "short_label" : "HW",
+                "weight" : 0.15
+            },
+            {
+                "type" : "Lab",
+                "min_count" : 12,
+                "drop_count" : 2,
+                "category" : "Labs",
+                "weight" : 0.15
+            },
+            {
+                "type" : "Midterm",
+                "name" : "Midterm Exam",
+                "short_label" : "Midterm",
+                "weight" : 0.3
+            },
+            {
+                "type" : "Final",
+                "name" : "Final Exam",
+                "short_label" : "Final",
+                "weight" : 0.4
+            }
+        ],
+        "GRADE_CUTOFFS" : {
+            "A" : 0.87,
+            "B" : 0.7,
+            "C" : 0.6
+        }
+    }
+    """
+    
+    # Load the global settings as a dictionary
+    grading_policy = json.loads(default_policy_string)
+
+    # Load the course policies as a dictionary
+    course_policy = {}
+    if course_policy_string:
+        course_policy = json.loads(course_policy_string)
+    
+    # Override any global settings with the course settings
+    grading_policy.update(course_policy)
+
+    # Here is where we should parse any configurations, so that we can fail early
+    grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER'])
+    
+    return grading_policy
+    
 
 def aggregate_scores(scores, section_name="summary"):
     """

common/lib/xmodule/xmodule/html_checker.py

+from lxml import etree
+
+def check_html(html):
+    '''
+    Check whether the passed in html string can be parsed by lxml.
+    Return bool success.
+    '''
+    parser = etree.HTMLParser()
+    try:
+        etree.fromstring(html, parser)
+        return True
+    except Exception as err:
+        pass
+    return False

common/lib/xmodule/xmodule/html_module.py

+import copy
+from fs.errors import ResourceNotFoundError
 import logging
 import os
+import sys
 from lxml import etree
 
-from xmodule.x_module import XModule
-from xmodule.raw_module import RawDescriptor
+from .x_module import XModule
+from .xml_module import XmlDescriptor
+from .editing_module import EditingDescriptor
+from .stringify import stringify_children
+from .html_checker import check_html
 
 log = logging.getLogger("mitx.courseware")
 
-
 class HtmlModule(XModule):
     def get_html(self):
         return self.html
@@ -19,33 +24,118 @@ class HtmlModule(XModule):
         self.html = self.definition['data']
 
 
-class HtmlDescriptor(RawDescriptor):
+class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
     """
     Module for putting raw html in a course
     """
     mako_template = "widgets/html-edit.html"
     module_class = HtmlModule
-    filename_extension = "html"
+    filename_extension = "xml"
 
-    # TODO (cpennington): Delete this method once all fall 2012 course are being
-    # edited in the cms
+    # VS[compat] TODO (cpennington): Delete this method once all fall 2012 course
+    # are being edited in the cms
     @classmethod
     def backcompat_paths(cls, path):
-        if path.endswith('.html.html'):
-            path = path[:-5]
+        origpath = path
+        if path.endswith('.html.xml'):
+            path = path[:-9] + '.html'  #backcompat--look for html instead of xml
         candidates = []
         while os.sep in path:
             candidates.append(path)
             _, _, path = path.partition(os.sep)
 
+        # also look for .html versions instead of .xml
+        if origpath.endswith('.xml'):
+            candidates.append(origpath[:-4] + '.html')
         return candidates
 
+    # NOTE: html descriptors are special.  We do not want to parse and
+    # export them ourselves, because that can break things (e.g. lxml
+    # adds body tags when it exports, but they should just be html
+    # snippets that will be included in the middle of pages.
+
     @classmethod
-    def file_to_xml(cls, file_object):
-        parser = etree.HTMLParser()
-        return etree.parse(file_object, parser).getroot()
+    def load_definition(cls, xml_object, system, location):
+        '''Load a descriptor from the specified xml_object:
+
+        If there is a filename attribute, load it as a string, and
+        log a warning if it is not parseable by etree.HTMLParser.
+
+        If there is not a filename attribute, the definition is the body
+        of the xml_object, without the root tag (do not want <html> in the
+        middle of a page)
+        '''
+        filename = xml_object.get('filename')
+        if filename is None:
+            definition_xml = copy.deepcopy(xml_object)
+            cls.clean_metadata_from_xml(definition_xml)
+            return {'data' : stringify_children(definition_xml)}
+        else:
+            filepath = cls._format_filepath(xml_object.tag, filename)
+
+            # VS[compat]
+            # TODO (cpennington): If the file doesn't exist at the right path,
+            # give the class a chance to fix it up. The file will be written out
+            # again in the correct format.  This should go away once the CMS is
+            # online and has imported all current (fall 2012) courses from xml
+            if not system.resources_fs.exists(filepath):
+                candidates = cls.backcompat_paths(filepath)
+                #log.debug("candidates = {0}".format(candidates))
+                for candidate in candidates:
+                    if system.resources_fs.exists(candidate):
+                        filepath = candidate
+                        break
+
+            try:
+                with system.resources_fs.open(filepath) as file:
+                    html = file.read()
+                    # Log a warning if we can't parse the file, but don't error
+                    if not check_html(html):
+                        msg = "Couldn't parse html in {0}.".format(filepath)
+                        log.warning(msg)
+                        system.error_tracker("Warning: " + msg)
+
+                    definition = {'data' : html}
+
+                    # TODO (ichuang): remove this after migration
+                    # for Fall 2012 LMS migration: keep filename (and unmangled filename)
+                    definition['filename'] = [ filepath, filename ]
+
+                    return definition
+
+            except (ResourceNotFoundError) as err:
+                msg = 'Unable to load file contents at path {0}: {1} '.format(
+                    filepath, err)
+                # add more info and re-raise
+                raise Exception(msg), None, sys.exc_info()[2]
 
     @classmethod
     def split_to_file(cls, xml_object):
-        # never include inline html
+        '''Never include inline html'''
         return True
+
+
+    # TODO (vshnayder): make export put things in the right places.
+
+    def definition_to_xml(self, resource_fs):
+        '''If the contents are valid xml, write them to filename.xml.  Otherwise,
+        write just the <html filename=""> tag to filename.xml, and the html
+        string to filename.html.
+        '''
+        try:
+            return etree.fromstring(self.definition['data'])
+        except etree.XMLSyntaxError:
+            pass
+
+        # Not proper format.  Write html to file, return an empty tag
+        filepath = u'{category}/{name}.html'.format(category=self.category,
+                                                    name=self.url_name)
+
+        resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
+        with resource_fs.open(filepath, 'w') as file:
+            file.write(self.definition['data'])
+
+        elt = etree.Element('html')
+        elt.set("filename", self.url_name)
+        return elt
+

common/lib/xmodule/xmodule/mako_module.py

@@ -2,10 +2,10 @@ from x_module import XModuleDescriptor, DescriptorSystem
 
 
 class MakoDescriptorSystem(DescriptorSystem):
-    def __init__(self, load_item, resources_fs, error_handler,
-                 render_template):
+    def __init__(self, load_item, resources_fs, error_tracker,
+                 render_template, **kwargs):
         super(MakoDescriptorSystem, self).__init__(
-            load_item, resources_fs, error_handler)
+            load_item, resources_fs, error_tracker, **kwargs)
 
         self.render_template = render_template
 

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -3,10 +3,13 @@ This module provides an abstraction for working with XModuleDescriptors
 that are stored in a database an accessible using their Location as an identifier
 """
 
+import logging
 import re
+
 from collections import namedtuple
-from .exceptions import InvalidLocationError
-import logging
+
+from .exceptions import InvalidLocationError, InsufficientSpecificationError
+from xmodule.errortracker import ErrorLog, make_error_tracker
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
@@ -38,15 +41,15 @@ class Location(_LocationBase):
     '''
     __slots__ = ()
 
-    @classmethod
-    def clean(cls, value):
+    @staticmethod
+    def clean(value):
         """
         Return value, made into a form legal for locations
         """
         return re.sub('_+', '_', INVALID_CHARS.sub('_', value))
 
-    @classmethod
-    def is_valid(cls, value):
+    @staticmethod
+    def is_valid(value):
         '''
         Check if the value is a valid location, in any acceptable format.
         '''
@@ -56,6 +59,21 @@ class Location(_LocationBase):
             return False
         return True
 
+    @staticmethod
+    def ensure_fully_specified(location):
+        '''Make sure location is valid, and fully specified.  Raises
+        InvalidLocationError or InsufficientSpecificationError if not.
+
+        returns a Location object corresponding to location.
+        '''
+        loc = Location(location)
+        for key, val in loc.dict().iteritems():
+            if key != 'revision' and val is None:
+                raise InsufficientSpecificationError(location)
+        return loc
+
+
+
     def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None,
                 name=None, revision=None):
         """
@@ -198,6 +216,18 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
+    def get_item_errors(self, location):
+        """
+        Return a list of (msg, exception-or-None) errors that the modulestore
+        encountered when loading the item at location.
+
+        location : something that can be passed to Location
+
+        Raises the same exceptions as get_item if the location isn't found or
+        isn't fully specified.
+        """
+        raise NotImplementedError
+
     def get_items(self, location, depth=0):
         """
         Returns a list of XModuleDescriptor instances for the items
@@ -254,25 +284,47 @@ class ModuleStore(object):
         '''
         raise NotImplementedError
 
-    def path_to_location(self, location, course=None, chapter=None, section=None):
+
+    def get_parent_locations(self, location):
+        '''Find all locations that are the parents of this location.  Needed
+        for path_to_location().
+
+        returns an iterable of things that can be passed to Location.
         '''
-        Try to find a course/chapter/section[/position] path to this location.
+        raise NotImplementedError
 
-        raise ItemNotFoundError if the location doesn't exist.
 
-        If course, chapter, section are not None, restrict search to paths with those
-            components as specified.
-            
-        raise NoPathToItem if the location exists, but isn't accessible via
-        a path that matches the course/chapter/section restrictions.
+class ModuleStoreBase(ModuleStore):
+    '''
+    Implement interface functionality that can be shared.
+    '''
+    def __init__(self):
+        '''
+        Set up the error-tracking logic.
+        '''
+        self._location_errors = {}    # location -> ErrorLog
 
-        In general, a location may be accessible via many paths. This method may
-        return any valid path.
+    def _get_errorlog(self, location):
+        """
+        If we already have an errorlog for this location, return it.  Otherwise,
+        create one.
+        """
+        location = Location(location)
+        if location not in self._location_errors:
+            self._location_errors[location] = make_error_tracker()
+        return self._location_errors[location]
 
-        Return a tuple (course, chapter, section, position).
+    def get_item_errors(self, location):
+        """
+        Return list of errors for this location, if any.  Raise the same
+        errors as get_item if location isn't present.
 
-        If the section a sequence, position should be the position of this location
-        in that sequence.  Otherwise, position should be None.
-        '''
-        raise NotImplementedError
+        NOTE: For now, the only items that track errors are CourseDescriptors in
+        the xml datastore.  This will return an empty list for all other items
+        and datastores.
+        """
+        # check that item is present and raise the promised exceptions if needed
+        self.get_item(location)
 
+        errorlog = self._get_errorlog(location)
+        return errorlog.errors

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -6,14 +6,13 @@ from itertools import repeat
 from path import path
 
 from importlib import import_module
-from xmodule.errorhandlers import strict_error_handler
+from xmodule.errortracker import null_error_tracker
 from xmodule.x_module import XModuleDescriptor
 from xmodule.mako_module import MakoDescriptorSystem
-from xmodule.course_module import CourseDescriptor
 from mitxmako.shortcuts import render_to_string
 
-from . import ModuleStore, Location
-from .exceptions import (ItemNotFoundError, InsufficientSpecificationError,
+from . import ModuleStoreBase, Location
+from .exceptions import (ItemNotFoundError,
                          NoPathToItem, DuplicateItemError)
 
 # TODO (cpennington): This code currently operates under the assumption that
@@ -27,7 +26,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
     from, with a backup of calling to the underlying modulestore for more data
     """
     def __init__(self, modulestore, module_data, default_class, resources_fs,
-                 error_handler, render_template):
+                 error_tracker, render_template):
         """
         modulestore: the module store that can be used to retrieve additional modules
 
@@ -39,13 +38,13 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
 
         resources_fs: a filesystem, as per MakoDescriptorSystem
 
-        error_handler:
+        error_tracker: a function that logs errors for later display to users
 
         render_template: a function for rendering templates, as per
             MakoDescriptorSystem
         """
         super(CachingDescriptorSystem, self).__init__(
-                self.load_item, resources_fs, error_handler, render_template)
+                self.load_item, resources_fs, error_tracker, render_template)
         self.modulestore = modulestore
         self.module_data = module_data
         self.default_class = default_class
@@ -74,13 +73,17 @@ def location_to_query(location):
     return query
 
 
-class MongoModuleStore(ModuleStore):
+class MongoModuleStore(ModuleStoreBase):
     """
     A Mongodb backed ModuleStore
     """
 
     # TODO (cpennington): Enable non-filesystem filestores
-    def __init__(self, host, db, collection, fs_root, port=27017, default_class=None):
+    def __init__(self, host, db, collection, fs_root, port=27017, default_class=None,
+                 error_tracker=null_error_tracker):
+
+        ModuleStoreBase.__init__(self)
+
         self.collection = pymongo.connection.Connection(
             host=host,
             port=port
@@ -91,13 +94,17 @@ class MongoModuleStore(ModuleStore):
 
         # Force mongo to maintain an index over _id.* that is in the same order
         # that is used when querying by a location
-        self.collection.ensure_index(zip(('_id.' + field for field in Location._fields), repeat(1)))
+        self.collection.ensure_index(
+            zip(('_id.' + field for field in Location._fields), repeat(1)))
 
-        # TODO (vshnayder): default arg default_class=None will make this error
-        module_path, _, class_name = default_class.rpartition('.')
-        class_ = getattr(import_module(module_path), class_name)
-        self.default_class = class_
+        if default_class is not None:
+            module_path, _, class_name = default_class.rpartition('.')
+            class_ = getattr(import_module(module_path), class_name)
+            self.default_class = class_
+        else:
+            self.default_class = None
         self.fs_root = path(fs_root)
+        self.error_tracker = error_tracker
 
     def _clean_item_data(self, item):
         """
@@ -149,7 +156,7 @@ class MongoModuleStore(ModuleStore):
             data_cache,
             self.default_class,
             resource_fs,
-            strict_error_handler,
+            self.error_tracker,
             render_to_string,
         )
         return system.load_item(item['location'])
@@ -172,12 +179,17 @@ class MongoModuleStore(ModuleStore):
         return self.get_items(course_filter)
 
     def _find_one(self, location):
-        '''Look for a given location in the collection.
-        If revision isn't specified, returns the latest.'''
-        return self.collection.find_one(
+        '''Look for a given location in the collection.  If revision is not
+        specified, returns the latest.  If the item is not present, raise
+        ItemNotFoundError.
+        '''
+        item = self.collection.find_one(
             location_to_query(location),
             sort=[('revision', pymongo.ASCENDING)],
         )
+        if item is None:
+            raise ItemNotFoundError(location)
+        return item
 
     def get_item(self, location, depth=0):
         """
@@ -197,14 +209,8 @@ class MongoModuleStore(ModuleStore):
             calls to get_children() to cache. None indicates to cache all descendents.
 
         """
-
-        for key, val in Location(location).dict().iteritems():
-            if key != 'revision' and val is None:
-                raise InsufficientSpecificationError(location)
-
+        location = Location.ensure_fully_specified(location)
         item = self._find_one(location)
-        if item is None:
-            raise ItemNotFoundError(location)
         return self._load_items([item], depth)[0]
 
     def get_items(self, location, depth=0):
@@ -282,96 +288,20 @@ class MongoModuleStore(ModuleStore):
         )
 
     def get_parent_locations(self, location):
-        '''Find all locations that are the parents of this location.
-        Mostly intended for use in path_to_location, but exposed for testing
-        and possible other usefulness.
+        '''Find all locations that are the parents of this location.  Needed
+        for path_to_location().
+
+        If there is no data at location in this modulestore, raise
+            ItemNotFoundError.
 
-        returns an iterable of things that can be passed to Location.
+        returns an iterable of things that can be passed to Location.  This may
+        be empty if there are no parents.
         '''
-        location = Location(location)
-        items = self.collection.find({'definition.children': str(location)},
+        location = Location.ensure_fully_specified(location)
+        # Check that it's actually in this modulestore.
+        item = self._find_one(location)
+        # now get the parents
+        items = self.collection.find({'definition.children': location.url()},
                                     {'_id': True})
         return [i['_id'] for i in items]
 
-    def path_to_location(self, location, course_name=None):
-        '''
-        Try to find a course_id/chapter/section[/position] path to this location.
-        The courseware insists that the first level in the course is chapter,
-        but any kind of module can be a "section".
-
-        location: something that can be passed to Location
-        course_name: [optional].  If not None, restrict search to paths
-            in that course.
-
-        raise ItemNotFoundError if the location doesn't exist.
-
-        raise NoPathToItem if the location exists, but isn't accessible via
-        a chapter/section path in the course(s) being searched.
-
-        Return a tuple (course_id, chapter, section, position) suitable for the
-        courseware index view.
-
-        A location may be accessible via many paths. This method may
-        return any valid path.
-
-        If the section is a sequence, position will be the position
-        of this location in that sequence.  Otherwise, position will
-        be None. TODO (vshnayder): Not true yet.
-        '''
-        # Check that location is present at all
-        if self._find_one(location) is None:
-            raise ItemNotFoundError(location)
-
-        def flatten(xs):
-            '''Convert lisp-style (a, (b, (c, ()))) lists into a python list.
-            Not a general flatten function. '''
-            p = []
-            while xs != ():
-                p.append(xs[0])
-                xs = xs[1]
-            return p
-
-        def find_path_to_course(location, course_name=None):
-            '''Find a path up the location graph to a node with the
-            specified category.  If no path exists, return None.  If a
-            path exists, return it as a list with target location
-            first, and the starting location last.
-            '''
-            # Standard DFS
-
-            # To keep track of where we came from, the work queue has
-            # tuples (location, path-so-far).  To avoid lots of
-            # copying, the path-so-far is stored as a lisp-style
-            # list--nested hd::tl tuples, and flattened at the end.
-            queue = [(location, ())]
-            while len(queue) > 0:
-                (loc, path) = queue.pop()  # Takes from the end
-                loc = Location(loc)
-                # print 'Processing loc={0}, path={1}'.format(loc, path)
-                if loc.category == "course":
-                    if course_name is None or course_name == loc.name:
-                        # Found it!
-                        path = (loc, path)
-                        return flatten(path)
-
-                # otherwise, add parent locations at the end
-                newpath = (loc, path)
-                parents = self.get_parent_locations(loc)
-                queue.extend(zip(parents, repeat(newpath)))
-
-            # If we're here, there is no path
-            return None
-
-        path = find_path_to_course(location, course_name)
-        if path is None:
-            raise(NoPathToItem(location))
-
-        n = len(path)
-        course_id = CourseDescriptor.location_to_id(path[0])
-        chapter = path[1].name if n > 1 else None
-        section = path[2].name if n > 2 else None
-
-        # TODO (vshnayder): not handling position at all yet...
-        position = None
-
-        return (course_id, chapter, section, position)

common/lib/xmodule/xmodule/modulestore/search.py

+from itertools import repeat
+
+from xmodule.course_module import CourseDescriptor
+
+from .exceptions import (ItemNotFoundError, NoPathToItem)
+from . import ModuleStore, Location
+
+
+def path_to_location(modulestore, location, course_name=None):
+    '''
+    Try to find a course_id/chapter/section[/position] path to location in
+    modulestore.  The courseware insists that the first level in the course is
+    chapter, but any kind of module can be a "section".
+
+    location: something that can be passed to Location
+    course_name: [optional].  If not None, restrict search to paths
+        in that course.
+
+    raise ItemNotFoundError if the location doesn't exist.
+
+    raise NoPathToItem if the location exists, but isn't accessible via
+    a chapter/section path in the course(s) being searched.
+
+    Return a tuple (course_id, chapter, section, position) suitable for the
+    courseware index view.
+
+    A location may be accessible via many paths. This method may
+    return any valid path.
+
+    If the section is a sequence, position will be the position
+    of this location in that sequence.  Otherwise, position will
+    be None. TODO (vshnayder): Not true yet.
+    '''
+
+    def flatten(xs):
+        '''Convert lisp-style (a, (b, (c, ()))) list into a python list.
+        Not a general flatten function. '''
+        p = []
+        while xs != ():
+            p.append(xs[0])
+            xs = xs[1]
+        return p
+
+    def find_path_to_course(location, course_name=None):
+        '''Find a path up the location graph to a node with the
+        specified category.
+
+        If no path exists, return None.
+
+        If a path exists, return it as a list with target location first, and
+        the starting location last.
+        '''
+        # Standard DFS
+
+        # To keep track of where we came from, the work queue has
+        # tuples (location, path-so-far).  To avoid lots of
+        # copying, the path-so-far is stored as a lisp-style
+        # list--nested hd::tl tuples, and flattened at the end.
+        queue = [(location, ())]
+        while len(queue) > 0:
+            (loc, path) = queue.pop()  # Takes from the end
+            loc = Location(loc)
+
+            # get_parent_locations should raise ItemNotFoundError if location
+            # isn't found so we don't have to do it explicitly.  Call this
+            # first to make sure the location is there (even if it's a course, and
+            # we would otherwise immediately exit).
+            parents = modulestore.get_parent_locations(loc)
+
+            # print 'Processing loc={0}, path={1}'.format(loc, path)
+            if loc.category == "course":
+                if course_name is None or course_name == loc.name:
+                    # Found it!
+                    path = (loc, path)
+                    return flatten(path)
+
+            # otherwise, add parent locations at the end
+            newpath = (loc, path)
+            queue.extend(zip(parents, repeat(newpath)))
+
+        # If we're here, there is no path
+        return None
+
+    path = find_path_to_course(location, course_name)
+    if path is None:
+        raise(NoPathToItem(location))
+
+    n = len(path)
+    course_id = CourseDescriptor.location_to_id(path[0])
+    # pull out the location names
+    chapter = path[1].name if n > 1 else None
+    section = path[2].name if n > 2 else None
+
+    # TODO (vshnayder): not handling position at all yet...
+    position = None
+
+    return (course_id, chapter, section, position)

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -8,6 +8,7 @@ from xmodule.modulestore import Location
 from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
 from xmodule.modulestore.mongo import MongoModuleStore
 from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.search import path_to_location
 
 # from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
 # to   ~/mitx_all/mitx/common/test
@@ -28,7 +29,7 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
 
 
 class TestMongoModuleStore(object):
-
+    '''Tests!'''
     @classmethod
     def setupClass(cls):
         cls.connection = pymongo.connection.Connection(HOST, PORT)
@@ -67,7 +68,7 @@ class TestMongoModuleStore(object):
 
     def test_init(self):
         '''Make sure the db loads, and print all the locations in the db.
-        Call this directly from failing tests to see what's loaded'''
+        Call this directly from failing tests to see what is loaded'''
         ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True}))
 
         pprint([Location(i['_id']).url() for i in ids])
@@ -93,8 +94,6 @@ class TestMongoModuleStore(object):
             self.store.get_item("i4x://edX/toy/video/Welcome"),
             None)
 
-
-
     def test_find_one(self):
         assert_not_equals(
             self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
@@ -117,13 +116,13 @@ class TestMongoModuleStore(object):
              ("edX/toy/2012_Fall", "Overview", "Toy_Videos", None)),
             )
         for location, expected in should_work:
-            assert_equals(self.store.path_to_location(location), expected)
+            assert_equals(path_to_location(self.store, location), expected)
 
         not_found = (
-            "i4x://edX/toy/video/WelcomeX",
+            "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
             )
         for location in not_found:
-            assert_raises(ItemNotFoundError, self.store.path_to_location, location)
+            assert_raises(ItemNotFoundError, path_to_location, self.store, location)
 
         # Since our test files are valid, there shouldn't be any
         # elements with no path to them.  But we can look for them in
@@ -132,5 +131,5 @@ class TestMongoModuleStore(object):
             "i4x://edX/simple/video/Lost_Video",
             )
         for location in no_path:
-            assert_raises(NoPathToItem, self.store.path_to_location, location, "toy")
+            assert_raises(NoPathToItem, path_to_location, self.store, location, "toy")
 

common/lib/xmodule/xmodule/modulestore/xml.py

 import logging
+import os
+import re
+
 from fs.osfs import OSFS
 from importlib import import_module
 from lxml import etree
 from path import path
-from xmodule.errorhandlers import logging_error_handler
+from xmodule.errortracker import ErrorLog, make_error_tracker
 from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
+from xmodule.course_module import CourseDescriptor
 from xmodule.mako_module import MakoDescriptorSystem
 from cStringIO import StringIO
-import os
-import re
 
-from . import ModuleStore, Location
+from . import ModuleStoreBase, Location
 from .exceptions import ItemNotFoundError
 
 etree.set_default_parser(
@@ -19,7 +21,6 @@ etree.set_default_parser(
 
 log = logging.getLogger('mitx.' + __name__)
 
-
 # VS[compat]
 # TODO (cpennington): Remove this once all fall 2012 courses have been imported
 # into the cms from xml
@@ -29,7 +30,7 @@ def clean_out_mako_templating(xml_string):
     return xml_string
 
 class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
-    def __init__(self, xmlstore, org, course, course_dir, error_handler):
+    def __init__(self, xmlstore, org, course, course_dir, error_tracker, **kwargs):
         """
         A class that handles loading from xml.  Does some munging to ensure that
         all elements have unique slugs.
@@ -40,6 +41,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
         self.used_slugs = set()
 
         def process_xml(xml):
+            """Takes an xml string, and returns a XModuleDescriptor created from
+            that xml.
+            """
             try:
                 # VS[compat]
                 # TODO (cpennington): Remove this once all fall 2012 courses
@@ -70,37 +74,36 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 # log.debug('-> slug=%s' % slug)
                 xml_data.set('url_name', slug)
 
-            module = XModuleDescriptor.load_from_xml(
+            descriptor = XModuleDescriptor.load_from_xml(
                 etree.tostring(xml_data), self, org,
                 course, xmlstore.default_class)
 
-            #log.debug('==> importing module location %s' % repr(module.location))
-            module.metadata['data_dir'] = course_dir
+            #log.debug('==> importing descriptor location %s' %
+            #          repr(descriptor.location))
+            descriptor.metadata['data_dir'] = course_dir
 
-            xmlstore.modules[module.location] = module
+            xmlstore.modules[descriptor.location] = descriptor
 
             if xmlstore.eager:
-                module.get_children()
-            return module
+                descriptor.get_children()
+            return descriptor
 
         render_template = lambda: ''
         load_item = xmlstore.get_item
         resources_fs = OSFS(xmlstore.data_dir / course_dir)
 
         MakoDescriptorSystem.__init__(self, load_item, resources_fs,
-                                      error_handler, render_template)
+                                      error_tracker, render_template, **kwargs)
         XMLParsingSystem.__init__(self, load_item, resources_fs,
-                                  error_handler, process_xml)
-
+                                  error_tracker, process_xml, **kwargs)
 
 
-class XMLModuleStore(ModuleStore):
+class XMLModuleStore(ModuleStoreBase):
     """
     An XML backed ModuleStore
     """
     def __init__(self, data_dir, default_class=None, eager=False,
-                 course_dirs=None,
-                 error_handler=logging_error_handler):
+                 course_dirs=None):
         """
         Initialize an XMLModuleStore from data_dir
 
@@ -114,17 +117,13 @@ class XMLModuleStore(ModuleStore):
 
         course_dirs: If specified, the list of course_dirs to load. Otherwise,
             load all course dirs
-
-        error_handler: The error handler used here and in the underlying
-                DescriptorSystem.  By default, raise exceptions for all errors.
-                See the comments in x_module.py:DescriptorSystem
         """
+        ModuleStoreBase.__init__(self)
 
         self.eager = eager
         self.data_dir = path(data_dir)
         self.modules = {}  # location -> XModuleDescriptor
         self.courses = {}  # course_dir -> XModuleDescriptor for the course
-        self.error_handler = error_handler
 
         if default_class is None:
             self.default_class = None
@@ -147,16 +146,32 @@ class XMLModuleStore(ModuleStore):
                            os.path.exists(self.data_dir / d / "course.xml")]
 
         for course_dir in course_dirs:
-            try:
-                course_descriptor = self.load_course(course_dir)
-                self.courses[course_dir] = course_descriptor
-            except:
-                msg = "Failed to load course '%s'" % course_dir
-                log.exception(msg)
-                error_handler(msg)
-
+            self.try_load_course(course_dir)
 
-    def load_course(self, course_dir):
+    def try_load_course(self,course_dir):
+        '''
+        Load a course, keeping track of errors as we go along.
+        '''
+        try:
+            # Special-case code here, since we don't have a location for the
+            # course before it loads.
+            # So, make a tracker to track load-time errors, then put in the right
+            # place after the course loads and we have its location
+            errorlog = make_error_tracker()
+            course_descriptor = self.load_course(course_dir, errorlog.tracker)
+            self.courses[course_dir] = course_descriptor
+            self._location_errors[course_descriptor.location] = errorlog
+        except:
+            msg = "Failed to load course '%s'" % course_dir
+            log.exception(msg)
+
+    def __unicode__(self):
+        '''
+        String representation - for debugging
+        '''
+        return '<XMLModuleStore>data_dir=%s, %d courses, %d modules' % (self.data_dir,len(self.courses),len(self.modules))
+
+    def load_course(self, course_dir, tracker):
         """
         Load a course into this module store
         course_path: Course directory name
@@ -190,13 +205,13 @@ class XMLModuleStore(ModuleStore):
                         ))
                 course = course_dir
 
-            system = ImportSystem(self, org, course, course_dir,
-                                  self.error_handler)
+            system = ImportSystem(self, org, course, course_dir, tracker)
 
             course_descriptor = system.process_xml(etree.tostring(course_data))
             log.debug('========> Done with course import from {0}'.format(course_dir))
             return course_descriptor
 
+
     def get_item(self, location, depth=0):
         """
         Returns an XModuleDescriptor instance for the item at location.
@@ -217,15 +232,19 @@ class XMLModuleStore(ModuleStore):
         except KeyError:
             raise ItemNotFoundError(location)
 
+
     def get_courses(self, depth=0):
         """
-        Returns a list of course descriptors
+        Returns a list of course descriptors.  If there were errors on loading,
+        some of these may be ErrorDescriptors instead.
         """
         return self.courses.values()
 
+
     def create_item(self, location):
         raise NotImplementedError("XMLModuleStores are read-only")
 
+
     def update_item(self, location, data):
         """
         Set the data in the item specified by the location to
@@ -236,6 +255,7 @@ class XMLModuleStore(ModuleStore):
         """
         raise NotImplementedError("XMLModuleStores are read-only")
 
+
     def update_children(self, location, children):
         """
         Set the children for the item specified by the location to
@@ -246,6 +266,7 @@ class XMLModuleStore(ModuleStore):
         """
         raise NotImplementedError("XMLModuleStores are read-only")
 
+
     def update_metadata(self, location, metadata):
         """
         Set the metadata for the item specified by the location to

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -35,6 +35,8 @@ def import_from_xml(store, data_dir, course_dirs=None, eager=True,
             store.update_item(module.location, module.definition['data'])
         if 'children' in module.definition:
             store.update_children(module.location, module.definition['children'])
-        store.update_metadata(module.location, dict(module.metadata))
+        # NOTE: It's important to use own_metadata here to avoid writing
+        # inherited metadata everywhere.
+        store.update_metadata(module.location, dict(module.own_metadata))
 
     return module_store

common/lib/xmodule/xmodule/raw_module.py

-from pkg_resources import resource_string
 from lxml import etree
-from xmodule.mako_module import MakoModuleDescriptor
+from xmodule.editing_module import EditingDescriptor
 from xmodule.xml_module import XmlDescriptor
 import logging
+import sys
 
 log = logging.getLogger(__name__)
 
-
-class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
+class RawDescriptor(XmlDescriptor, EditingDescriptor):
     """
-    Module that provides a raw editing view of its data and children
+    Module that provides a raw editing view of its data and children.  It
+    requires that the definition xml is valid.
     """
-    mako_template = "widgets/raw-edit.html"
-
-    js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]}
-    js_module_name = "RawDescriptor"
-
-    def get_context(self):
-        return {
-            'module': self,
-            'data': self.definition['data'],
-        }
-
     @classmethod
     def definition_from_xml(cls, xml_object, system):
         return {'data': etree.tostring(xml_object)}
@@ -30,13 +19,12 @@ class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
         try:
             return etree.fromstring(self.definition['data'])
         except etree.XMLSyntaxError as err:
+            # Can't recover here, so just add some info and
+            # re-raise
             lines = self.definition['data'].split('\n')
             line, offset = err.position
             msg = ("Unable to create xml for problem {loc}. "
                    "Context: '{context}'".format(
                     context=lines[line - 1][offset - 40:offset + 40],
                     loc=self.location))
-            log.exception(msg)
-            self.system.error_handler(msg)
-            # no workaround possible, so just re-raise
-            raise
+            raise Exception, msg, sys.exc_info()[2]

common/lib/xmodule/xmodule/stringify.py

+from itertools import chain
+from lxml import etree
+
+def stringify_children(node):
+    '''
+    Return all contents of an xml tree, without the outside tags.
+    e.g. if node is parse of
+        "<html a="b" foo="bar">Hi <div>there <span>Bruce</span><b>!</b></div><html>"
+    should return
+        "Hi <div>there <span>Bruce</span><b>!</b></div>"
+
+    fixed from
+    http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
+    '''
+    parts = ([node.text] +
+            list(chain(*([etree.tostring(c), c.tail]
+                         for c in node.getchildren())
+                         )))
+    # filter removes possible Nones in texts and tails
+    return ''.join(filter(None, parts))

common/lib/xmodule/xmodule/template_module.py

@@ -7,16 +7,14 @@ from mako.template import Template
 class CustomTagModule(XModule):
     """
     This module supports tags of the form
-    <customtag option="val" option2="val2">
-        <impl>$tagname</impl>
-    </customtag>
+    <customtag option="val" option2="val2" impl="tagname"/>
 
     In this case, $tagname should refer to a file in data/custom_tags, which contains
     a mako template that uses ${option} and ${option2} for the content.
 
     For instance:
 
-    data/custom_tags/book::
+    data/mycourse/custom_tags/book::
         More information given in <a href="/book/${page}">the text</a>
 
     course.xml::
@@ -34,7 +32,18 @@ class CustomTagModule(XModule):
                          instance_state, shared_state, **kwargs)
 
         xmltree = etree.fromstring(self.definition['data'])
-        template_name = xmltree.attrib['impl']
+        if 'impl' in xmltree.attrib:
+            template_name = xmltree.attrib['impl']
+        else:
+            # VS[compat]  backwards compatibility with old nested customtag structure
+            child_impl = xmltree.find('impl')
+            if child_impl is not None:
+                template_name = child_impl.text
+            else:
+                # TODO (vshnayder): better exception type
+                raise Exception("Could not find impl attribute in customtag {0}"
+                                .format(location))
+
         params = dict(xmltree.items())
         with self.system.filestore.open(
                 'custom_tags/{name}'.format(name=template_name)) as template:

common/lib/xmodule/tests/__init__.py → common/lib/xmodule/xmodule/tests/__init__.py

@@ -31,7 +31,8 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))),
     debug=True,
-    xqueue_callback_url='/'
+    xqueue_callback_url='/',
+    is_staff=False
 )
 
 

common/lib/xmodule/tests/test_export.py → common/lib/xmodule/xmodule/tests/test_export.py

 from xmodule.modulestore.xml import XMLModuleStore
 from nose.tools import assert_equals
+from nose import SkipTest
 from tempfile import mkdtemp
 from fs.osfs import OSFS
 
@@ -26,3 +27,10 @@ def check_export_roundtrip(data_dir):
     for location in initial_import.modules.keys():
         print "Checking", location
         assert_equals(initial_import.modules[location], second_import.modules[location])
+
+
+def test_toy_roundtrip():
+    dir = ""
+    # TODO: add paths and make this run.
+    raise SkipTest()
+    check_export_roundtrip(dir)

common/lib/xmodule/xmodule/tests/test_import.py

+from path import path
+import unittest
+from fs.memoryfs import MemoryFS
+
+from lxml import etree
+
+from xmodule.x_module import XMLParsingSystem, XModuleDescriptor
+from xmodule.errortracker import make_error_tracker
+from xmodule.modulestore import Location
+from xmodule.modulestore.exceptions import ItemNotFoundError
+
+ORG = 'test_org'
+COURSE = 'test_course'
+
+class DummySystem(XMLParsingSystem):
+    def __init__(self):
+
+        self.modules = {}
+        self.resources_fs = MemoryFS()
+        self.errorlog = make_error_tracker()
+
+        def load_item(loc):
+            loc = Location(loc)
+            if loc in self.modules:
+                return self.modules[loc]
+
+            print "modules: "
+            print self.modules
+            raise ItemNotFoundError("Can't find item at loc: {0}".format(loc))
+
+        def process_xml(xml):
+            print "loading {0}".format(xml)
+            descriptor = XModuleDescriptor.load_from_xml(xml, self, ORG, COURSE, None)
+            # Need to save module so we can find it later
+            self.modules[descriptor.location] = descriptor
+
+            # always eager
+            descriptor.get_children()
+            return descriptor
+
+
+        XMLParsingSystem.__init__(self, load_item, self.resources_fs,
+                                  self.errorlog.tracker, process_xml)
+
+    def render_template(self, template, context):
+            raise Exception("Shouldn't be called")
+
+
+
+
+class ImportTestCase(unittest.TestCase):
+    '''Make sure module imports work properly, including for malformed inputs'''
+
+
+    @staticmethod
+    def get_system():
+        '''Get a dummy system'''
+        return DummySystem()
+
+    def test_fallback(self):
+        '''Make sure that malformed xml loads as an ErrorDescriptor.'''
+
+        bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
+
+        system = self.get_system()
+
+        descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course',
+                                                     None)
+
+        self.assertEqual(descriptor.__class__.__name__,
+                         'ErrorDescriptor')
+
+    def test_reimport(self):
+        '''Make sure an already-exported error xml tag loads properly'''
+
+        self.maxDiff = None
+        bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
+        system = self.get_system()
+        descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course',
+                                                     None)
+        resource_fs = None
+        tag_xml = descriptor.export_to_xml(resource_fs)
+        re_import_descriptor = XModuleDescriptor.load_from_xml(tag_xml, system,
+                                                               'org', 'course',
+                                                               None)
+        self.assertEqual(re_import_descriptor.__class__.__name__,
+                         'ErrorDescriptor')
+
+        self.assertEqual(descriptor.definition['data'],
+                         re_import_descriptor.definition['data'])
+
+    def test_fixed_xml_tag(self):
+        """Make sure a tag that's been fixed exports as the original tag type"""
+
+        # create a error tag with valid xml contents
+        root = etree.Element('error')
+        good_xml = '''<sequential display_name="fixed"><video url="hi"/></sequential>'''
+        root.text = good_xml
+
+        xml_str_in = etree.tostring(root)
+
+        # load it
+        system = self.get_system()
+        descriptor = XModuleDescriptor.load_from_xml(xml_str_in, system, 'org', 'course',
+                                                     None)
+        # export it
+        resource_fs = None
+        xml_str_out = descriptor.export_to_xml(resource_fs)
+
+        # Now make sure the exported xml is a sequential
+        xml_out = etree.fromstring(xml_str_out)
+        self.assertEqual(xml_out.tag, 'sequential')
+
+    def test_metadata_inherit(self):
+        """Make sure metadata inherits properly"""
+        system = self.get_system()
+        v = "1 hour"
+        start_xml = '''<course graceperiod="{grace}" url_name="test1" display_name="myseq">
+                      <chapter url="hi" url_name="ch" display_name="CH">
+                       <html url_name="h" display_name="H">Two houses, ...</html></chapter>
+                   </course>'''.format(grace=v)
+        descriptor = XModuleDescriptor.load_from_xml(start_xml, system,
+                                                     'org', 'course')
+
+        print "Errors: {0}".format(system.errorlog.errors)
+        print descriptor, descriptor.metadata
+        self.assertEqual(descriptor.metadata['graceperiod'], v)
+
+        # Check that the child inherits correctly
+        child = descriptor.get_children()[0]
+        self.assertEqual(child.metadata['graceperiod'], v)
+
+        # Now export and see if the chapter tag has a graceperiod attribute
+        resource_fs = MemoryFS()
+        exported_xml = descriptor.export_to_xml(resource_fs)
+        print "Exported xml:", exported_xml
+        root = etree.fromstring(exported_xml)
+        chapter_tag = root[0]
+        self.assertEqual(chapter_tag.tag, 'chapter')
+        self.assertFalse('graceperiod' in chapter_tag.attrib)

common/lib/xmodule/xmodule/tests/test_stringify.py

+from nose.tools import assert_equals
+from lxml import etree
+from xmodule.stringify import stringify_children
+
+def test_stringify():
+    text = 'Hi <div x="foo">there <span>Bruce</span><b>!</b></div>'
+    html = '''<html a="b" foo="bar">{0}</html>'''.format(text)
+    xml = etree.fromstring(html)
+    out = stringify_children(xml)
+    assert_equals(out, text)

common/lib/xmodule/xmodule/video_module.py

@@ -23,11 +23,12 @@ class VideoModule(XModule):
     css = {'scss': [resource_string(__name__, 'css/video/display.scss')]}
     js_module_name = "Video"
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition,
+                 instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition,
+                         instance_state, shared_state, **kwargs)
         xmltree = etree.fromstring(self.definition['data'])
         self.youtube = xmltree.get('youtube')
-        self.name = xmltree.get('name')
         self.position = 0
 
         if instance_state is not None:
@@ -71,7 +72,7 @@ class VideoModule(XModule):
             'streams': self.video_list(),
             'id': self.location.html_id(),
             'position': self.position,
-            'name': self.name,
+            'display_name': self.display_name,
             # TODO (cpennington): This won't work when we move to data that isn't on the filesystem
             'data_dir': self.metadata['data_dir'],
         })

common/lib/xmodule/xmodule/x_module.py

-from lxml import etree
-import pkg_resources
 import logging
+import pkg_resources
+import sys
 
-from xmodule.modulestore import Location
-
+from fs.errors import ResourceNotFoundError
 from functools import partial
+from lxml import etree
+from lxml.etree import XMLSyntaxError
+
+from xmodule.modulestore import Location
+from xmodule.errortracker import exc_info_to_str
 
 log = logging.getLogger('mitx.' + __name__)
 
@@ -187,13 +191,21 @@ class XModule(HTMLSnippet):
         self.instance_state = instance_state
         self.shared_state = shared_state
         self.id = self.location.url()
-        self.name = self.location.name
+        self.url_name = self.location.name
         self.category = self.location.category
         self.metadata = kwargs.get('metadata', {})
         self._loaded_children = None
 
-    def get_name(self):
-        return self.name
+    @property
+    def display_name(self):
+        '''
+        Return a display name for the module: use display_name if defined in
+        metadata, otherwise convert the url name.
+        '''
+        return self.metadata.get('display_name',
+                                 self.url_name.replace('_', ' '))
+    def __unicode__(self):
+        return '<x_module(name=%s, category=%s, id=%s)>' % (self.name, self.category, self.id)
 
     def get_children(self):
         '''
@@ -338,6 +350,8 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
                     module
                 display_name: The name to use for displaying this module to the
                     user
+                url_name: The name to use for this module in urls and other places
+                    where a unique name is needed.
                 format: The format of this module ('Homework', 'Lab', etc)
                 graded (bool): Whether this module is should be graded or not
                 start (string): The date for which this module will be available
@@ -352,13 +366,30 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
         self.metadata = kwargs.get('metadata', {})
         self.definition = definition if definition is not None else {}
         self.location = Location(kwargs.get('location'))
-        self.name = self.location.name
+        self.url_name = self.location.name
         self.category = self.location.category
         self.shared_state_key = kwargs.get('shared_state_key')
 
         self._child_instances = None
         self._inherited_metadata = set()
 
+    @property
+    def display_name(self):
+        '''
+        Return a display name for the module: use display_name if defined in
+        metadata, otherwise convert the url name.
+        '''
+        return self.metadata.get('display_name',
+                                 self.url_name.replace('_', ' '))
+
+    @property
+    def own_metadata(self):
+        """
+        Return the metadata that is not inherited, but was defined on this module.
+        """
+        return dict((k,v) for k,v in self.metadata.items()
+                    if k not in self._inherited_metadata)
+
     def inherit_metadata(self, metadata):
         """
         Updates this module with metadata inherited from a containing module.
@@ -443,16 +474,32 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
         system is an XMLParsingSystem
 
         org and course are optional strings that will be used in the generated
-            modules url identifiers
+            module's url identifiers
         """
-        class_ = XModuleDescriptor.load_class(
-            etree.fromstring(xml_data).tag,
-            default_class
-        )
-        # leave next line, commented out - useful for low-level debugging
-        # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % (
-        #        etree.fromstring(xml_data).tag,class_))
-        return class_.from_xml(xml_data, system, org, course)
+        try:
+            class_ = XModuleDescriptor.load_class(
+                etree.fromstring(xml_data).tag,
+                default_class
+                )
+            # leave next line, commented out - useful for low-level debugging
+            # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % (
+            #        etree.fromstring(xml_data).tag,class_))
+
+            descriptor = class_.from_xml(xml_data, system, org, course)
+        except Exception as err:
+            # Didn't load properly.  Fall back on loading as an error
+            # descriptor.  This should never error due to formatting.
+
+            # Put import here to avoid circular import errors
+            from xmodule.error_module import ErrorDescriptor
+            msg = "Error loading from xml."
+            log.exception(msg)
+            system.error_tracker(msg)
+            err_msg = msg + "\n" + exc_info_to_str(sys.exc_info())
+            descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course,
+                                                  err_msg)
+
+        return descriptor
 
     @classmethod
     def from_xml(cls, xml_data, system, org=None, course=None):
@@ -521,16 +568,19 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
 
 
 class DescriptorSystem(object):
-    def __init__(self, load_item, resources_fs, error_handler):
+    def __init__(self, load_item, resources_fs, error_tracker, **kwargs):
         """
         load_item: Takes a Location and returns an XModuleDescriptor
 
         resources_fs: A Filesystem object that contains all of the
             resources needed for the course
 
-        error_handler: A hook for handling errors in loading the descriptor.
-            Must be a function of (error_msg, exc_info=None).
-            See errorhandlers.py for some simple ones.
+        error_tracker: A hook for tracking errors in loading the descriptor.
+            Used for example to get a list of all non-fatal problems on course
+            load, and display them to the user.
+
+            A function of (error_msg). errortracker.py provides a
+            handy make_error_tracker() function.
 
             Patterns for using the error handler:
                try:
@@ -539,10 +589,8 @@ class DescriptorSystem(object):
                except SomeProblem:
                   msg = 'Grommet {0} is broken'.format(x)
                   log.exception(msg) # don't rely on handler to log
-                  self.system.error_handler(msg)
-                  # if we get here, work around if possible
-                  raise # if no way to work around
-                       OR
+                  self.system.error_tracker(msg)
+                  # work around
                   return 'Oops, couldn't load grommet'
 
                OR, if not in an exception context:
@@ -550,25 +598,27 @@ class DescriptorSystem(object):
                if not check_something(thingy):
                   msg = "thingy {0} is broken".format(thingy)
                   log.critical(msg)
-                  error_handler(msg)
-                  # if we get here, work around
-                  pass   # e.g. if no workaround needed
+                  self.system.error_tracker(msg)
+
+               NOTE: To avoid duplication, do not call the tracker on errors
+               that you're about to re-raise---let the caller track them.
         """
 
         self.load_item = load_item
         self.resources_fs = resources_fs
-        self.error_handler = error_handler
+        self.error_tracker = error_tracker
 
 
 class XMLParsingSystem(DescriptorSystem):
-    def __init__(self, load_item, resources_fs, error_handler, process_xml):
+    def __init__(self, load_item, resources_fs, error_tracker, process_xml, **kwargs):
         """
-        load_item, resources_fs, error_handler: see DescriptorSystem
+        load_item, resources_fs, error_tracker: see DescriptorSystem
 
         process_xml: Takes an xml string, and returns a XModuleDescriptor
             created from that xml
         """
-        DescriptorSystem.__init__(self, load_item, resources_fs, error_handler)
+        DescriptorSystem.__init__(self, load_item, resources_fs, error_tracker,
+                                  **kwargs)
         self.process_xml = process_xml
 
 
@@ -584,10 +634,18 @@ class ModuleSystem(object):
     Note that these functions can be closures over e.g. a django request
     and user, or other environment-specific info.
     '''
-    def __init__(self, ajax_url, track_function,
-                 get_module, render_template, replace_urls,
-                 user=None, filestore=None, debug=False,
-                 xqueue=None):
+    def __init__(self,
+                 ajax_url,
+                 track_function,
+                 get_module,
+                 render_template,
+                 replace_urls,
+                 user=None,
+                 filestore=None,
+                 debug=False,
+                 xqueue_callback_url=None,
+                 xqueue_default_queuename="null",
+                 is_staff=False):
         '''
         Create a closure around the system environment.
 
@@ -613,9 +671,13 @@ class ModuleSystem(object):
         replace_urls - TEMPORARY - A function like static_replace.replace_urls
                          that capa_module can use to fix up the static urls in
                          ajax results.
+
+        is_staff - Is the user making the request a staff user?
+             TODO (vshnayder): this will need to change once we have real user roles.
         '''
         self.ajax_url = ajax_url
-        self.xqueue = xqueue
+        self.xqueue_callback_url = xqueue_callback_url
+        self.xqueue_default_queuename = xqueue_default_queuename
         self.track_function = track_function
         self.filestore = filestore
         self.get_module = get_module
@@ -623,6 +685,7 @@ class ModuleSystem(object):
         self.DEBUG = self.debug = debug
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
+        self.is_staff = is_staff
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''

common/lib/xmodule/xmodule/xml_module.py

-from collections import MutableMapping
 from xmodule.x_module import XModuleDescriptor
 from xmodule.modulestore import Location
 from lxml import etree
@@ -8,74 +7,12 @@ import traceback
 from collections import namedtuple
 from fs.errors import ResourceNotFoundError
 import os
+import sys
 
 log = logging.getLogger(__name__)
 
-# TODO (cpennington): This was implemented in an attempt to improve performance,
-# but the actual improvement wasn't measured (and it was implemented late at night).
-# We should check if it hurts, and whether there's a better way of doing lazy loading
-
-
-class LazyLoadingDict(MutableMapping):
-    """
-    A dictionary object that lazily loads its contents from a provided
-    function on reads (of members that haven't already been set).
-    """
-
-    def __init__(self, loader):
-        '''
-        On the first read from this dictionary, it will call loader() to
-        populate its contents.  loader() must return something dict-like. Any
-        elements set before the first read will be preserved.
-        '''
-        self._contents = {}
-        self._loaded = False
-        self._loader = loader
-        self._deleted = set()
-
-    def __getitem__(self, name):
-        if not (self._loaded or name in self._contents or name in self._deleted):
-            self.load()
-
-        return self._contents[name]
-
-    def __setitem__(self, name, value):
-        self._contents[name] = value
-        self._deleted.discard(name)
-
-    def __delitem__(self, name):
-        del self._contents[name]
-        self._deleted.add(name)
-
-    def __contains__(self, name):
-        self.load()
-        return name in self._contents
-
-    def __len__(self):
-        self.load()
-        return len(self._contents)
-
-    def __iter__(self):
-        self.load()
-        return iter(self._contents)
-
-    def __repr__(self):
-        self.load()
-        return repr(self._contents)
-
-    def load(self):
-        if self._loaded:
-            return
-
-        loaded_contents = self._loader()
-        loaded_contents.update(self._contents)
-        self._contents = loaded_contents
-        self._loaded = True
-
-
 _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata')
 
-
 class AttrMap(_AttrMapBase):
     """
     A class that specifies a metadata_key, and two functions:
@@ -104,6 +41,7 @@ class XmlDescriptor(XModuleDescriptor):
     # to definition_from_xml, and from the xml returned by definition_to_xml
     metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize',
         'start', 'due', 'graded', 'display_name', 'url_name', 'hide_from_toc',
+        'ispublic', 	# if True, then course is listed for all users; see 
         # VS[compat] Remove once unused.
         'name', 'slug')
 
@@ -164,6 +102,52 @@ class XmlDescriptor(XModuleDescriptor):
         return etree.parse(file_object).getroot()
 
     @classmethod
+    def load_definition(cls, xml_object, system, location):
+        '''Load a descriptor definition from the specified xml_object.
+        Subclasses should not need to override this except in special
+        cases (e.g. html module)'''
+
+        filename = xml_object.get('filename')
+        if filename is None:
+            definition_xml = copy.deepcopy(xml_object)
+            filepath = ''
+        else:
+            filepath = cls._format_filepath(xml_object.tag, filename)
+
+            # VS[compat]
+            # TODO (cpennington): If the file doesn't exist at the right path,
+            # give the class a chance to fix it up. The file will be written out
+            # again in the correct format.  This should go away once the CMS is
+            # online and has imported all current (fall 2012) courses from xml
+            if not system.resources_fs.exists(filepath) and hasattr(
+                    cls,
+                    'backcompat_paths'):
+                candidates = cls.backcompat_paths(filepath)
+                for candidate in candidates:
+                    if system.resources_fs.exists(candidate):
+                        filepath = candidate
+                        break
+
+            try:
+                with system.resources_fs.open(filepath) as file:
+                    definition_xml = cls.file_to_xml(file)
+            except Exception:
+                msg = 'Unable to load file contents at path %s for item %s' % (
+                    filepath, location.url())
+                # Add info about where we are, but keep the traceback
+                raise Exception, msg, sys.exc_info()[2]
+
+        cls.clean_metadata_from_xml(definition_xml)
+        definition = cls.definition_from_xml(definition_xml, system)
+
+        # TODO (ichuang): remove this after migration
+        # for Fall 2012 LMS migration: keep filename (and unmangled filename)
+        definition['filename'] = [ filepath, filename ]
+
+        return definition
+
+
+    @classmethod
     def from_xml(cls, xml_data, system, org=None, course=None):
         """
         Creates an instance of this descriptor from the supplied xml_data.
@@ -180,7 +164,7 @@ class XmlDescriptor(XModuleDescriptor):
         slug = xml_object.get('url_name', xml_object.get('slug'))
         location = Location('i4x', org, course, xml_object.tag, slug)
 
-        def metadata_loader():
+        def load_metadata():
             metadata = {}
             for attr in cls.metadata_attributes:
                 val = xml_object.get(attr)
@@ -192,49 +176,15 @@ class XmlDescriptor(XModuleDescriptor):
                     metadata[attr_map.metadata_key] = attr_map.to_metadata(val)
             return metadata
 
-        def definition_loader():
-            filename = xml_object.get('filename')
-            if filename is None:
-                definition_xml = copy.deepcopy(xml_object)
-            else:
-                filepath = cls._format_filepath(xml_object.tag, filename)
-
-                # VS[compat]
-                # TODO (cpennington): If the file doesn't exist at the right path,
-                # give the class a chance to fix it up. The file will be written out again
-                # in the correct format.
-                # This should go away once the CMS is online and has imported all current (fall 2012)
-                # courses from xml
-                if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'):
-                    candidates = cls.backcompat_paths(filepath)
-                    for candidate in candidates:
-                        if system.resources_fs.exists(candidate):
-                            filepath = candidate
-                            break
-
-                try:
-                    with system.resources_fs.open(filepath) as file:
-                        definition_xml = cls.file_to_xml(file)
-                except (ResourceNotFoundError, etree.XMLSyntaxError):
-                    msg = 'Unable to load file contents at path %s for item %s' % (filepath, location.url())
-                    log.exception(msg)
-                    system.error_handler(msg)
-                    # if error_handler didn't reraise, work around problem.
-                    error_elem = etree.Element('error')
-                    message_elem = etree.SubElement(error_elem, 'error_message')
-                    message_elem.text = msg
-                    stack_elem = etree.SubElement(error_elem, 'stack_trace')
-                    stack_elem.text = traceback.format_exc()
-                    return {'data': etree.tostring(error_elem)}
-
-            cls.clean_metadata_from_xml(definition_xml)
-            return cls.definition_from_xml(definition_xml, system)
-
+        definition = cls.load_definition(xml_object, system, location)
+        metadata = load_metadata()
+        # VS[compat] -- just have the url_name lookup once translation is done
+        slug = xml_object.get('url_name', xml_object.get('slug'))
         return cls(
             system,
-            LazyLoadingDict(definition_loader),
+            definition,
             location=location,
-            metadata=LazyLoadingDict(metadata_loader),
+            metadata=metadata,
         )
 
     @classmethod
@@ -282,8 +232,8 @@ class XmlDescriptor(XModuleDescriptor):
 
         # Write it to a file if necessary
         if self.split_to_file(xml_object):
-            # Put this object in it's own file
-            filepath = self.__class__._format_filepath(self.category, self.name)
+            # Put this object in its own file
+            filepath = self.__class__._format_filepath(self.category, self.url_name)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
                 file.write(etree.tostring(xml_object, pretty_print=True))
@@ -296,10 +246,10 @@ class XmlDescriptor(XModuleDescriptor):
             xml_object.tail = ''
 
 
-            xml_object.set('filename', self.name)
+            xml_object.set('filename', self.url_name)
 
         # Add the metadata
-        xml_object.set('url_name', self.name)
+        xml_object.set('url_name', self.url_name)
         for attr in self.metadata_attributes:
             attr_map = self.xml_attribute_map.get(attr, AttrMap(attr))
             metadata_key = attr_map.metadata_key

create-dev-env.sh

@@ -17,8 +17,10 @@ ouch() {
     !! ERROR !!
 
     The last command did not complete successfully, 
-    see $LOG for more details or trying running the
+    For more details or trying running the
     script again with the -v flag.  
+    
+    Output of the script is recorded in $LOG
 
 EOL
     printf '\E[0m'
@@ -36,7 +38,7 @@ usage() {
     Usage: $PROG [-c] [-v] [-h]
     
             -c        compile scipy and numpy
-            -s        do _not_ set --no-site-packages for virtualenv
+            -s        give access to global site-packages for virtualenv 
             -v        set -x + spew
             -h        this
 
@@ -61,28 +63,21 @@ clone_repos() {
     if [[ -d "$BASE/mitx/.git" ]]; then
         output "Pulling mitx"
         cd "$BASE/mitx"
-        git pull >>$LOG
+        git pull 
     else
         output "Cloning mitx"
         if [[ -d "$BASE/mitx" ]]; then
             mv "$BASE/mitx" "${BASE}/mitx.bak.$$"
         fi
-        git clone git@github.com:MITx/mitx.git >>$LOG 
+        git clone git@github.com:MITx/mitx.git
     fi
-    
-    cd "$BASE"
-    if [[ -d "$BASE/askbot-devel/.git" ]]; then
-        output "Pulling askbot-devel"
-        cd "$BASE/askbot-devel"
-        git pull >>$LOG
-    else
-        output "Cloning askbot-devel"
-        if [[ -d "$BASE/askbot-devel" ]]; then
-            mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$"
-        fi
-        git clone git@github.com:MITx/askbot-devel >>$LOG 
+     
+    if [[ ! -d "$BASE/mitx/askbot/.git" ]]; then
+        output "Cloning askbot as a submodule of mitx"
+        cd "$BASE/mitx"
+        git submodule update --init
     fi
-
+    
     # By default, dev environments start with a copy of 6.002x
     cd "$BASE"
     mkdir -p "$BASE/data"
@@ -90,14 +85,14 @@ clone_repos() {
     if [[ -d "$BASE/data/$REPO/.git" ]]; then
         output "Pulling $REPO"
         cd "$BASE/data/$REPO"
-        git pull >>$LOG
+        git pull 
     else
         output "Cloning $REPO"
         if [[ -d "$BASE/data/$REPO" ]]; then
             mv "$BASE/data/$REPO" "${BASE}/data/$REPO.bak.$$"
         fi
 	cd "$BASE/data"
-        git clone git@github.com:MITx/$REPO >>$LOG
+        git clone git@github.com:MITx/$REPO 
     fi
 }
 
@@ -109,8 +104,8 @@ RUBY_VER="1.9.3"
 NUMPY_VER="1.6.2"
 SCIPY_VER="0.10.1"
 BREW_FILE="$BASE/mitx/brew-formulas.txt"
-LOG="/var/tmp/install.log"
-APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript"
+LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
+APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript"
 
 if [[ $EUID -eq 0 ]]; then
     error "This script should not be run using sudo or as the root user"
@@ -163,23 +158,30 @@ cat<<EO
 
   To compile scipy and numpy from source use the -c option
 
-  Most of STDOUT is redirected to /var/tmp/install.log, run
-  $ tail -f /var/tmp/install.log
-  to monitor progress
+  !!! Do not run this script from an existing virtualenv !!!
+  
+  If you are in a ruby/python virtualenv please start a new
+  shell. 
 
 EO
 info
 output "Press return to begin or control-C to abort"
 read dummy
 
-if [[ -f $HOME/.rvmrc ]]; then
-    output "$HOME/.rvmrc alredy exists, not adding $RUBY_DIR"
-else
-    output "Creating $HOME/.rmrc so rvm uses $RUBY_DIR"
+# log all stdout and stderr 
+exec > >(tee $LOG)
+exec 2>&1
+
+if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then
+    if [[ -f $HOME/.rvmrc ]]; then
+        output "Copying existing .rvmrc to .rvmrc.bak"
+        cp $HOME/.rvmrc $HOME/.rvmrc.bak
+    fi
+    output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR"
     echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc
 fi
+
 mkdir -p $BASE
-rm -f $LOG
 case `uname -s` in
     [Ll]inux)
         command -v lsb_release &>/dev/null || {
@@ -201,17 +203,31 @@ case `uname -s` in
         esac
         ;;
     Darwin)
+    
+        if [[ ! -w /usr/local ]]; then
+            cat<<EO
+   
+        You need to be able to write to /usr/local for 
+        the installation of brew and brew packages.
+
+        Either make sure the group you are in (most likely 'staff')
+        can write to that directory or simply execute the following
+        and re-run the script:
+
+        $ sudo chown -R $USER /usr/local
+EO
+
+            exit 1
+
+        fi
+
         command -v brew &>/dev/null || { 
             output "Installing brew"
             /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" 
         } 
         command -v git &>/dev/null || {
             output "Installing git"
-            brew install git >> $LOG
-        }
-        command -v hg &>/dev/null || {
-            output "Installaing mercurial"
-            brew install mercurial >> $LOG
+            brew install git 
         }
 
         clone_repos
@@ -225,20 +241,22 @@ case `uname -s` in
         for pkg in $(cat $BREW_FILE); do
             grep $pkg <(brew list) &>/dev/null || {
                 output "Installing $pkg"
-                brew install $pkg  >>$LOG 
+                brew install $pkg  
             }
         done
         command -v pip &>/dev/null || {
             output "Installing pip"
-            sudo easy_install pip  >>$LOG 
-        }
-        command -v virtualenv &>/dev/null || {
-            output "Installing virtualenv"
-            sudo pip install virtualenv virtualenvwrapper >> $LOG
+            sudo easy_install pip  
         }
+
+        if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then
+            output "Installing virtualenv >1.7"
+            sudo pip install 'virtualenv>1.7' virtualenvwrapper
+        fi
+
         command -v coffee &>/dev/null || {
             output "Installing coffee script"
-            curl http://npmjs.org/install.sh | sh
+            curl --insecure https://npmjs.org/install.sh | sh
             npm install -g coffee-script
         }
         ;;
@@ -253,10 +271,12 @@ curl -sL get.rvm.io | bash -s stable
 source $RUBY_DIR/scripts/rvm
 # skip the intro 
 LESS="-E" rvm install $RUBY_VER
-if [[ -n $systempkgs ]]; then
-    virtualenv "$PYTHON_DIR"
+if [[ $systempkgs ]]; then
+    virtualenv --system-site-packages "$PYTHON_DIR"
 else
-    virtualenv --no-site-packages "$PYTHON_DIR"
+    # default behavior for virtualenv>1.7 is 
+    # --no-site-packages
+    virtualenv  "$PYTHON_DIR"
 fi
 source $PYTHON_DIR/bin/activate
 output "Installing gem bundler"
@@ -277,24 +297,24 @@ if [[ -n $compile ]]; then
     rm -f numpy.tar.gz scipy.tar.gz
     output "Compiling numpy"
     cd "$BASE/numpy-${NUMPY_VER}"
-    python setup.py install >>$LOG  2>&1
+    python setup.py install 
     output "Compiling scipy"
     cd "$BASE/scipy-${SCIPY_VER}"
-    python setup.py install >>$LOG  2>&1
+    python setup.py install 
     cd "$BASE"
     rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER}
 fi
 
-output "Installing askbot requirements"
-pip install -r askbot-devel/askbot_requirements.txt >>$LOG 
-output "Installing askbot-dev requirements"
-pip install -r askbot-devel/askbot_requirements_dev.txt >>$LOG 
 output "Installing MITx pre-requirements"
-pip install -r mitx/pre-requirements.txt >> $LOG
+pip install -r mitx/pre-requirements.txt 
 # Need to be in the mitx dir to get the paths to local modules right
 output "Installing MITx requirements"
 cd mitx
-pip install -r requirements.txt >>$LOG 
+pip install -r requirements.txt 
+output "Installing askbot requirements"
+pip install -r askbot/askbot_requirements.txt 
+pip install -r askbot/askbot_requirements_dev.txt 
+
 
 mkdir "$BASE/log" || true
 mkdir "$BASE/db" || true

lms/askbot/skins/mitx/templates/base.html

@@ -20,7 +20,7 @@
         {% include "widgets/header.html" %} {# Logo, user tool navigation and meta navitation #}
         {# include "widgets/secondary_header.html" #} {# Scope selector, search input and ask button #}
 
-        <section class="main-content">
+        <section class="container">
             {% block body %}
             {% endblock %}
         </section>

lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html

 {% load extra_filters_jinja %}
 <!--<link href="{{"/style/style.css"|media }}" rel="stylesheet" type="text/css" />-->
 {{ 'application' | compressed_css }}
+{{ 'course' | compressed_css }}

lms/askbot/skins/mitx/templates/navigation.jinja.html

-<header class="app" aria-label="Global Navigation">
+<header class="global" aria-label="Global Navigation">
   <nav>
-    <a href="{{ MITX_ROOT_URL }}" class="logo">
-      <img src="/static/images/logo.png" />
-    </a>
-
-    {%if request.user.is_authenticated(): %}
-    <h1>Circuits and Electronics</h1>
-    <ol class="user">
-      <li>
-        <a href="/dashboard" class="user-dashboard">
-          <span class="avatar"><img src="/static/images/profile.jpg" /></span>
-          {{ request.user.username }}
-        </a>
+    <h1 class="logo"><a href="${reverse('root')}"></a></h1>
+    <ol class="left">
+      <li class="primary">
+        <a href="${reverse('courses')}">Find Courses</a>
       </li>
-      <li>
-        <a href="#" class="options">&#9662</a>
-        <ol class="user-options">
-          <li><a href="#">Account Settings</a></li>
-          <li><a href="/logout">Log Out</a></li>
-        </ol>
-      </li>
-    </ol>
-    {%else:%}
-    <ol>
-      <li><a href="/courses">Courses</a></li>
-      <li><a href="#">How It Works</a></li>
     </ol>
+
     <ol class="user">
-      <li><a href="/dashboard">Log In</a></li>
-      <li><a href="#">Sign Up</a></li>
-    </ol>
-    <ol class="secondary">
-      <li>
-        <a href="#">About</a>
-      </li>
-      <li>
-        <a href="#">Jobs</a>
+      <li class="primary">
+        <a href="${reverse('dashboard')}" class="user-link">
+          <span class="avatar"></span>
+          ${user.username}
+        </a>
       </li>
-      <li>
-        <a href="#">faq</a>
+      <li class="primary">
+        <a href="#" class="dropdown">&#9662</a>
+        <ul class="dropdown-menu">
+          ##          <li><a href="#">Account Settings</a></li>
+          <li><a href="${reverse('help_edx')}">Help</a></li>
+          <li><a href="${reverse('logout')}">Log Out</a></li>
+        </ul>
       </li>
     </ol>
-    {%endif %}
   </nav>
 </header>

lms/askbot/skins/mitx/templates/widgets/footer.html

 <!-- template footer.html -->
-  <footer>
-    <!-- Template based on a design from http://www.dotemplate.com/ - Donated $10 (pmitros) so we don't need to include credit. -->
-    <p> Copyright &copy; 2012. MIT. <a href="/t/copyright.html">Some rights reserved.</a>
-</p>
-      <nav>
-      <ul class="social">
-        <li class="linkedin">
-          <a href="http://www.linkedin.com/groups/Friends-Alumni-MITx-4316538">Linked In</a>
-        </li>
-        <li class="twitter">
-          <a href="https://twitter.com/#!/MyMITx">Twitter</a>
-        </li>
-        <li class="facebook">
-          <a href="http://www.facebook.com/pages/MITx/378592442151504">Facebook</a>
-        </li>
-      </ul>
-    <ul>
+<footer>
+  <nav>
+    <section class="top">
+      <section class="primary">
+        <a href="${reverse('root')}" class="logo"></a>
+        <a href="${reverse('courses')}">Find Courses</a>
+        <a href="${reverse('about_edx')}">About</a>
+        <a href="http://edxonline.tumblr.com/">Blog</a>
+        <a href="${reverse('jobs')}">Jobs</a>
+        <a href="${reverse('contact')}">Contact</a>
+      </section>
 
-      <li><a href="/s/help.html">Help</a></li>
-      <li><a href="/logout">Log out</a></li>
-    </ul>
-      </nav>
-  </footer>
+      <section class="social">
+        <a href="http://youtube.com/user/edxonline"><img src="${static.url('images/social/youtube-sharing.png')}" /></a>
+        <a href="https://plus.google.com/108235383044095082735"><img src="${static.url('images/social/google-plus-sharing.png')}" /></a>
+        <a href="http://www.facebook.com/EdxOnline"><img src="${static.url('images/social/facebook-sharing.png')}" /></a>
+        <a href="https://twitter.com/edXOnline"><img src="${static.url('images/social/twitter-sharing.png')}" /></a>
+      </section>
+    </section>
+
+    <section class="bottom">
+      <section class="copyright">
+        <p style="float:left;">&copy; 2012 edX, some rights reserved.</p>
+      </section>
+
+      <section class="secondary">
+        <a href="${reverse('tos')}">Terms of Service</a>
+        <a href="${reverse('privacy_edx')}">Privacy Policy</a>
+        <a href="${reverse('honor')}">Honor Code</a>
+        <a href="${reverse('help_edx')}">Help</a>
+      </section>
+    </section>
+
+  </nav>
+</footer>
 
 <!-- end template footer.html -->

lms/djangoapps/courseware/course_settings.py

-"""
-Course settings module. All settings in the global_settings are
-first applied, and then any settings in the settings.DATA_DIR/course_settings.json
-are applied. A setting must be in ALL_CAPS.
-
-Settings are used by calling
-
-from courseware.course_settings import course_settings
-
-Note that courseware.course_settings.course_settings is not a module -- it's an object. So
-importing individual settings is not possible:
-
-from courseware.course_settings.course_settings import GRADER  # This won't work.
-
-"""
-import json
-import logging
-
-from django.conf import settings
-
-from xmodule import graders
-
-log = logging.getLogger("mitx.courseware")
-
-global_settings_json = """
-{
-    "GRADER" : [
-        {
-            "type" : "Homework",
-            "min_count" : 12,
-            "drop_count" : 2,
-            "short_label" : "HW",
-            "weight" : 0.15
-        },
-        {
-            "type" : "Lab",
-            "min_count" : 12,
-            "drop_count" : 2,
-            "category" : "Labs",
-            "weight" : 0.15
-        },
-        {
-            "type" : "Midterm",
-            "name" : "Midterm Exam",
-            "short_label" : "Midterm",
-            "weight" : 0.3
-        },
-        {
-            "type" : "Final",
-            "name" : "Final Exam",
-            "short_label" : "Final",
-            "weight" : 0.4
-        }
-    ],
-    "GRADE_CUTOFFS" : {
-        "A" : 0.87,
-        "B" : 0.7,
-        "C" : 0.6
-    }
-}
-"""
-
-
-class Settings(object):
-    def __init__(self):
-
-        # Load the global settings as a dictionary
-        global_settings = json.loads(global_settings_json)
-
-        # Load the course settings as a dictionary
-        course_settings = {}
-        try:
-            # TODO: this doesn't work with multicourse
-            with open(settings.DATA_DIR + "/course_settings.json") as course_settings_file:
-                course_settings_string = course_settings_file.read()
-            course_settings = json.loads(course_settings_string)
-        except IOError:
-            log.warning("Unable to load course settings file from " + str(settings.DATA_DIR) + "/course_settings.json")
-
-        # Override any global settings with the course settings
-        global_settings.update(course_settings)
-
-        # Now, set the properties from the course settings on ourselves
-        for setting in global_settings:
-            setting_value = global_settings[setting]
-            setattr(self, setting, setting_value)
-
-        # Here is where we should parse any configurations, so that we can fail early
-        self.GRADER = graders.grader_from_conf(self.GRADER)
-
-course_settings = Settings()

lms/djangoapps/courseware/courses.py

+from collections import defaultdict
 from fs.errors import ResourceNotFoundError
 from functools import wraps
 import logging
@@ -33,6 +34,7 @@ def check_course(course_id, course_must_be_open=True, course_required=True):
         try:
             course_loc = CourseDescriptor.id_to_location(course_id)
             course = modulestore().get_item(course_loc)
+
         except (KeyError, ItemNotFoundError):
             raise Http404("Course not found.")
 
@@ -82,7 +84,7 @@ def get_course_about_section(course, section_key):
             log.warning("Missing about section {key} in course {url}".format(key=section_key, url=course.location.url()))
             return None
     elif section_key == "title":
-        return course.metadata.get('display_name', course.name)
+        return course.metadata.get('display_name', course.url_name)
     elif section_key == "university":
         return course.location.org
     elif section_key == "number":
@@ -113,3 +115,57 @@ def get_course_info_section(course, section_key):
             return "! Info section missing !"
 
     raise KeyError("Invalid about key " + str(section_key))
+
+def course_staff_group_name(course):
+    '''
+    course should be either a CourseDescriptor instance, or a string (the .course entry of a Location)
+    '''
+    if isinstance(course,str):
+        coursename = course
+    else:
+        coursename = course.metadata.get('data_dir','UnknownCourseName')
+    if not coursename:					# Fall 2012: not all course.xml have metadata correct yet
+        coursename = course.metadata.get('course','')
+    return 'staff_%s' % coursename
+
+def has_staff_access_to_course(user,course):
+    '''
+    Returns True if the given user has staff access to the course.
+    This means that user is in the staff_* group, or is an overall admin.
+    '''
+    if user is None or (not user.is_authenticated()) or course is None:
+        return False
+    if user.is_staff:
+        return True
+    user_groups = [x[1] for x in user.groups.values_list()]	# note this is the Auth group, not UserTestGroup
+    staff_group = course_staff_group_name(course)
+    log.debug('course %s user %s groups %s' % (staff_group, user, user_groups))
+    if staff_group in user_groups:
+        return True
+    return False
+
+def has_access_to_course(user,course):
+    if course.metadata.get('ispublic'):
+        return True
+    return has_staff_access_to_course(user,course)
+
+def get_courses_by_university(user):
+    '''
+    Returns dict of lists of courses available, keyed by course.org (ie university).
+    Courses are sorted by course.number.
+
+    if ACCESS_REQUIRE_STAFF_FOR_COURSE then list only includes those accessible to user.
+    '''
+    # TODO: Clean up how 'error' is done.
+    # filter out any courses that errored.
+    courses = [c for c in modulestore().get_courses()
+               if isinstance(c, CourseDescriptor)]
+    courses = sorted(courses, key=lambda course: course.number)
+    universities = defaultdict(list)
+    for course in courses:
+        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+            if not has_access_to_course(user,course):
+                continue
+        universities[course.org].append(course)
+    return universities
+    

lms/djangoapps/courseware/grades.py

@@ -3,7 +3,6 @@ import logging
 
 from django.conf import settings
 
-from courseware.course_settings import course_settings
 from xmodule import graders
 from xmodule.graders import Score
 from models import StudentModule
@@ -11,20 +10,25 @@ from models import StudentModule
 _log = logging.getLogger("mitx.courseware")
 
 
-def grade_sheet(student, course, student_module_cache):
+def grade_sheet(student, course, grader, student_module_cache):
     """
-    This pulls a summary of all problems in the course. It returns a dictionary with two datastructures:
+    This pulls a summary of all problems in the course. It returns a dictionary
+    with two datastructures:
 
-    - courseware_summary is a summary of all sections with problems in the course. It is organized as an array of chapters,
-    each containing an array of sections, each containing an array of scores. This contains information for graded and ungraded
-    problems, and is good for displaying a course summary with due dates, etc.
+    - courseware_summary is a summary of all sections with problems in the
+    course. It is organized as an array of chapters, each containing an array of
+    sections, each containing an array of scores. This contains information for
+    graded and ungraded problems, and is good for displaying a course summary
+    with due dates, etc.
 
-    - grade_summary is the output from the course grader. More information on the format is in the docstring for CourseGrader.
+    - grade_summary is the output from the course grader. More information on
+      the format is in the docstring for CourseGrader.
 
     Arguments:
         student: A User object for the student to grade
         course: An XModule containing the course to grade
-        student_module_cache: A StudentModuleCache initialized with all instance_modules for the student
+        student_module_cache: A StudentModuleCache initialized with all
+             instance_modules for the student
     """
     totaled_scores = {}
     chapters = []
@@ -52,12 +56,16 @@ def grade_sheet(student, course, student_module_cache):
                         correct = total
 
                 if not total > 0:
-                    #We simply cannot grade a problem that is 12/0, because we might need it as a percentage
+                    #We simply cannot grade a problem that is 12/0, because we
+                    #might need it as a percentage
                     graded = False
 
-                scores.append(Score(correct, total, graded, module.metadata.get('display_name')))
+                scores.append(Score(correct, total, graded,
+                                    module.metadata.get('display_name')))
+
+            section_total, graded_total = graders.aggregate_scores(
+                scores, s.metadata.get('display_name'))
 
-            section_total, graded_total = graders.aggregate_scores(scores, s.metadata.get('display_name'))
             #Add the graded total to totaled_scores
             format = s.metadata.get('format', "")
             if format and graded_total.possible > 0:
@@ -66,7 +74,8 @@ def grade_sheet(student, course, student_module_cache):
                 totaled_scores[format] = format_scores
 
             sections.append({
-                'section': s.metadata.get('display_name'),
+                'display_name': s.display_name,
+                'url_name': s.url_name,
                 'scores': scores,
                 'section_total': section_total,
                 'format': format,
@@ -74,11 +83,11 @@ def grade_sheet(student, course, student_module_cache):
                 'graded': graded,
             })
 
-        chapters.append({'course': course.metadata.get('display_name'),
-                         'chapter': c.metadata.get('display_name'),
+        chapters.append({'course': course.display_name,
+                         'display_name': c.display_name,
+                         'url_name': c.url_name,
                          'sections': sections})
 
-    grader = course_settings.GRADER
     grade_summary = grader.grade(totaled_scores)
 
     return {'courseware_summary': chapters,

lms/djangoapps/courseware/management/commands/clean_xml.py

@@ -10,37 +10,17 @@ from lxml import etree
 from django.core.management.base import BaseCommand
 
 from xmodule.modulestore.xml import XMLModuleStore
-
+from xmodule.errortracker import make_error_tracker
 
 def traverse_tree(course):
     '''Load every descriptor in course.  Return bool success value.'''
     queue = [course]
     while len(queue) > 0:
         node = queue.pop()
-#        print '{0}:'.format(node.location)
-#        if 'data' in node.definition:
-#            print '{0}'.format(node.definition['data'])
         queue.extend(node.get_children())
 
     return True
 
-def make_logging_error_handler():
-    '''Return a tuple (handler, error_list), where
-    the handler appends the message and any exc_info
-    to the error_list on every call.
-    '''
-    errors = []
-
-    def error_handler(msg, exc_info=None):
-        '''Log errors'''
-        if exc_info is None:
-            if sys.exc_info() != (None, None, None):
-                exc_info = sys.exc_info()
-
-        errors.append((msg, exc_info))
-
-    return (error_handler, errors)
-
 
 def export(course, export_dir):
     """Export the specified course to course_dir.  Creates dir if it doesn't exist.
@@ -73,32 +53,18 @@ def import_with_checks(course_dir, verbose=True):
     data_dir = course_dir.dirname()
     course_dirs = [course_dir.basename()]
 
-    (error_handler, errors) = make_logging_error_handler()
     # No default class--want to complain if it doesn't find plugins for any
     # module.
     modulestore = XMLModuleStore(data_dir,
                    default_class=None,
                    eager=True,
-                   course_dirs=course_dirs,
-                   error_handler=error_handler)
+                   course_dirs=course_dirs)
 
     def str_of_err(tpl):
-        (msg, exc_info) = tpl
-        if exc_info is None:
-            return msg
-
-        exc_str = '\n'.join(traceback.format_exception(*exc_info))
+        (msg, exc_str) = tpl
         return '{msg}\n{exc}'.format(msg=msg, exc=exc_str)
 
     courses = modulestore.get_courses()
-    if len(errors) != 0:
-        all_ok = False
-        print '\n'
-        print "=" * 40
-        print 'ERRORs during import:'
-        print '\n'.join(map(str_of_err,errors))
-        print "=" * 40
-        print '\n'
 
     n = len(courses)
     if n != 1:
@@ -107,6 +73,16 @@ def import_with_checks(course_dir, verbose=True):
         return (False, None)
 
     course = courses[0]
+    errors = modulestore.get_item_errors(course.location)
+    if len(errors) != 0:
+        all_ok = False
+        print '\n'
+        print "=" * 40
+        print 'ERRORs during import:'
+        print '\n'.join(map(str_of_err, errors))
+        print "=" * 40
+        print '\n'
+
 
     #print course
     validators = (
@@ -143,6 +119,7 @@ def check_roundtrip(course_dir):
     # dircmp doesn't do recursive diffs.
     # diff = dircmp(course_dir, export_dir, ignore=[], hide=[])
     print "======== Roundtrip diff: ========="
+    sys.stdout.flush()  # needed to make diff appear in the right place
     os.system("diff -r {0} {1}".format(course_dir, export_dir))
     print "======== ideally there is no diff above this ======="
 

lms/djangoapps/courseware/module_render.py

@@ -16,6 +16,8 @@ from xmodule.exceptions import NotFoundError
 from xmodule.x_module import ModuleSystem
 from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule
 
+from courseware.courses import has_staff_access_to_course
+
 log = logging.getLogger("mitx.courseware")
 
 
@@ -36,10 +38,12 @@ def toc_for_course(user, request, course, active_chapter, active_section):
     Create a table of contents from the module store
 
     Return format:
-    [ {'name': name, 'sections': SECTIONS, 'active': bool}, ... ]
+    [ {'display_name': name, 'url_name': url_name,
+       'sections': SECTIONS, 'active': bool}, ... ]
 
     where SECTIONS is a list
-    [ {'name': name, 'format': format, 'due': due, 'active' : bool}, ...]
+    [ {'display_name': name, 'url_name': url_name,
+       'format': format, 'due': due, 'active' : bool}, ...]
 
     active is set for the section and chapter corresponding to the passed
     parameters.  Everything else comes from the xml, or defaults to "".
@@ -55,19 +59,21 @@ def toc_for_course(user, request, course, active_chapter, active_section):
         sections = list()
         for section in chapter.get_display_items():
 
-            active = (chapter.metadata.get('display_name') == active_chapter and
-                      section.metadata.get('display_name') == active_section)
+            active = (chapter.display_name == active_chapter and
+                      section.display_name == active_section)
             hide_from_toc = section.metadata.get('hide_from_toc', 'false').lower() == 'true'
 
             if not hide_from_toc:
-                sections.append({'name': section.metadata.get('display_name'),
+                sections.append({'display_name': section.display_name,
+                                 'url_name': section.url_name,
                                  'format': section.metadata.get('format', ''),
                                  'due': section.metadata.get('due', ''),
                                  'active': active})
 
-        chapters.append({'name': chapter.metadata.get('display_name'),
+        chapters.append({'display_name': chapter.display_name,
+                         'url_name': chapter.url_name,
                          'sections': sections,
-                         'active': chapter.metadata.get('display_name') == active_chapter})
+                         'active': chapter.display_name == active_chapter})
     return chapters
 
 
@@ -77,8 +83,8 @@ def get_section(course_module, chapter, section):
     or None if this doesn't specify a valid section
 
     course: Course url
-    chapter: Chapter name
-    section: Section name
+    chapter: Chapter url_name
+    section: Section url_name
     """
 
     if course_module is None:
@@ -86,7 +92,7 @@ def get_section(course_module, chapter, section):
 
     chapter_module = None
     for _chapter in course_module.get_children():
-        if _chapter.metadata.get('display_name') == chapter:
+        if _chapter.url_name == chapter:
             chapter_module = _chapter
             break
 
@@ -95,7 +101,7 @@ def get_section(course_module, chapter, section):
 
     section_module = None
     for _section in chapter_module.get_children():
-        if _section.metadata.get('display_name') == section:
+        if _section.url_name == section:
             section_module = _section
             break
 
@@ -142,12 +148,12 @@ def get_module(user, request, location, student_module_cache, position=None):
     # Setup system context for module instance
     ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/'
 
-    # Fully qualified callback URL for external queueing system 
-    xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL + 
-                          'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' + 
+    # Fully qualified callback URL for external queueing system
+    xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL +
+                          'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' +
                           'score_update')
 
-    # Default queuename is course-specific and is derived from the course that 
+    # Default queuename is course-specific and is derived from the course that
     #   contains the current module.
     # TODO: Queuename should be derived from 'course_settings.json' of each course
     xqueue_default_queuename = descriptor.location.org + '-' + descriptor.location.course
@@ -176,6 +182,7 @@ def get_module(user, request, location, student_module_cache, position=None):
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
+                          is_staff=user.is_staff,
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
@@ -187,8 +194,9 @@ def get_module(user, request, location, student_module_cache, position=None):
         module.metadata['data_dir']
     )
 
-    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
-        module.get_html = add_histogram(module.get_html)
+    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF'):
+        if has_staff_access_to_course(user, module.location.course):
+            module.get_html = add_histogram(module.get_html, module)
 
     # If StudentModule for this instance wasn't already in the database,
     # and this isn't a guest user, create it.

lms/djangoapps/courseware/tests/tests.py

 import copy
 import json
+from path import path
 import os
 
 from pprint import pprint
+from nose import SkipTest
 
 from django.test import TestCase
 from django.test.client import Client
-from mock import patch, Mock
-from override_settings import override_settings
 from django.conf import settings
 from django.core.urlresolvers import reverse
-from path import path
+from mock import patch, Mock
+from override_settings import override_settings
 
-from student.models import Registration
 from django.contrib.auth.models import User
+from student.models import Registration
 
 from xmodule.modulestore.django import modulestore
 import xmodule.modulestore.django
@@ -189,11 +190,12 @@ class RealCoursesLoadTestCase(PageLoader):
         xmodule.modulestore.django._MODULESTORES = {}
         xmodule.modulestore.django.modulestore().collection.drop()
 
-    # TODO: Disabled test for now..  Fix once things are cleaned up.
-    def Xtest_real_courses_loads(self):
+    def test_real_courses_loads(self):
         '''See if any real courses are available at the REAL_DATA_DIR.
         If they are, check them.'''
 
+        # TODO: Disabled test for now..  Fix once things are cleaned up.
+        raise SkipTest
         # TODO: adjust staticfiles_dirs
         if not os.path.isdir(REAL_DATA_DIR):
             # No data present.  Just pass.

lms/djangoapps/courseware/views.py

-from collections import defaultdict
 import json
 import logging
 import urllib
@@ -19,8 +18,8 @@ from django.views.decorators.cache import cache_control
 from module_render import toc_for_course, get_module, get_section
 from models import StudentModuleCache
 from student.models import UserProfile
-from multicourse import multicourse_settings
 from xmodule.modulestore import Location
+from xmodule.modulestore.search import path_to_location
 from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
 from xmodule.modulestore.django import modulestore
 from xmodule.course_module import CourseDescriptor
@@ -28,7 +27,7 @@ from xmodule.course_module import CourseDescriptor
 from util.cache import cache, cache_if_anonymous
 from student.models import UserTestGroup, CourseEnrollment
 from courseware import grades
-from courseware.courses import check_course
+from courseware.courses import check_course, get_courses_by_university
 
 
 log = logging.getLogger("mitx.courseware")
@@ -54,22 +53,16 @@ def user_groups(user):
     return group_names
 
 
-def format_url_params(params):
-    return [urllib.quote(string.replace(' ', '_')) for string in params]
-
 
 @ensure_csrf_cookie
 @cache_if_anonymous
 def courses(request):
-    # TODO: Clean up how 'error' is done.
-    courses = sorted(modulestore().get_courses(), key=lambda course: course.number)
-    universities = defaultdict(list)
-    for course in courses:
-        universities[course.org].append(course)
-
+    '''
+    Render "find courses" page.  The course selection work is done in courseware.courses.
+    '''
+    universities = get_courses_by_university(request.user)
     return render_to_response("courses.html", {'universities': universities})
 
-
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def gradebook(request, course_id):
     if 'course_admin' not in user_groups(request.user):
@@ -110,7 +103,7 @@ def profile(request, course_id, student_id=None):
     user_info = UserProfile.objects.get(user=student)
 
     student_module_cache = StudentModuleCache(request.user, course)
-    course, _, _, _ = get_module(request.user, request, course.location, student_module_cache)
+    course_module, _, _, _ = get_module(request.user, request, course.location, student_module_cache)
 
     context = {'name': user_info.name,
                'username': student.username,
@@ -118,10 +111,9 @@ def profile(request, course_id, student_id=None):
                'language': user_info.language,
                'email': student.email,
                'course': course,
-               'format_url_params': format_url_params,
                'csrf': csrf(request)['csrf_token']
                }
-    context.update(grades.grade_sheet(student, course, student_module_cache))
+    context.update(grades.grade_sheet(student, course_module, course.grader, student_module_cache))
 
     return render_to_response('profile.html', context)
 
@@ -132,9 +124,9 @@ def render_accordion(request, course, chapter, section):
 
         If chapter and section are '' or None, renders a default accordion.
 
-        Returns (initialization_javascript, content)'''
+        Returns the html string'''
 
-    # TODO (cpennington): do the right thing with courses
+    # grab the table of contents
     toc = toc_for_course(request.user, request, course, chapter, section)
 
     active_chapter = 1
@@ -146,11 +138,11 @@ def render_accordion(request, course, chapter, section):
                     ('toc', toc),
                     ('course_name', course.title),
                     ('course_id', course.id),
-                    ('format_url_params', format_url_params),
                     ('csrf', csrf(request)['csrf_token'])] + template_imports.items())
     return render_to_string('accordion.html', context)
 
 
+@login_required
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def index(request, course_id, chapter=None, section=None,
@@ -163,9 +155,9 @@ def index(request, course_id, chapter=None, section=None,
     Arguments:
 
      - request    : HTTP request
-     - course     : coursename (str)
-     - chapter    : chapter name (str)
-     - section    : section name (str)
+     - course_id  : course id (str: ORG/course/URL_NAME)
+     - chapter    : chapter url_name (str)
+     - section    : section url_name (str)
      - position   : position in module, eg of <sequential> module (str)
 
     Returns:
@@ -173,50 +165,63 @@ def index(request, course_id, chapter=None, section=None,
      - HTTPresponse
     '''
     course = check_course(course_id)
+    registered = registered_for_course(course, request.user)
+    if not registered:
+        log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url()))
+        return redirect(reverse('about_course', args=[course.id]))
 
-    def clean(s):
-        ''' Fixes URLs -- we convert spaces to _ in URLs to prevent
-        funny encoding characters and keep the URLs readable.  This undoes
-        that transformation.
-        '''
-        return s.replace('_', ' ') if s is not None else None
-
-    chapter = clean(chapter)
-    section = clean(section)
-
-    if settings.ENABLE_MULTICOURSE:
-        settings.MODULESTORE['default']['OPTIONS']['data_dir'] = settings.DATA_DIR + multicourse_settings.get_course_xmlpath(course)
-
-    context = {
-        'csrf': csrf(request)['csrf_token'],
-        'accordion': render_accordion(request, course, chapter, section),
-        'COURSE_TITLE': course.title,
-        'course': course,
-        'init': '',
-        'content': ''
-    }
-
-    look_for_module = chapter is not None and section is not None
-    if look_for_module:
-        # TODO (cpennington): Pass the right course in here
-
-        section_descriptor = get_section(course, chapter, section)
-        if section_descriptor is not None:
-            student_module_cache = StudentModuleCache(request.user,
-                                                      section_descriptor)
-            module, _, _, _ = get_module(request.user, request,
-                                         section_descriptor.location,
-                                         student_module_cache)
-            context['content'] = module.get_html()
+    try:
+        context = {
+            'csrf': csrf(request)['csrf_token'],
+            'accordion': render_accordion(request, course, chapter, section),
+            'COURSE_TITLE': course.title,
+            'course': course,
+            'init': '',
+            'content': ''
+            }
+
+        look_for_module = chapter is not None and section is not None
+        if look_for_module:
+            section_descriptor = get_section(course, chapter, section)
+            if section_descriptor is not None:
+                student_module_cache = StudentModuleCache(request.user,
+                                                          section_descriptor)
+                module, _, _, _ = get_module(request.user, request,
+                                             section_descriptor.location,
+                                             student_module_cache)
+                context['content'] = module.get_html()
+            else:
+                log.warning("Couldn't find a section descriptor for course_id '{0}',"
+                            "chapter '{1}', section '{2}'".format(
+                                course_id, chapter, section))
         else:
-            log.warning("Couldn't find a section descriptor for course_id '{0}',"
-                        "chapter '{1}', section '{2}'".format(
-                        course_id, chapter, section))
-
+            if request.user.is_staff:
+                # Add a list of all the errors...
+                context['course_errors'] = modulestore().get_item_errors(course.location)
+
+        result = render_to_response('courseware.html', context)
+    except:
+        # In production, don't want to let a 500 out for any reason
+        if settings.DEBUG:
+            raise
+        else:
+            log.exception("Error in index view: user={user}, course={course},"
+                          " chapter={chapter} section={section}"
+                          "position={position}".format(
+                              user=request.user,
+                              course=course,
+                              chapter=chapter,
+                              section=section,
+                              position=position
+                              ))
+            try:
+                result = render_to_response('courseware-error.html', {})
+            except:
+                result = HttpResponse("There was an unrecoverable error")
 
-    result = render_to_response('courseware.html', context)
     return result
 
+
 @ensure_csrf_cookie
 def jump_to(request, location):
     '''
@@ -237,13 +242,13 @@ def jump_to(request, location):
 
     # Complain if there's not data for this location
     try:
-        (course_id, chapter, section, position) = modulestore().path_to_location(location)
+        (course_id, chapter, section, position) = path_to_location(modulestore(), location)
     except ItemNotFoundError:
         raise Http404("No data at this location: {0}".format(location))
     except NoPathToItem:
         raise Http404("This location is not in any class: {0}".format(location))
 
-
+    # Rely on index to do all error handling
     return index(request, course_id, chapter, section, position)
 
 @ensure_csrf_cookie
@@ -258,14 +263,18 @@ def course_info(request, course_id):
     return render_to_response('info.html', {'course': course})
 
 
+def registered_for_course(course, user):
+    '''Return CourseEnrollment if user is registered for course, else False'''
+    if user is None:
+        return False
+    if user.is_authenticated():
+        return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists()
+    else:
+        return False
+
 @ensure_csrf_cookie
 @cache_if_anonymous
 def course_about(request, course_id):
-    def registered_for_course(course, user):
-        if user.is_authenticated():
-            return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists()
-        else:
-            return False
     course = check_course(course_id, course_must_be_open=False)
     registered = registered_for_course(course, request.user)
     return render_to_response('portal/course_about.html', {'course': course, 'registered': registered})
@@ -280,7 +289,7 @@ def university_profile(request, org_id):
         raise Http404("University Profile not found for {0}".format(org_id))
 
     # Only grab courses for this org...
-    courses = [c for c in all_courses if c.org == org_id]
+    courses = get_courses_by_university(request.user)[org_id]
     context = dict(courses=courses, org_id=org_id)
     template_file = "university_profile/{0}.html".format(org_id).lower()
 

lms/djangoapps/dashboard/models.py

+from django.db import models
+
+# Create your models here.

lms/djangoapps/dashboard/tests.py

+"""
+This file demonstrates writing tests using the unittest module. These will pass
+when you run "manage.py test".
+
+Replace this with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+
+class SimpleTest(TestCase):
+    def test_basic_addition(self):
+        """
+        Tests that 1 + 1 always equals 2.
+        """
+        self.assertEqual(1 + 1, 2)

lms/djangoapps/dashboard/views.py

+# Create your views here.
+import json
+from datetime import datetime
+from django.http import HttpResponse, Http404
+
+def dictfetchall(cursor):
+    '''Returns all rows from a cursor as a dict.
+    Borrowed from Django documentation'''
+    desc = cursor.description
+    return [
+        dict(zip([col[0] for col in desc], row))
+        for row in cursor.fetchall()
+    ]
+
+def dashboard(request):
+    """
+    Quick hack to show staff enrollment numbers.  This should be
+    replaced with a real dashboard later. This version is a short-term
+    bandaid for the next couple weeks. 
+    """
+    if not request.user.is_staff:
+        raise Http404
+
+    query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc"
+
+    from django.db import connection
+    cursor = connection.cursor()
+    cursor.execute(query)
+    results = dictfetchall(cursor)
+
+    return HttpResponse(json.dumps(results, indent=4))

lms/djangoapps/lms_migration/migrate.py

+#
+# migration tools for content team to go from stable-edx4edx to LMS+CMS
+#
+
+import logging
+from pprint import pprint
+import xmodule.modulestore.django as xmodule_django
+from xmodule.modulestore.django import modulestore
+
+from django.http import HttpResponse
+from django.conf import settings
+
+log = logging.getLogger("mitx.lms_migrate")
+LOCAL_DEBUG = True
+ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS
+
+def escape(s):
+    """escape HTML special characters in string"""
+    return str(s).replace('<','&lt;').replace('>','&gt;')
+
+def manage_modulestores(request,reload_dir=None):
+    '''
+    Manage the static in-memory modulestores.
+
+    If reload_dir is not None, then instruct the xml loader to reload that course directory.
+    '''
+    html = "<html><body>"
+
+    def_ms = modulestore()
+    courses = def_ms.get_courses()
+
+    #----------------------------------------
+    # check on IP address of requester
+
+    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
+    if not ip:
+        ip = request.META.get('REMOTE_ADDR','None')
+    
+    if LOCAL_DEBUG:
+        html += '<h3>IP address: %s ' % ip
+        html += '<h3>User: %s ' % request.user
+        log.debug('request from ip=%s, user=%s' % (ip,request.user))
+
+    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
+        if request.user and request.user.is_staff:
+            log.debug('request allowed because user=%s is staff' % request.user)
+        else:
+            html += 'Permission denied'
+            html += "</body></html>"
+            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
+            return HttpResponse(html)    
+
+    #----------------------------------------
+    # reload course if specified
+
+    if reload_dir is not None:
+        if reload_dir not in def_ms.courses:
+            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
+        else:
+            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
+            def_ms.try_load_course(reload_dir)
+
+    #----------------------------------------
+
+    html += '<h2>Courses loaded in the modulestore</h2>'
+    html += '<ol>'
+    for cdir, course in def_ms.courses.items():
+        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (settings.MITX_ROOT_URL,
+                                                            escape(cdir),
+                                                            escape(cdir),
+                                                            course.location.url())
+    html += '</ol>'
+
+    #----------------------------------------
+
+    dumpfields = ['definition','location','metadata']
+    
+    for cdir, course in def_ms.courses.items():
+        html += '<hr width="100%"/>'
+        html += '<h2>Course: %s (%s)</h2>' % (course.metadata['display_name'],cdir)
+
+        for field in dumpfields:
+            data = getattr(course,field)
+            html += '<h3>%s</h3>' % field
+            if type(data)==dict:
+                html += '<ul>'
+                for k,v in data.items():
+                    html += '<li>%s:%s</li>' % (escape(k),escape(v))
+                html += '</ul>'
+            else:
+                html += '<ul><li>%s</li></ul>' % escape(data)
+        
+
+    #----------------------------------------
+
+    html += '<hr width="100%"/>'
+    html += "courses: <pre>%s</pre>" % escape(courses)
+
+    ms = xmodule_django._MODULESTORES
+    html += "modules: <pre>%s</pre>" % escape(ms)
+    html += "default modulestore: <pre>%s</pre>" % escape(unicode(def_ms))
+
+    #----------------------------------------
+
+    log.debug('_MODULESTORES=%s' % ms)
+    log.debug('courses=%s' % courses)
+    log.debug('def_ms=%s' % unicode(def_ms))
+
+    html += "</body></html>"
+    return HttpResponse(html)    

lms/djangoapps/ssl_auth/ssl_auth.py

-"""
-User authentication backend for ssl (no pw required)
-"""
-
-from django.conf import settings
-from django.contrib import auth
-from django.contrib.auth.models import User, check_password
-from django.contrib.auth.backends import ModelBackend
-from django.contrib.auth.middleware import RemoteUserMiddleware
-from django.core.exceptions import ImproperlyConfigured
-import os
-import string
-import re
-from random import choice
-
-from student.models import UserProfile
-
-#-----------------------------------------------------------------------------
-
-
-def ssl_dn_extract_info(dn):
-    '''
-    Extract username, email address (may be anyuser@anydomain.com) and full name
-    from the SSL DN string.  Return (user,email,fullname) if successful, and None
-    otherwise.
-    '''
-    ss = re.search('/emailAddress=(.*)@([^/]+)', dn)
-    if ss:
-        user = ss.group(1)
-        email = "%s@%s" % (user, ss.group(2))
-    else:
-        return None
-    ss = re.search('/CN=([^/]+)/', dn)
-    if ss:
-        fullname = ss.group(1)
-    else:
-        return None
-    return (user, email, fullname)
-
-
-def check_nginx_proxy(request):
-    '''
-    Check for keys in the HTTP header (META) to se if we are behind an ngix reverse proxy.
-    If so, get user info from the SSL DN string and return that, as (user,email,fullname)
-    '''
-    m = request.META
-    if m.has_key('HTTP_X_REAL_IP'):	 # we're behind a nginx reverse proxy, which has already done ssl auth
-        if not m.has_key('HTTP_SSL_CLIENT_S_DN'):
-            return None
-        dn = m['HTTP_SSL_CLIENT_S_DN']
-        return ssl_dn_extract_info(dn)
-    return None
-
-#-----------------------------------------------------------------------------
-
-
-def get_ssl_username(request):
-    x = check_nginx_proxy(request)
-    if x:
-        return x[0]
-    env = request._req.subprocess_env
-    if env.has_key('SSL_CLIENT_S_DN_Email'):
-        email = env['SSL_CLIENT_S_DN_Email']
-        user = email[:email.index('@')]
-        return user
-    return None
-
-#-----------------------------------------------------------------------------
-
-
-class NginxProxyHeaderMiddleware(RemoteUserMiddleware):
-    '''
-    Django "middleware" function for extracting user information from HTTP request.
-
-    '''
-    # this field is generated by nginx's reverse proxy
-    header = 'HTTP_SSL_CLIENT_S_DN'	 # specify the request.META field to use
-
-    def process_request(self, request):
-        # AuthenticationMiddleware is required so that request.user exists.
-        if not hasattr(request, 'user'):
-            raise ImproperlyConfigured(
-                "The Django remote user auth middleware requires the"
-                " authentication middleware to be installed.  Edit your"
-                " MIDDLEWARE_CLASSES setting to insert"
-                " 'django.contrib.auth.middleware.AuthenticationMiddleware'"
-                " before the RemoteUserMiddleware class.")
-
-        #raise ImproperlyConfigured('[ProxyHeaderMiddleware] request.META=%s' % repr(request.META))
-
-        try:
-            username = request.META[self.header]	 # try the nginx META key first
-        except KeyError:
-            try:
-                env = request._req.subprocess_env	 # else try the direct apache2 SSL key
-                if env.has_key('SSL_CLIENT_S_DN'):
-                    username = env['SSL_CLIENT_S_DN']
-                else:
-                    raise ImproperlyConfigured('no ssl key, env=%s' % repr(env))
-                    username = ''
-            except:
-                # If specified header doesn't exist then return (leaving
-                # request.user set to AnonymousUser by the
-                # AuthenticationMiddleware).
-                return
-        # If the user is already authenticated and that user is the user we are
-        # getting passed in the headers, then the correct user is already
-        # persisted in the session and we don't need to continue.
-
-        #raise ImproperlyConfigured('[ProxyHeaderMiddleware] username=%s' % username)
-
-        if request.user.is_authenticated():
-            if request.user.username == self.clean_username(username, request):
-                #raise ImproperlyConfigured('%s already authenticated (%s)' % (username,request.user.username))
-                return
-        # We are seeing this user for the first time in this session, attempt
-        # to authenticate the user.
-        #raise ImproperlyConfigured('calling auth.authenticate, remote_user=%s' % username)
-        user = auth.authenticate(remote_user=username)
-        if user:
-            # User is valid.  Set request.user and persist user in the session
-            # by logging the user in.
-            request.user = user
-            if settings.DEBUG: print "[ssl_auth.ssl_auth.NginxProxyHeaderMiddleware] logging in user=%s" % user
-            auth.login(request, user)
-
-    def clean_username(self, username, request):
-        '''
-        username is the SSL DN string - extract the actual username from it and return
-        '''
-        info = ssl_dn_extract_info(username)
-        if not info:
-            return None
-        (username, email, fullname) = info
-        return username
-
-#-----------------------------------------------------------------------------
-
-
-class SSLLoginBackend(ModelBackend):
-    '''
-    Django authentication back-end which auto-logs-in a user based on having
-    already authenticated with an MIT certificate (SSL).
-    '''
-    def authenticate(self, username=None, password=None, remote_user=None):
-
-        # remote_user is from the SSL_DN string.  It will be non-empty only when
-        # the user has already passed the server authentication, which means
-        # matching with the certificate authority.
-        if not remote_user:
-            # no remote_user, so check username (but don't auto-create user)
-            if not username:
-                return None
-            return None  # pass on to another authenticator backend
-            #raise ImproperlyConfigured("in SSLLoginBackend, username=%s, remote_user=%s" % (username,remote_user))
-            try:
-                user = User.objects.get(username=username)	 # if user already exists don't create it
-                return user
-            except User.DoesNotExist:
-                return None
-            return None
-
-        #raise ImproperlyConfigured("in SSLLoginBackend, username=%s, remote_user=%s" % (username,remote_user))
-        #if not os.environ.has_key('HTTPS'):
-        #    return None
-        #if not os.environ.get('HTTPS')=='on':	# only use this back-end if HTTPS on
-        #    return None
-
-        def GenPasswd(length=8, chars=string.letters + string.digits):
-            return ''.join([choice(chars) for i in range(length)])
-
-        # convert remote_user to user, email, fullname
-        info = ssl_dn_extract_info(remote_user)
-        #raise ImproperlyConfigured("[SSLLoginBackend] looking up %s" % repr(info))
-        if not info:
-            #raise ImproperlyConfigured("[SSLLoginBackend] remote_user=%s, info=%s" % (remote_user,info))
-            return None
-        (username, email, fullname) = info
-
-        try:
-            user = User.objects.get(username=username)	 # if user already exists don't create it
-        except User.DoesNotExist:
-            if not settings.DEBUG:
-                raise "User does not exist. Not creating user; potential schema consistency issues"
-            #raise ImproperlyConfigured("[SSLLoginBackend] creating %s" % repr(info))
-            user = User(username=username, password=GenPasswd())	 # create new User
-            user.is_staff = False
-            user.is_superuser = False
-            # get first, last name from fullname
-            name = fullname
-            if not name.count(' '):
-                user.first_name = " "
-                user.last_name = name
-                mn = ''
-            else:
-                user.first_name = name[:name.find(' ')]
-                ml = name[name.find(' '):].strip()
-                if ml.count(' '):
-                    user.last_name = ml[ml.rfind(' '):]
-                    mn = ml[:ml.rfind(' ')]
-                else:
-                    user.last_name = ml
-                    mn = ''
-            # set email
-            user.email = email
-            # cleanup last name
-            user.last_name = user.last_name.strip()
-            # save
-            user.save()
-
-            # auto-create user profile
-            up = UserProfile(user=user)
-            up.name = fullname
-            up.save()
-
-            #tui = user.get_profile()
-            #tui.middle_name = mn
-            #tui.role = 'Misc'
-            #tui.section = None	# no section assigned at first
-            #tui.save()
-            # return None
-        return user
-
-    def get_user(self, user_id):
-        #if not os.environ.has_key('HTTPS'):
-        #    return None
-        #if not os.environ.get('HTTPS')=='on':	# only use this back-end if HTTPS on
-        #    return None
-        try:
-            return User.objects.get(pk=user_id)
-        except User.DoesNotExist:
-            return None
-
-#-----------------------------------------------------------------------------
-# OLD!
-
-
-class AutoLoginBackend:
-    def authenticate(self, username=None, password=None):
-        raise ImproperlyConfigured("in AutoLoginBackend, username=%s" % username)
-        if not os.environ.has_key('HTTPS'):
-            return None
-        if not os.environ.get('HTTPS') == 'on':#  only use this back-end if HTTPS on
-            return None
-
-        def GenPasswd(length=8, chars=string.letters + string.digits):
-            return ''.join([choice(chars) for i in range(length)])
-
-        try:
-                user = User.objects.get(username=username)
-        except User.DoesNotExist:
-            user = User(username=username, password=GenPasswd())
-            user.is_staff = False
-            user.is_superuser = False
-            # get first, last name
-            name = os.environ.get('SSL_CLIENT_S_DN_CN').strip()
-            if not name.count(' '):
-                user.first_name = " "
-                user.last_name = name
-                mn = ''
-            else:
-                user.first_name = name[:name.find(' ')]
-                ml = name[name.find(' '):].strip()
-                if ml.count(' '):
-                    user.last_name = ml[ml.rfind(' '):]
-                    mn = ml[:ml.rfind(' ')]
-                else:
-                    user.last_name = ml
-                    mn = ''
-            # get email
-            user.email = os.environ.get('SSL_CLIENT_S_DN_Email')
-            # save
-            user.save()
-            tui = user.get_profile()
-            tui.middle_name = mn
-            tui.role = 'Misc'
-            tui.section = None#  no section assigned at first
-            tui.save()
-            # return None
-            return user
-
-    def get_user(self, user_id):
-        if not os.environ.has_key('HTTPS'):
-            return None
-        if not os.environ.get('HTTPS') == 'on':#  only use this back-end if HTTPS on
-            return None
-        try:
-            return User.objects.get(pk=user_id)
-        except User.DoesNotExist:
-            return None

lms/envs/common.py

@@ -48,6 +48,17 @@ MITX_FEATURES = {
     ## DO NOT SET TO True IN THIS FILE
     ## Doing so will cause all courses to be released on production
     'DISABLE_START_DATES': False,  # When True, all courses will be active, regardless of start date
+
+    'ENABLE_TEXTBOOK' : True,
+    'ENABLE_DISCUSSION' : True,
+
+    'ENABLE_SQL_TRACKING_LOGS': False,
+    'ENABLE_LMS_MIGRATION': False,
+
+    # extrernal access methods
+    'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
+    'AUTH_USE_OPENID': False,
+    'AUTH_USE_MIT_CERTIFICATES' : False,
 }
 
 # Used for A/B testing
@@ -304,7 +315,7 @@ PIPELINE_CSS = {
         'output_filename': 'css/lms-application.css',
     },
     'course': {
-      'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css'],
+      'source_filenames': ['js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', 'sass/course.scss'],
       'output_filename': 'css/lms-course.css',
       },
     'ie-fixes': {

lms/envs/dev.py

@@ -14,10 +14,11 @@ DEBUG = True
 TEMPLATE_DEBUG = True
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
+MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
 
 WIKI_ENABLED = True
 
-LOGGING = get_logger_config(ENV_ROOT / "log", 
+LOGGING = get_logger_config(ENV_ROOT / "log",
                             logging_env="dev",
                             tracking_filename="tracking.log",
                             debug=True)
@@ -30,7 +31,7 @@ DATABASES = {
 }
 
 CACHES = {
-    # This is the cache used for most things. Askbot will not work without a 
+    # This is the cache used for most things. Askbot will not work without a
     # functioning cache -- it relies on caching to load its settings in places.
     # In staging/prod envs, the sessions also live here.
     'default': {
@@ -52,11 +53,35 @@ CACHES = {
     }
 }
 
+# Make the keyedcache startup warnings go away
+CACHE_TIMEOUT = 0
+
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 
+################################ LMS Migration #################################
+MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
+MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
+
+LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
+
+################################ OpenID Auth #################################
+MITX_FEATURES['AUTH_USE_OPENID'] = True
+MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
+
+INSTALLED_APPS += ('external_auth',) 
+INSTALLED_APPS += ('django_openid_auth',)
+
+OPENID_CREATE_USERS = False
+OPENID_UPDATE_DETAILS_FROM_SREG = True
+OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'	# TODO: accept more endpoints
+OPENID_USE_AS_ADMIN_LOGIN = False
+
+################################ MIT Certificates SSL Auth #################################
+MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
+
 ################################ DEBUG TOOLBAR #################################
-INSTALLED_APPS += ('debug_toolbar',) 
+INSTALLED_APPS += ('debug_toolbar',)
 MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
 INTERNAL_IPS = ('127.0.0.1',)
 
@@ -71,8 +96,8 @@ DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.logger.LoggingPanel',
 
 #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
-#  Django=1.3.1/1.4 where requests to views get duplicated (your method gets 
-#  hit twice). So you can uncomment when you need to diagnose performance 
+#  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
+#  hit twice). So you can uncomment when you need to diagnose performance
 #  problems, but you shouldn't leave it on.
 #  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
 )

lms/envs/dev_ike.py

@@ -7,142 +7,12 @@ sessions. Assumes structure:
         /mitx # The location of this repo
         /log  # Where we're going to write log files
 """
-
-import socket
-
-if 'eecs1' in socket.gethostname():
-    MITX_ROOT_URL = '/mitx2'
-
 from .common import *
 from .logsettings import get_logger_config
 from .dev import *
 
-if 'eecs1' in socket.gethostname():
-    # MITX_ROOT_URL = '/mitx2'
-    MITX_ROOT_URL = 'https://eecs1.mit.edu/mitx2'
-
-#-----------------------------------------------------------------------------
-# edx4edx content server
-
-EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
-MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mit.edu'
-EDX4EDX_ROOT = ENV_ROOT / "data/edx4edx"
-
-#EMAIL_BACKEND = 'django_ses.SESBackend'
-
-#-----------------------------------------------------------------------------
-# ichuang
-
-DEBUG = True
-ENABLE_MULTICOURSE = True     # set to False to disable multicourse display (see lib.util.views.mitxhome)
-QUICKEDIT = False		
-
-MAKO_TEMPLATES['course'] = [DATA_DIR, EDX4EDX_ROOT ]
-
-#MITX_FEATURES['USE_DJANGO_PIPELINE'] = False
-MITX_FEATURES['DISPLAY_HISTOGRAMS_TO_STAFF'] = False
-MITX_FEATURES['DISPLAY_EDIT_LINK'] = True
-MITX_FEATURES['DEBUG_LEVEL'] = 10				# 0 = lowest level, least verbose, 255 = max level, most verbose
-
-COURSE_SETTINGS =  {'6.002x_Fall_2012': {'number' : '6.002x',
-                                          'title'  :  'Circuits and Electronics',
-                                          'xmlpath': '/6002x-fall-2012/',
-                                          'active' : True,
-                                          'default_chapter' : 'Week_1',
-                                          'default_section' : 'Administrivia_and_Circuit_Elements',
-                                          'location': 'i4x://edx/6002xs12/course/6.002x_Fall_2012',
-                                          },
-                    '8.02_Spring_2013': {'number' : '8.02x',
-                                         'title'  : 'Electricity & Magnetism',
-                                         'xmlpath': '/802x/',
-                                         'github_url': 'https://github.com/MITx/8.02x',
-                                         'active' : True,
-                                         'default_chapter' : 'Introduction',
-                                         'default_section' : 'Introduction_%28Lewin_2002%29',
-                                         },
-                    '6.189_Spring_2013': {'number' : '6.189x',
-                                         'title'  : 'IAP Python Programming',
-                                         'xmlpath': '/6.189x/',
-                                         'github_url': 'https://github.com/MITx/6.189x',
-                                         'active' : True,
-                                         'default_chapter' : 'Week_1',
-                                         'default_section' : 'Variables_and_Binding',
-                                         },
-                    '8.01_Fall_2012': {'number' : '8.01x',
-                                         'title'  : 'Mechanics',
-                                         'xmlpath': '/8.01x/',
-                                         'github_url': 'https://github.com/MITx/8.01x',
-                                         'active': True,
-                                         'default_chapter' : 'Mechanics_Online_Spring_2012',
-                                         'default_section' : 'Introduction_to_the_course',
-                                         'location': 'i4x://edx/6002xs12/course/8.01_Fall_2012',
-                                         },
-                    'edx4edx': {'number' : 'edX.01',
-                                    'title'  : 'edx4edx: edX Author Course',
-                                    'xmlpath': '/edx4edx/',
-                                    'github_url': 'https://github.com/MITx/edx4edx',
-                                    'active' : True,
-                                    'default_chapter' : 'Introduction',
-                                    'default_section' : 'edx4edx_Course',
-                                    'location': 'i4x://edx/6002xs12/course/edx4edx',
-                                    },
-                    '7.03x_Fall_2012': {'number' : '7.03x',
-                                    'title'  : 'Genetics',
-                                    'xmlpath': '/7.03x/',
-                                    'github_url': 'https://github.com/MITx/7.03x',
-                                    'active' : True,
-                                    'default_chapter' : 'Week_2',
-                                    'default_section' : 'ps1_question_1',
-                                    },
-                    '3.091x_Fall_2012': {'number' : '3.091x',
-                                    'title'  : 'Introduction to Solid State Chemistry',
-                                    'xmlpath': '/3.091x/',
-                                    'github_url': 'https://github.com/MITx/3.091x',
-                                    'active' : True,
-                                    'default_chapter' : 'Week_1',
-                                    'default_section' : 'Problem_Set_1',
-                                    },
-                    '18.06x_Linear_Algebra': {'number' : '18.06x',
-                                    'title'  : 'Linear Algebra',
-                                    'xmlpath': '/18.06x/',
-                                    'github_url': 'https://github.com/MITx/18.06x',
-                                    'default_chapter' : 'Unit_1',
-                                    'default_section' : 'Midterm_1',
-                                    'active' : True,
-                                    },
-                    '6.00x_Fall_2012': {'number' : '6.00x',
-                                    'title'  : 'Introduction to Computer Science and Programming',
-                                    'xmlpath': '/6.00x/',
-                                    'github_url': 'https://github.com/MITx/6.00x',
-                                    'active' : True,
-                                    'default_chapter' : 'Week_0',
-                                    'default_section' : 'Problem_Set_0',
-                                    'location': 'i4x://edx/6002xs12/course/6.00x_Fall_2012',
-                                    },
-                    '7.00x_Fall_2012': {'number' : '7.00x',
-                                    'title'  : 'Introduction to Biology',
-                                    'xmlpath': '/7.00x/',
-                                    'github_url': 'https://github.com/MITx/7.00x',
-                                    'active' : True,
-                                    'default_chapter' : 'Unit 1',
-                                    'default_section' : 'Introduction',
-                                    },
-                    }
-
-#-----------------------------------------------------------------------------
-
-MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + (
-    'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware',		# ssl authentication behind nginx proxy
-    )
-
-AUTHENTICATION_BACKENDS = (
-    'ssl_auth.ssl_auth.SSLLoginBackend',
-    'django.contrib.auth.backends.ModelBackend',
-    )
-
-INSTALLED_APPS = INSTALLED_APPS + (
-    'ssl_auth',
-    )
+WIKI_ENABLED = False
+MITX_FEATURES['ENABLE_TEXTBOOK'] = False
+MITX_FEATURES['ENABLE_DISCUSSION'] = False
+MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 
-LOGIN_REDIRECT_URL = MITX_ROOT_URL + '/'
-LOGIN_URL = MITX_ROOT_URL + '/'

lms/static/sass/application.scss

@@ -2,9 +2,9 @@
 
 @import 'base/reset';
 @import 'base/font_face';
+@import 'base/mixins';
 @import 'base/variables';
 @import 'base/base';
-@import 'base/mixins';
 @import 'base/extends';
 @import 'base/animations';
 

lms/static/sass/base/_mixins.scss

+@function em($pxval, $base: 16) {
+  @return #{$pxval / $base}em;
+}
+
 // Line-height
 @function lh($amount: 1) {
   @return $body-line-height * $amount;

lms/static/sass/base/_variables.scss

@@ -4,10 +4,15 @@ $gw-gutter: 20px;
 $fg-column: $gw-column;
 $fg-gutter: $gw-gutter;
 $fg-max-columns: 12;
+$fg-max-width: 1400px;
+$fg-min-width: 810px;
 
 $sans-serif: 'Open Sans', $verdana;
+$body-font-family: $sans-serif;
 $serif: $georgia;
 
+$body-font-size: em(14);
+$body-line-height: golden-ratio(.875em, 1);
 $base-font-color: rgb(60,60,60);
 $lighter-base-font-color: rgb(160,160,160);
 
@@ -15,18 +20,11 @@ $blue: rgb(29,157,217);
 $pink: rgb(182,37,104);
 $yellow: rgb(255, 252, 221);
 $error-red: rgb(253, 87, 87);
+$border-color: #C8C8C8;
 
 // old variables
-$body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
-$body-font-size: 14px;
-$body-line-height: golden-ratio($body-font-size, 1);
-
-$fg-max-width: 1400px;
-$fg-min-width: 810px;
 
 $light-gray: #ddd;
 $dark-gray: #333;
 $mit-red: #993333;
-$cream: #F6EFD4;
 $text-color: $dark-gray;
-$border-color: $light-gray;

lms/static/sass/course.scss

@@ -2,9 +2,9 @@
 
 @import 'base/reset';
 @import 'base/font_face';
+@import 'base/mixins';
 @import 'base/variables';
 @import 'base/base';
-@import 'base/mixins';
 @import 'base/extends';
 @import 'base/animations';
 

lms/static/sass/course/_help.scss

-section.help.main-content {
-  padding: lh();
-
-  h1 {
-    border-bottom: 1px solid #ddd;
-    margin-bottom: lh();
-    margin-top: 0;
-    padding-bottom: lh();
-  }
-
-  p {
-    max-width: 700px;
-  }
-
-  h2 {
-    margin-top: 0;
-  }
-
-  section.self-help {
-    float: left;
-    margin-bottom: lh();
-    margin-right: flex-gutter();
-    width: flex-grid(6);
-
-    ul {
-      margin-left: flex-gutter(6);
-
-      li {
-        margin-bottom: lh(.5);
-      }
-    }
-  }
-
-  section.help-email {
-    float: left;
-    width: flex-grid(6);
-
-    dl {
-      display: block;
-      margin-bottom: lh();
-
-      dd {
-        margin-bottom: lh();
-      }
-
-      dt {
-        clear: left;
-        float: left;
-        font-weight: bold;
-        width: flex-grid(2, 6);
-      }
-    }
-  }
-}

lms/static/sass/course/_info.scss

@@ -3,6 +3,7 @@ div.info-wrapper {
 
   section.updates {
     @extend .content;
+    line-height: lh();
 
     > h1 {
       @extend .top-header;
@@ -15,30 +16,35 @@ div.info-wrapper {
     > ol {
       list-style: none;
       padding-left: 0;
+      margin-bottom: lh();
 
       > li {
         @extend .clearfix;
         border-bottom: 1px solid #e3e3e3;
-        margin-bottom: lh(.5);
+        margin-bottom: lh();
         padding-bottom: lh(.5);
         list-style-type: disk;
 
         &:first-child {
-          background: $cream;
-          border-bottom: 1px solid darken($cream, 10%);
           margin: 0 (-(lh(.5))) lh();
           padding: lh(.5);
         }
 
         ol, ul {
-          margin: lh() 0 0 lh();
-          list-style-type: circle;
+          margin: 0;
+          list-style-type: disk;
+
+          ol,ul {
+            list-style-type: circle;
+          }
         }
 
         h2 {
           float: left;
           margin: 0 flex-gutter() 0 0;
           width: flex-grid(2, 9);
+          font-size: $body-font-size;
+          font-weight: bold;
         }
 
         section.update-description {
@@ -64,16 +70,20 @@ div.info-wrapper {
     @extend .sidebar;
     border-left: 1px solid #d3d3d3;
     @include border-radius(0 4px 4px 0);
+    @include box-shadow(none);
     border-right: 0;
 
-    header {
+    h1 {
       @extend .bottom-border;
-      padding: lh(.5) lh(.75);
+      padding: lh(.5) lh(.5);
+    }
 
-      h1 {
-        font-size: 18px;
-        margin: 0 ;
-      }
+    header {
+
+      // h1 {
+      //   font-weight: 100;
+      //   font-style: italic;
+      // }
 
       p {
          color: #666;
@@ -94,7 +104,7 @@ div.info-wrapper {
         border-bottom: 1px solid #d3d3d3;
         @include box-shadow(0 1px 0 #eee);
         @include box-sizing(border-box);
-        padding: 7px lh(.75);
+        padding: em(7) lh(.75);
         position: relative;
 
         &.expandable,
@@ -108,13 +118,13 @@ div.info-wrapper {
 
         ul {
           background: none;
-          margin: 7px (-(lh(.75))) 0;
+          margin: em(7) (-(lh(.75))) 0;
 
           li {
             border-bottom: 0;
             border-top: 1px solid #d3d3d3;
             @include box-shadow(inset 0 1px 0 #eee);
-            padding-left: 18px + lh(.75);
+            padding-left: lh(1.5);
           }
         }
 
@@ -150,7 +160,7 @@ div.info-wrapper {
           border-bottom: 0;
           @include box-shadow(none);
           color: #999;
-          font-size: 12px;
+          font-size: $body-font-size;
           font-weight: bold;
           text-transform: uppercase;
         }

lms/static/sass/course/_textbook.scss

@@ -62,7 +62,6 @@ div.book-wrapper {
         @extend .clearfix;
 
         li {
-          background-color: darken($cream, 4%);
 
           &.last {
             display: block;

lms/static/sass/course/base/_base.scss

@@ -5,3 +5,17 @@ body {
 h1, h2, h3, h4, h5, h6 {
   font-family: $sans-serif;
 }
+
+table {
+  table-layout: fixed;
+}
+
+.container {
+  padding: lh(2);
+
+  > div {
+    display: table;
+    width: 100%;
+    table-layout: fixed;
+  }
+}

lms/static/sass/course/base/_extends.scss

-.wrapper {
-  margin: 0 auto;
-  max-width: $fg-max-width;
-  min-width: $fg-min-width;
-  text-align: left;
-  width: flex-grid(12);
-
-  div.table-wrapper {
-    display: table;
-    width: flex-grid(12);
-    overflow: hidden;
-  }
-}
-
 h1.top-header {
-  background: #f3f3f3;
   border-bottom: 1px solid #e3e3e3;
-  margin: (-(lh())) (-(lh())) lh();
-  padding: lh();
   text-align: left;
-}
-
-.button {
-  border: 1px solid darken(#888, 10%);
-  @include border-radius(3px);
-  @include box-shadow(inset 0 1px 0 lighten(#888, 10%), 0 0 3px #ccc);
-  color: #fff;
-  cursor: pointer;
-  font: bold $body-font-size $body-font-family;
-  @include linear-gradient(lighten(#888, 5%), darken(#888, 5%));
-  padding: 4px 8px;
-  text-decoration: none;
-  text-shadow: none;
-  -webkit-font-smoothing: antialiased;
-
-  &:hover, &:focus {
-    border: 1px solid darken(#888, 20%);
-    @include box-shadow(inset 0 1px 0 lighten(#888, 20%), 0 0 3px #ccc);
-    @include linear-gradient(lighten(#888, 10%), darken(#888, 5%));
-  }
+  font-size: 24px;
+  font-weight: 100;
+  padding-bottom: lh();
 }
 
 .light-button, a.light-button {
@@ -84,7 +50,8 @@ h1.top-header {
 }
 
 .sidebar {
-  border-right: 1px solid #d3d3d3;
+  border-right: 1px solid #C8C8C8;
+  @include box-shadow(inset -1px 0 0 #e6e6e6);
   @include box-sizing(border-box);
   display: table-cell;
   font-family: $sans-serif;
@@ -93,11 +60,13 @@ h1.top-header {
   width: flex-grid(3);
 
   h1, h2 {
-   font-size: 18px;
-   font-weight: bold;
+   font-size: em(18);
+   font-weight: 100;
    letter-spacing: 0;
    text-transform: none;
    font-family: $sans-serif;
+   text-align: left;
+   font-style: italic;
   }
 
   a {
@@ -146,27 +115,20 @@ h1.top-header {
   }
 
   header#open_close_accordion {
-    border-bottom: 1px solid #d3d3d3;
-    @include box-shadow(0 1px 0 #eee);
-    padding: lh(.5) lh();
     position: relative;
 
-    h2 {
-      margin: 0;
-      padding-right: 20px;
-    }
-
     a {
-      background: #eee url('../images/slide-left-icon.png') center center no-repeat;
+      background: #f6f6f6 url('../images/slide-left-icon.png') center center no-repeat;
       border: 1px solid #D3D3D3;
       @include border-radius(3px 0 0 3px);
       height: 16px;
-      padding: 8px;
+      padding: 6px;
       position: absolute;
       right: -1px;
       text-indent: -9999px;
       top: 6px;
       width: 16px;
+      z-index: 99;
 
       &:hover {
         background-color: white;
@@ -181,33 +143,17 @@ h1.top-header {
 
 .topbar {
   @extend .clearfix;
-  background: $cream;
-  border-bottom: 1px solid darken($cream, 10%);
-  border-top: 1px solid #fff;
-  font-size: 12px;
-  line-height: 46px;
-  text-shadow: 0 1px 0 #fff;
+  border-bottom: 1px solid $border-color;
+  font-size: 14px;
 
   @media print {
     display: none;
   }
 
   a {
-    line-height: 46px;
-    border-bottom: 0;
-    color: darken($cream, 80%);
-
-    &:hover {
-      color: darken($cream, 60%);
-      text-decoration: none;
-    }
-
     &.block-link {
-      // background: darken($cream, 5%);
-      border-left: 1px solid darken($cream, 20%);
-      @include box-shadow(inset 1px 0 0 lighten($cream, 5%));
+      border-left: 1px solid lighten($border-color, 10%);
       display: block;
-      text-transform: uppercase;
 
       &:hover {
         background: none;
@@ -219,12 +165,3 @@ h1.top-header {
 .tran {
   @include transition( all, .2s, $ease-in-out-quad);
 }
-
-p.ie-warning {
-  background: yellow;
-  display: block !important;
-  line-height: 1.3em;
-  margin-bottom: 0;
-  padding: lh();
-  text-align: left;
-}

lms/static/sass/course/courseware/_courseware.scss

@@ -3,22 +3,6 @@ html {
   max-height: 100%;
 }
 
-body.courseware {
-  height: 100%;
-  max-height: 100%;
-
-  .container {
-    padding-bottom: 40px;
-    margin-top: 20px;
-  }
-
-  footer {
-    &.fixed-bottom {
-      Position: static;
-    }
-  }
-}
-
 div.course-wrapper {
   @extend .table-wrapper;
 
@@ -59,6 +43,9 @@ div.course-wrapper {
     }
 
     ol.vert-mod {
+      padding: 0;
+      margin: 0;
+
       > li {
         @extend .clearfix;
         @extend .problem-set;
@@ -194,17 +181,9 @@ div.course-wrapper {
       overflow: hidden;
 
       header#open_close_accordion {
-        padding: 0;
-        min-height: 47px;
-
         a {
           background-image: url('../images/slide-right-icon.png');
         }
-
-        h2 {
-          visibility: hidden;
-          width: 10px;
-        }
       }
 
       div#accordion {

lms/static/sass/course/courseware/_sidebar.scss

@@ -13,44 +13,51 @@ section.course-index {
   div#accordion {
 
     h3 {
-      @include box-shadow(inset 0 1px 0 0 #eee);
-      border-top: 1px solid #d3d3d3;
-      overflow: hidden;
+      @include border-radius(0);
+      border-top: 1px solid #e3e3e3;
       margin: 0;
+      overflow: hidden;
 
       &:first-child {
         border: none;
       }
 
       &:hover {
-        @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225)));
+        background: #f6f6f6;
+        text-decoration: none;
       }
 
       &.ui-accordion-header {
         color: #000;
 
         a {
-          font-size: $body-font-size;
+          @include border-radius(0);
+          @include box-shadow(none);
           color: lighten($text-color, 10%);
+          font-size: $body-font-size;
         }
 
         &.ui-state-active {
-          @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225)));
           @extend .active;
-          border-bottom: 1px solid #d3d3d3;
+          border-bottom: none;
+
+          &:hover {
+            background: none;
+          }
         }
       }
     }
 
     ul.ui-accordion-content {
       @include border-radius(0);
-      @include box-shadow(inset -1px 0 0 #e6e6e6);
+      background: transparent;
       border: none;
       font-size: 12px;
       margin: 0;
       padding: 1em 1.5em;
 
       li {
+        @include border-radius(0);
         margin-bottom: lh(.5);
 
         a {
@@ -98,7 +105,7 @@ section.course-index {
             &:after {
               opacity: 1;
               right: 15px;
-              @include transition(all, 0.2s, linear);
+              @include transition();
             }
 
             > a p {
@@ -120,8 +127,6 @@ section.course-index {
           font-weight: bold;
 
           > a {
-            background: rgb(240,240,240);
-            @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(230,230,230)));
             border-color: rgb(200,200,200);
 
             &:after {

lms/static/sass/course/discussion/_discussion.scss

 // Generic layout styles for the discussion forums
-
 body.askbot {
-
-  section.main-content {
+  section.container {
     div.discussion-wrapper {
       @extend .table-wrapper;
 

lms/static/sass/course/discussion/_profile.scss

@@ -72,7 +72,6 @@ body.user-profile-page {
         margin-bottom: 30px;
 
         li {
-          background-color: lighten($cream, 3%);
           background-position: 10px center;
           background-repeat: no-repeat;
           @include border-radius(4px);

lms/static/sass/course/discussion/_question-view.scss

@@ -32,8 +32,6 @@ div.question-header {
 
         &.post-vote {
           @include border-radius(4px);
-          background-color: lighten($cream, 5%);
-          border: 1px solid darken( $cream, 10% );
           @include box-shadow(inset 0 1px 0px #fff);
         }
 
@@ -149,7 +147,7 @@ div.question-header {
 
           &.revision {
             text-align: center;
-            background:lighten($cream, 7%);
+            // background:lighten($cream, 7%);
 
             a {
               color: black;
@@ -313,7 +311,6 @@ div.question-header {
               }
 
               a.edit {
-                @extend .button;
                 font-size: 12px;
                 padding: 2px 10px;
               }

lms/static/sass/course/layout/_courseware_subnav.scss

 nav.course-material {
-  background: rgb(210,210,210);
   @include clearfix;
   @include box-sizing(border-box);
-  @include box-shadow(inset 0 1px 5px 0 rgba(0,0,0, 0.05));
-  border-bottom: 1px solid rgb(190,190,190);
+  background: #f6f6f6;
+  border-bottom: 1px solid rgb(200,200,200);
   margin: 0px auto 0px;
   padding: 0px;
   width: 100%;
@@ -24,12 +23,14 @@ nav.course-material {
       list-style: none;
 
       a {
-        color: $lighter-base-font-color;
+        color: darken($lighter-base-font-color, 20%);
         display: block;
         text-align: center;
-        padding: 5px 13px;
+        padding: 8px 13px 12px;
+        font-size: 14px;
+        font-weight: 400;
         text-decoration: none;
-        text-shadow: 0 1px rgba(255,255,255, 0.4);
+        text-shadow: 0 1px rgb(255,255,255);
 
         &:hover {
           color: $base-font-color;
@@ -41,7 +42,7 @@ nav.course-material {
           border-bottom: 0px;
           @include border-top-radius(4px);
           @include box-shadow(0 2px 0 0 rgba(255,255,255, 1));
-          color: $base-font-color;
+          color: $blue;
         }
       }
     }

lms/static/sass/course/old/.gitignore

-*.css

lms/static/sass/course/wiki/_wiki.scss

@@ -24,7 +24,6 @@ div.wiki-wrapper {
       }
 
       p {
-        color: darken($cream, 55%);
         float: left;
         line-height: 46px;
         margin-bottom: 0;
@@ -40,14 +39,12 @@ div.wiki-wrapper {
 
           input[type="button"] {
             @extend .block-link;
-            background-color: darken($cream, 5%);
             background-position: 12px center;
             background-repeat: no-repeat;
             border: 0;
             border-left: 1px solid darken(#f6efd4, 20%);
             @include border-radius(0);
             @include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%));
-            color: darken($cream, 80%);
             display: block;
             font-size: 12px;
             font-weight: normal;

lms/static/sass/multicourse/_dashboard.scss

@@ -2,6 +2,27 @@
   @include clearfix;
   padding: 60px 0px 120px;
 
+  .dashboard-banner {
+    background: $yellow;
+    border: 1px solid rgb(200,200,200);
+    @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
+    padding: 10px;
+    margin-bottom: 30px;
+
+    &:empty {
+      display: none;
+      background-color: #FFF;
+    }
+
+    h2 {
+      margin-bottom: 0;
+    }
+
+    p {
+      margin-bottom: 0;
+    }
+  }
+
   .profile-sidebar {
     background: transparent;
     float: left;

lms/templates/accordion.html

 <%! from django.core.urlresolvers import reverse %>
 
 <%def name="make_chapter(chapter)">
-  <h3><a href="#">${chapter['name']}</a></h3>
+  <h3><a href="#">${chapter['display_name']}</a></h3>
 
   <ul>
     % for section in chapter['sections']:
         <li${' class="active"' if 'active' in section and section['active'] else ''}>
-          <a href="${reverse('courseware_section', args=[course_id] + format_url_params([chapter['name'], section['name']]))}">
-            <p>${section['name']}
+          <a href="${reverse('courseware_section', args=[course_id, chapter['url_name'], section['url_name']])}">
+            <p>${section['display_name']}
             <span class="subtitle">
               ${section['format']} ${"due " + section['due'] if 'due' in section and section['due'] != '' else ''}
             </span>

lms/templates/course_navigation.html

@@ -14,10 +14,16 @@ def url_class(url):
       <li class="courseware"><a href="${reverse('courseware', args=[course.id])}" class="${url_class('courseware')}">Courseware</a></li>
       <li class="info"><a href="${reverse('info', args=[course.id])}" class="${url_class('info')}">Course Info</a></li>
 % if user.is_authenticated():
+% if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
       <li class="book"><a href="${reverse('book', args=[course.id])}" class="${url_class('book')}">Textbook</a></li>
+% endif 
+% if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
       <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
+% endif 
 % endif
+% if settings.WIKI_ENABLED:
       <li class="wiki"><a href="${reverse('wiki_root', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
+% endif
 % if user.is_authenticated():
       <li class="profile"><a href="${reverse('profile', args=[course.id])}" class="${url_class('profile')}">Profile</a></li>
 % endif

lms/templates/courseware.html

@@ -18,14 +18,6 @@
   ##  <script type="text/javascript" src="${static.url('js/vendor/CodeMirror-2.25/mode/xml/xml.js')}"></script>
   ##  <script type="text/javascript" src="${static.url('js/vendor/CodeMirror-2.25/mode/python/python.js')}"></script>
 
-  ## image input: for clicking on images (see imageinput.html)
-  <script type="text/javascript" src="${static.url('js/vendor/imageinput.js')}"></script>
-
-  ## TODO (cpennington): Remove this when we have a good way for modules to specify js to load on the page
-  ## and in the wiki
-  <script type="text/javascript" src="${static.url('js/schematic.js')}"></script>
-  
-  
    <%static:js group='courseware'/>
 
   <%include file="mathjax_include.html" />
@@ -43,7 +35,6 @@
   <div class="course-wrapper">
     <section aria-label="Course Navigation" class="course-index">
       <header id="open_close_accordion">
-        <h2>Courseware Index</h2>
         <a href="#">close</a>
       </header>
 
@@ -56,6 +47,19 @@
 
     <section class="course-content">
       ${content}
+
+      % if course_errors is not UNDEFINED:
+          <h2>Course errors</h2>
+          <div id="course-errors">
+            <ul>
+          % for (msg, err) in course_errors:
+            <li>${msg}
+              <ul><li><pre>${err}</pre></li></ul>
+            </li>
+          % endfor
+            </ul>
+          </div>
+      % endif
     </section>
   </div>
 </section>

lms/templates/dashboard.html

@@ -34,10 +34,11 @@
 
 <section class="container dashboard">
   
-  <section class="dashboard-banner">
-    ${message}
-    <br/>
-  </section>
+  %if message:
+    <section class="dashboard-banner">
+      ${message}
+    </section>
+  %endif
 
   <section class="profile-sidebar">
     <header class="profile">

lms/templates/extauth_failure.html

+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
+  "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+<head>
+    <title>OpenID failed</title>
+</head>
+<body>
+    <h1>OpenID failed</h1>
+    <p>${message}</p>
+</body>
+</html>

lms/templates/index.html

@@ -144,3 +144,31 @@
     <iframe width="640" height="360" src="http://www.youtube.com/embed/C2OQ51tu7W4?showinfo=0" frameborder="0" allowfullscreen></iframe>
   </div>
 </section>
+
+% if show_signup_immediately is not UNDEFINED:
+<script type="text/javascript">
+function dosignup(){
+    comp = document.getElementById('signup');
+try { //in firefox
+    comp.click();
+    return;
+} catch(ex) {}
+try { // in old chrome
+    if(document.createEvent) {
+        var e = document.createEvent('MouseEvents');
+        e.initEvent( 'click', true, true );
+        comp.dispatchEvent(e);
+        return;
+    }
+} catch(ex) {}
+try { // in IE, safari
+    if(document.createEventObject) {
+         var evObj = document.createEventObject();
+         comp.fireEvent("onclick", evObj);
+         return;
+    }
+} catch(ex) {}
+}
+$(window).load(dosignup);
+</script>
+% endif

lms/templates/info.html

@@ -20,23 +20,25 @@ $(document).ready(function(){
 </%block>
 
 <section class="container">
-  <section class="courseware">
-    <div class="info-wrapper">
-      % if user.is_authenticated():
-      <section class="updates">
-        ${get_course_info_section(course, 'updates')}
-      </section>
-      <section aria-label="Handout Navigation" class="handouts">
-        ${get_course_info_section(course, 'handouts')}
-      </section>
-      % else:
-      <section class="updates">
-        ${get_course_info_section(course, 'guest_updates')}
-      </section>
-      <section aria-label="Handout Navigation" class="handouts">
-        ${get_course_info_section(course, 'guest_handouts')}
-      </section>
-      % endif
-    </div>
-  </section>
+  <div class="info-wrapper">
+    % if user.is_authenticated():
+    <section class="updates">
+      <h1>Course Updates &amp; News</h1>
+      ${get_course_info_section(course, 'updates')}
+    </section>
+    <section aria-label="Handout Navigation" class="handouts">
+      <h1>Course Handouts</h1>
+      ${get_course_info_section(course, 'handouts')}
+    </section>
+    % else:
+    <section class="updates">
+      <h1>Course Updates &amp; News</h1>
+      ${get_course_info_section(course, 'guest_updates')}
+    </section>
+    <section aria-label="Handout Navigation" class="handouts">
+      <h1>Course Handouts</h1>
+      ${get_course_info_section(course, 'guest_handouts')}
+    </section>
+    % endif
+  </div>
 </section>

lms/templates/login_modal.html

@@ -27,6 +27,9 @@
         <span>Not enrolled? <a href="#signup-modal" class="close-login" rel="leanModal">Sign up.</a></span>
         <a href="#forgot-password-modal" rel="leanModal" class="pwd-reset">Forgot password?</a>
       </p>
+      <p>
+      <a href="${MITX_ROOT_URL}/openid/login/">login via openid</a>
+      </p>
     </section>
 
     <div class="close-modal">

lms/templates/main.html

@@ -3,7 +3,7 @@
 <html>
 <head>
   <%block name="title"><title>edX</title></%block>
-  
+
   <link rel="icon" type="image/x-icon" href="${static.url('images/favicon.ico')}" />
 
   <%static:css group='application'/>

lms/templates/module-error.html

 <section class="outside-app">
   <h1>There has been an error on the <em>MITx</em> servers</h1>
   <p>We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible. Please email us at <a href="mailto:technical@mitx.mit.edu">technical@mitx.mit.edu</a> to report any problems or downtime.</p>
+
+% if is_staff:
+<h1>Staff-only details below:</h1>
+
+<p>Error: ${error | h}</p>
+
+<p>Raw data: ${data | h}</p>
+% endif
+
 </section>

lms/templates/portal/course_about.html

@@ -19,6 +19,8 @@
         $(document).delegate('#class_enroll_form', 'ajax:success', function(data, json, xhr) {
           if(json.success) {
             location.href="${reverse('dashboard')}";
+          }else{
+	    $('#register_message).html("<p><font color='red'>" + json.error + "</font></p>")
           }
         });
       })(this)
@@ -60,9 +62,24 @@
         <div class="main-cta">
           %if user.is_authenticated():
             %if registered:
+   	        <%
+		  ## TODO: move this logic into a view
+                  if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']:
+                      course_target = reverse('info', args=[course.id])
+                  else:
+                      course_target = reverse('about_course', args=[course.id])
+                  show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION')
+                %>
+	        %if show_link:
+                <a href="${course_target}">
+                %endif
                 <span class="register disabled">You are registered for this course (${course.number}).</span>
+	        %if show_link:
+                </a>
+                %endif
             %else:
                 <a href="#" class="register">Register for ${course.number}</a>
+	        <div id="register_message"></div>
             %endif
           %else:
             <a href="#signup-modal" class="register" rel="leanModal" data-notice='You must Sign Up or <a href="#login-modal" rel="leanModal">Log In</a> to enroll.'>Register for ${course.number}</a>

lms/templates/profile.html

@@ -4,6 +4,7 @@
 <%block name="headextra">
   <%static:css group='course'/>
 </%block>
+
 <%namespace name="profile_graphs" file="profile_graphs.js"/>
 
 <%block name="title"><title>Profile - edX 6.002x</title></%block>
@@ -71,7 +72,7 @@ $(function() {
       var new_email = $('#new_email_field').val();
       var new_password = $('#new_email_password').val();
 
-      postJSON('/change_email',{"new_email":new_email, 
+      postJSON('/change_email',{"new_email":new_email,
                                 "password":new_password},
                function(data){
           if(data.success){
@@ -80,7 +81,7 @@ $(function() {
             $("#change_email_error").html(data.error);
           }
        });
-       log_event("profile", {"type":"email_change_request", 
+       log_event("profile", {"type":"email_change_request",
                                "old_email":"${email}",
                                "new_email":new_email});
        return false;
@@ -90,7 +91,7 @@ $(function() {
       var new_name = $('#new_name_field').val();
       var rationale = $('#name_rationale_field').val();
 
-      postJSON('/change_name',{"new_name":new_name, 
+      postJSON('/change_name',{"new_name":new_name,
                                 "rationale":rationale},
                function(data){
           if(data.success){
@@ -99,7 +100,7 @@ $(function() {
             $("#change_name_error").html(data.error);
           }
        });
-       log_event("profile", {"type":"name_change_request", 
+       log_event("profile", {"type":"name_change_request",
                                "new_name":new_name,
                                "rationale":rationale});
        return false;
@@ -110,9 +111,9 @@ $(function() {
 </%block>
 
 
-<%include file="navigation.html" args="active_page='profile'" />
+<%include file="course_navigation.html" args="active_page='profile'" />
 
-<section class="main-content">
+<section class="container">
   <div class="profile-wrapper">
 
     <section class="course-info">
@@ -124,10 +125,9 @@ $(function() {
 
       <ol class="chapters">
         %for chapter in courseware_summary:
-        %if not chapter['chapter'] == "hidden":
+        %if not chapter['display_name'] == "hidden":
         <li>
-          <h2><a href="${reverse('courseware_chapter', args=format_url_params([chapter['course'], chapter['chapter']])) }">
-          ${ chapter['chapter'] }</a></h2>
+          <h2>${ chapter['display_name'] }</h2>
 
           <ol class="sections">
             %for section in chapter['sections']:
@@ -137,14 +137,14 @@ $(function() {
               total = section['section_total'].possible
               percentageString = "{0:.0%}".format( float(earned)/total) if earned > 0 and total > 0 else ""
               %>
-              
-              <h3><a href="${reverse('courseware_section', args=format_url_params([chapter['course'], chapter['chapter'], section['section']])) }">
-                ${ section['section'] }</a> ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}</h3>
+
+              <h3><a href="${reverse('courseware_section', kwargs={'course_id' : course.id, 'chapter' : chapter['url_name'], 'section' : section['url_name']})}">
+                ${ section['display_name'] }</a> ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}</h3>
               ${section['format']}
               %if 'due' in section and section['due']!="":
                 due ${section['due']}
               %endif
-              
+
               %if len(section['scores']) > 0:
                 <ol class="scores">
                   ${ "Problem Scores: " if section['graded'] else "Practice Scores: "}
@@ -153,7 +153,7 @@ $(function() {
                   %endfor
                 </ol>
               %endif
-              
+
             </li> <!--End section-->
             %endfor
           </ol> <!--End sections-->
@@ -181,7 +181,7 @@ $(function() {
         </li>
 
         <li>
-          Forum name: <strong>${username}</strong> 
+          Forum name: <strong>${username}</strong>
         </li>
 
         <li>
@@ -215,7 +215,7 @@ $(function() {
   <form id="change_name_form">
     <div id="change_name_error"> </div>
     <fieldset>
-      <p>To uphold the credibility of <span class="edx">edX</span> certificates, name changes must go through an approval process. A member of the course staff will review your request, and if approved, update your information. Please allow up to a week for your request to be processed. Thank you.</p> 
+      <p>To uphold the credibility of <span class="edx">edX</span> certificates, name changes must go through an approval process. A member of the course staff will review your request, and if approved, update your information. Please allow up to a week for your request to be processed. Thank you.</p>
       <ul>
         <li>
           <label>Enter your desired full name, as it will appear on the <span class="edx">edX</span> Certificate: </label>
@@ -234,7 +234,7 @@ $(function() {
 </div>
 
 <div id="change_email" class="leanModal_box">
-  <h1>Change e-mail</h1> 
+  <h1>Change e-mail</h1>
   <div id="apply_name_change_error"></div>
   <form id="change_email_form">
     <div id="change_email_error"> </div>

lms/templates/signup_modal.html

@@ -19,6 +19,7 @@
         <div id="register_error" name="register_error"></div>
 
         <div class="input-group">
+	  % if has_extauth_info is UNDEFINED:
           <label data-field="email">E-mail*</label>
           <input name="email" type="email" placeholder="E-mail*">
           <label data-field="password">Password*</label>
@@ -27,6 +28,12 @@
           <input name="username" type="text" placeholder="Public Username*">
           <label data-field="name">Full Name</label>
           <input name="name" type="text" placeholder="Full Name*">
+	  % else:
+	  <p><i>Welcome</i> ${extauth_email}</p><br/>
+	  <p><i>Enter a public username:</i></p>
+          <label data-field="username">Public Username*</label>
+          <input name="username" type="text" value="${extauth_username}" placeholder="Public Username*">
+	  % endif
         </div>
 
         <div class="input-group">
@@ -93,11 +100,13 @@
         </div>
       </form>
 
+      % if has_extauth_info is UNDEFINED:
       <section class="login-extra">
         <p>
           <span>Already have an account? <a href="#login-modal" class="close-signup" rel="leanModal">Login.</a></span>
         </p>
       </section>
+      % endif
 
     </div>
 

lms/templates/staff_problem_info.html

 ${module_content}
-<div class="staff_info">
-definition = ${definition | h}
-metadata = ${metadata | h}
-</div>
 %if edit_link:
 <div><a href="${edit_link}">Edit</a></div>
 % endif
+<div class="staff_info">
+definition = <pre>${definition | h}</pre>
+metadata = ${metadata | h}
+</div>
 %if render_histogram:
 <div id="histogram_${element_id}" class="histogram" data-histogram="${histogram}"></div>
 %endif

lms/templates/staticbook.html

@@ -67,7 +67,6 @@ $("#open_close_accordion a").click(function(){
 
     <section aria-label="Textbook Navigation" class="book-sidebar">
       <header id="open_close_accordion">
-        <h2>Table of Contents</h2>
         <a href="#">close</a>
       </header>
 

lms/templates/tracking_log.html

+<html>
+<h1>Tracking Log</h1>
+<table border="1"><tr><th>datetime</th><th>username</th><th>ipaddr</th><th>source</th><th>type</th></tr>
+% for rec in records:
+    <tr>
+    <td>${rec.time}</td> 
+    <td>${rec.username}</td>  
+    <td>${rec.ip}</td>  
+    <td>${rec.event_source}</td>   
+    <td>${rec.event_type}</td>   
+    </tr>
+% endfor
+</table>    
+</html>
\ No newline at end of file

lms/templates/video.html

 % if name is not UNDEFINED and name is not None:
-  <h1> ${name} </h1>
+  <h1> ${display_name} </h1>
 % endif
 
 <div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}">

lms/urls.py

@@ -14,6 +14,8 @@ urlpatterns = ('',
     url(r'^$', 'student.views.index', name="root"), # Main marketing page, or redirect to courseware
     url(r'^dashboard$', 'student.views.dashboard', name="dashboard"),
 
+    url(r'^admin_dashboard$', 'dashboard.views.dashboard'),
+    
     url(r'^change_email$', 'student.views.change_email_request'),
     url(r'^email_confirm/(?P<key>[^/]*)$', 'student.views.confirm_email_change'),
     url(r'^change_name$', 'student.views.change_name_request'),
@@ -160,12 +162,29 @@ if settings.DEBUG:
     ## Jasmine
     urlpatterns=urlpatterns + (url(r'^_jasmine/', include('django_jasmine.urls')),)
 
+if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
+    urlpatterns += (
+        url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'),
+        url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'),
+        url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
+        )
+
+if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+    urlpatterns += (
+        url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
+        url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
+        )
+
+if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
+    urlpatterns += (
+        url(r'^event_logs$', 'track.views.view_tracking_log'),
+        )
+
 urlpatterns = patterns(*urlpatterns)
 
 if settings.DEBUG:
     urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
 
-
 #Custom error pages
 handler404 = 'static_template_view.views.render_404'
 handler500 = 'static_template_view.views.render_500'

prod-requirements.txt

+Django==1.3.1
+flup==1.0.3.dev-20110405
+lxml==2.3.4
+Mako==0.7.0
+Markdown==2.1.1
+markdown2==1.4.2
+python-memcached==1.48
+numpy==1.6.1
+Pygments==1.5
+boto==2.3.0
+django-storages==1.1.4
+django-masquerade==0.1.5
+fs==0.4.0
+django-jasmine==0.3.2
+path.py==2.2.2
+requests==0.12.1
+BeautifulSoup==3.2.1
+BeautifulSoup4==4.1.1
+newrelic==1.3.0.289
+ipython==0.12.1
+django-pipeline==1.2.12
+django-staticfiles==1.2.1
+glob2==0.3
+sympy==0.7.1
+pymongo==2.2.1
+rednose==0.3.3
+mock==0.8.0
+GitPython==0.3.2.RC1
+PyYAML==3.10
+feedparser==5.1.2
+MySQL-python==1.2.3
+matplotlib==1.1.0
+scipy==0.10.1
+akismet==0.2.0
+Coffin==0.3.6
+django-celery==2.2.7
+django-countries==1.0.5
+django-followit==0.0.3
+django-keyedcache==1.4-6
+django-kombu==0.9.2
+django-mako==0.1.5pre
+django-recaptcha-works==0.3.4
+django-robots==0.8.1
+django-ses==0.4.1
+django-threaded-multihost==1.4-1
+html5lib==0.90
+Jinja2==2.6
+oauth2==1.5.211
+pystache==0.3.1
+python-openid==2.2.5
+South==0.7.5
+Unidecode==0.04.9

requirements.txt

@@ -2,19 +2,20 @@ django<1.4
 pip
 numpy
 scipy
-matplotlib
 markdown
 pygments
 lxml
 boto
 mako
 python-memcached
+python-openid 
 path.py
 django_debug_toolbar
 -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
 django-staticfiles>=1.2.1
 fs
-beautifulsoup
+beautifulsoup 
+beautifulsoup4
 feedparser
 requests
 sympy
@@ -37,6 +38,7 @@ django-jasmine
 django-keyedcache
 django-mako
 django-masquerade
+django-openid-auth 
 django-robots
 django-ses
 django-storages

utility-scripts/create_groups.py

+#!/usr/bin/python
+#
+# File:   create_groups.py
+#
+# Create all staff_* groups for classes in data directory.
+
+import os, sys, string, re
+
+sys.path.append(os.path.abspath('.'))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
+
+try:
+    from lms.envs.dev import *
+except Exception as err:
+    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
+    sys.exit(-1)
+
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+from path import path
+
+data_dir = settings.DATA_DIR
+print "data_dir = %s" % data_dir
+
+for course_dir in os.listdir(data_dir):
+    # print course_dir
+    if not os.path.isdir(path(data_dir) / course_dir):
+        continue
+    gname = 'staff_%s' % course_dir
+    if Group.objects.filter(name=gname):
+        print "group exists for %s" % gname
+        continue
+    g = Group(name=gname)
+    g.save()
+    print "created group %s" % gname

utility-scripts/create_user.py

+#!/usr/bin/python
+#
+# File:   create_user.py
+#
+# Create user.  Prompt for groups and ExternalAuthMap
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+sys.path.append(os.path.abspath('.'))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
+
+try:
+    from lms.envs.dev import *
+except Exception as err:
+    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
+    sys.exit(-1)
+
+from student.models import UserProfile, Registration
+from external_auth.models import ExternalAuthMap
+from django.contrib.auth.models import User, Group
+from random import choice
+
+class MyCompleter(object):  # Custom completer
+
+    def __init__(self, options):
+        self.options = sorted(options)
+
+    def complete(self, text, state):
+        if state == 0:  # on first trigger, build possible matches
+            if text:  # cache matches (entries that start with entered text)
+                self.matches = [s for s in self.options 
+                                    if s and s.startswith(text)]
+            else:  # no text entered, all matches possible
+                self.matches = self.options[:]
+
+        # return match indexed by state
+        try: 
+            return self.matches[state]
+        except IndexError:
+            return None
+
+def GenPasswd(length=8, chars=string.letters + string.digits):
+    return ''.join([choice(chars) for i in range(length)])
+
+#-----------------------------------------------------------------------------
+# main
+
+while True:
+    uname = raw_input('username: ')
+    if User.objects.filter(username=uname):
+        print "username %s already taken" % uname
+    else:
+        break
+
+make_eamap = False
+if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
+    email = '%s@MIT.EDU' % uname
+    if not email.endswith('@MIT.EDU'):
+        print "Failed - email must be @MIT.EDU"
+        sys.exit(-1)
+    mit_domain = 'ssl:MIT'
+    if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
+        print "Failed - email %s already exists as external_id" % email
+        sys.exit(-1)
+    make_eamap = True
+    password = GenPasswd(12)
+    
+    # get name from kerberos
+    kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+    name = raw_input('Full name: [%s] ' % kname).strip()
+    if name=='':
+        name = kname
+    print "name = %s" % name
+else:
+    while True:
+        password = getpass()
+        password2 = getpass()
+        if password == password2:
+            break
+        print "Oops, passwords do not match, please retry"
+
+    while True:
+        email = raw_input('email: ')
+        if User.objects.filter(email=email):
+            print "email %s already taken" % email
+        else:
+            break
+        
+    name = raw_input('Full name: ')
+
+        
+user = User(username=uname, email=email, is_active=True)
+user.set_password(password)
+try:
+    user.save()
+except IntegrityError:
+    print "Oops, failed to create user %s, IntegrityError" % user
+    raise
+    
+r = Registration()
+r.register(user)
+
+up = UserProfile(user=user)
+up.name = name
+up.save()
+
+if make_eamap:
+    credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
+    eamap = ExternalAuthMap(external_id = email,
+                            external_email = email,
+                            external_domain = mit_domain,
+                            external_name = name,
+                            internal_password = password,
+                            external_credentials = json.dumps(credentials),
+        )
+    eamap.user = user
+    eamap.dtsignup = datetime.datetime.now()
+    eamap.save()
+
+print "User %s created successfully!" % user
+
+if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
+    sys.exit(0)
+
+print "Here are the groups available:"
+
+groups = [str(g.name) for g in Group.objects.all()]
+print groups
+
+completer = MyCompleter(groups)
+readline.set_completer(completer.complete)
+readline.parse_and_bind('tab: complete')
+
+while True:
+    gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
+    if not gname:
+        break
+    if not gname in groups:
+        print "Unknown group %s" % gname
+        continue
+    g = Group.objects.get(name=gname)
+    user.groups.add(g)
+    print "Added %s to group %s" % (user,g)
+
+print "Done!"