Merge branch 'master' of github.com:MITx/mitx

Conflicts:
	lms/djangoapps/courseware/module_render.py
	lms/djangoapps/courseware/views.py
	lms/envs/common.py

.gitmodules

+[submodule "askbot"]
+	path = askbot
+	url = git@github.com:MITx/askbot-devel.git

askbot @ 1c338104

+Subproject commit 1c3381046c78e055439ba1c78e0df48410fcc13e

cms/djangoapps/contentstore/management/commands/import.py

 ###
-### One-off script for importing courseware form XML format
+### Script for importing courseware from XML format
 ###
 
 from django.core.management.base import BaseCommand, CommandError
-from contentstore import import_from_xml
+from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.django import modulestore
+
 
 unnamed_modules = 0
 
@@ -21,4 +23,4 @@ class Command(BaseCommand):
             course_dirs = args[1:]
         else:
             course_dirs = None
-        import_from_xml(data_dir, course_dirs)
+        import_from_xml(modulestore(), data_dir, course_dirs)

cms/djangoapps/contentstore/tests/tests.py

@@ -12,7 +12,7 @@ from django.contrib.auth.models import User
 from xmodule.modulestore.django import modulestore
 import xmodule.modulestore.django
 from xmodule.modulestore import Location
-from contentstore import import_from_xml
+from xmodule.modulestore.xml_importer import import_from_xml
 import copy
 
 
@@ -74,7 +74,7 @@ class ContentStoreTestCase(TestCase):
         return resp
 
     def _activate_user(self, email):
-        '''look up the user's activation key in the db, then hit the activate view.
+        '''Look up the activation key for the user, then hit the activate view.
         No error checking'''
         activation_key = registration(email).activation_key
 
@@ -102,7 +102,7 @@ class AuthTestCase(ContentStoreTestCase):
         resp = self.client.get(url)
         self.assertEqual(resp.status_code, expected)
         return resp
-        
+
     def test_public_pages_load(self):
         """Make sure pages that don't require login load without error."""
         pages = (
@@ -196,7 +196,7 @@ class EditTestCase(ContentStoreTestCase):
         xmodule.modulestore.django.modulestore().collection.drop()
 
     def check_edit_item(self, test_course_name):
-        import_from_xml('common/test/data/', test_course_name)
+        import_from_xml(modulestore(), 'common/test/data/', [test_course_name])
 
         for descriptor in modulestore().get_items(Location(None, None, None, None, None)):
             print "Checking ", descriptor.location.url()

cms/djangoapps/github_sync/__init__.py

@@ -5,35 +5,53 @@ from django.conf import settings
 from fs.osfs import OSFS
 from git import Repo, PushInfo
 
-from contentstore import import_from_xml
-from xmodule.modulestore import Location
+from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.django import modulestore
+from collections import namedtuple
 
-from .exceptions import GithubSyncError
+from .exceptions import GithubSyncError, InvalidRepo
 
 log = logging.getLogger(__name__)
 
+RepoSettings = namedtuple('RepoSettings', 'path branch origin')
+
+
+def sync_all_with_github():
+    """
+    Sync all defined repositories from github
+    """
+    for repo_name in settings.REPOS:
+        sync_with_github(load_repo_settings(repo_name))
+
+
+def sync_with_github(repo_settings):
+    """
+    Sync specified repository from github
+
+    repo_settings: A RepoSettings defining which repo to sync
+    """
+    revision, course = import_from_github(repo_settings)
+    export_to_github(course, "Changes from cms import of revision %s" % revision, "CMS <cms@edx.org>")
+
 
 def setup_repo(repo_settings):
     """
     Reset the local github repo specified by repo_settings
 
-    repo_settings is a dictionary with the following keys:
-        path: file system path to the local git repo
-        branch: name of the branch to track on github
-        origin: git url for the repository to track
+    repo_settings (RepoSettings): The settings for the repo to reset
     """
-    course_dir = repo_settings['path']
+    course_dir = repo_settings.path
     repo_path = settings.GITHUB_REPO_ROOT / course_dir
 
     if not os.path.isdir(repo_path):
-        Repo.clone_from(repo_settings['origin'], repo_path)
+        Repo.clone_from(repo_settings.origin, repo_path)
 
     git_repo = Repo(repo_path)
     origin = git_repo.remotes.origin
     origin.fetch()
 
     # Do a hard reset to the remote branch so that we have a clean import
-    git_repo.git.checkout(repo_settings['branch'])
+    git_repo.git.checkout(repo_settings.branch)
 
     return git_repo
 
@@ -42,21 +60,22 @@ def load_repo_settings(course_dir):
     """
     Returns the repo_settings for the course stored in course_dir
     """
-    for repo_settings in settings.REPOS.values():
-        if repo_settings['path'] == course_dir:
-            return repo_settings
-    raise InvalidRepo(course_dir)
+    if course_dir not in settings.REPOS:
+        raise InvalidRepo(course_dir)
+
+    return RepoSettings(course_dir, **settings.REPOS[course_dir])
 
 
 def import_from_github(repo_settings):
     """
     Imports data into the modulestore based on the XML stored on github
     """
-    course_dir = repo_settings['path']
+    course_dir = repo_settings.path
     git_repo = setup_repo(repo_settings)
-    git_repo.head.reset('origin/%s' % repo_settings['branch'], index=True, working_tree=True)
+    git_repo.head.reset('origin/%s' % repo_settings.branch, index=True, working_tree=True)
 
-    module_store = import_from_xml(settings.GITHUB_REPO_ROOT, course_dirs=[course_dir])
+    module_store = import_from_xml(modulestore(),
+                                   settings.GITHUB_REPO_ROOT, course_dirs=[course_dir])
     return git_repo.head.commit.hexsha, module_store.courses[course_dir]
 
 

cms/djangoapps/github_sync/exceptions.py

 class GithubSyncError(Exception):
     pass
+
+
+class InvalidRepo(Exception):
+    pass

cms/djangoapps/github_sync/management/commands/sync_with_github.py

+###
+### Script for syncing CMS with defined github repos
+###
+
+from django.core.management.base import NoArgsCommand
+from github_sync import sync_all_with_github
+
+
+class Command(NoArgsCommand):
+    help = \
+'''Sync the CMS with the defined github repos'''
+
+    def handle_noargs(self, **options):
+        sync_all_with_github()

cms/djangoapps/github_sync/tests/__init__.py

 from django.test import TestCase
 from path import path
 import shutil
-import os
-from github_sync import import_from_github, export_to_github
+from github_sync import (
+    import_from_github, export_to_github, load_repo_settings,
+    sync_all_with_github, sync_with_github
+)
 from git import Repo
 from django.conf import settings
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore import Location
 from override_settings import override_settings
 from github_sync.exceptions import GithubSyncError
+from mock import patch, Mock
 
 REPO_DIR = settings.GITHUB_REPO_ROOT / 'local_repo'
 WORKING_DIR = path(settings.TEST_ROOT)
@@ -16,8 +19,7 @@ REMOTE_DIR = WORKING_DIR / 'remote_repo'
 
 
 @override_settings(REPOS={
-    'local': {
-        'path': 'local_repo',
+    'local_repo': {
         'origin': REMOTE_DIR,
         'branch': 'master',
     }
@@ -40,7 +42,7 @@ class GithubSyncTestCase(TestCase):
         remote.git.commit(m='Initial commit')
         remote.git.config("receive.denyCurrentBranch", "ignore")
 
-        self.import_revision, self.import_course = import_from_github(settings.REPOS['local'])
+        self.import_revision, self.import_course = import_from_github(load_repo_settings('local_repo'))
 
     def tearDown(self):
         self.cleanup()
@@ -57,10 +59,23 @@ class GithubSyncTestCase(TestCase):
         """
         self.assertEquals('Toy Course', self.import_course.metadata['display_name'])
         self.assertIn(
-            Location('i4x://edx/local_repo/chapter/Overview'),
+            Location('i4x://edX/toy/chapter/Overview'),
             [child.location for child in self.import_course.get_children()])
         self.assertEquals(1, len(self.import_course.get_children()))
 
+    @patch('github_sync.sync_with_github')
+    def test_sync_all_with_github(self, sync_with_github):
+        sync_all_with_github()
+        sync_with_github.assert_called_with(load_repo_settings('local_repo'))
+
+    def test_sync_with_github(self):
+        with patch('github_sync.import_from_github', Mock(return_value=(Mock(), Mock()))) as import_from_github:
+            with patch('github_sync.export_to_github') as export_to_github:
+                settings = load_repo_settings('local_repo')
+                sync_with_github(settings)
+                import_from_github.assert_called_with(settings)
+                export_to_github.assert_called
+
     @override_settings(MITX_FEATURES={'GITHUB_PUSH': False})
     def test_export_no_pash(self):
         """

cms/djangoapps/github_sync/tests/test_views.py

 import json
 from django.test.client import Client
 from django.test import TestCase
-from mock import patch, Mock
+from mock import patch
 from override_settings import override_settings
-from django.conf import settings
+from github_sync import load_repo_settings
 
 
-@override_settings(REPOS={'repo': {'path': 'path', 'branch': 'branch'}})
+@override_settings(REPOS={'repo': {'branch': 'branch', 'origin': 'origin'}})
 class PostReceiveTestCase(TestCase):
     def setUp(self):
         self.client = Client()
 
-    @patch('github_sync.views.export_to_github')
-    @patch('github_sync.views.import_from_github')
-    def test_non_branch(self, import_from_github, export_to_github):
+    @patch('github_sync.views.sync_with_github')
+    def test_non_branch(self, sync_with_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/tags/foo'})
         })
-        self.assertFalse(import_from_github.called)
-        self.assertFalse(export_to_github.called)
+        self.assertFalse(sync_with_github.called)
 
-    @patch('github_sync.views.export_to_github')
-    @patch('github_sync.views.import_from_github')
-    def test_non_watched_repo(self, import_from_github, export_to_github):
+    @patch('github_sync.views.sync_with_github')
+    def test_non_watched_repo(self, sync_with_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/heads/branch',
             'repository': {'name': 'bad_repo'}})
         })
-        self.assertFalse(import_from_github.called)
-        self.assertFalse(export_to_github.called)
+        self.assertFalse(sync_with_github.called)
 
-    @patch('github_sync.views.export_to_github')
-    @patch('github_sync.views.import_from_github')
-    def test_non_tracked_branch(self, import_from_github, export_to_github):
+    @patch('github_sync.views.sync_with_github')
+    def test_non_tracked_branch(self, sync_with_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/heads/non_branch',
             'repository': {'name': 'repo'}})
         })
-        self.assertFalse(import_from_github.called)
-        self.assertFalse(export_to_github.called)
+        self.assertFalse(sync_with_github.called)
 
-    @patch('github_sync.views.export_to_github')
-    @patch('github_sync.views.import_from_github', return_value=(Mock(), Mock()))
-    def test_tracked_branch(self, import_from_github, export_to_github):
+    @patch('github_sync.views.sync_with_github')
+    def test_tracked_branch(self, sync_with_github):
         self.client.post('/github_service_hook', {'payload': json.dumps({
             'ref': 'refs/heads/branch',
             'repository': {'name': 'repo'}})
         })
-        import_from_github.assert_called_with(settings.REPOS['repo'])
-        mock_revision, mock_course = import_from_github.return_value
-        export_to_github.assert_called_with(mock_course, 'path', "Changes from cms import of revision %s" % mock_revision)
+        sync_with_github.assert_called_with(load_repo_settings('repo'))

cms/djangoapps/github_sync/views.py

@@ -5,7 +5,7 @@ from django.http import HttpResponse
 from django.conf import settings
 from django_future.csrf import csrf_exempt
 
-from . import import_from_github, export_to_github
+from . import sync_with_github, load_repo_settings
 
 log = logging.getLogger()
 
@@ -40,13 +40,12 @@ def github_post_receive(request):
         log.info('No repository matching %s found' % repo_name)
         return HttpResponse('No Repo Found')
 
-    repo = settings.REPOS[repo_name]
+    repo = load_repo_settings(repo_name)
 
-    if repo['branch'] != branch_name:
+    if repo.branch != branch_name:
         log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name))
         return HttpResponse('Ignoring non-tracked branch')
 
-    revision, course = import_from_github(repo)
-    export_to_github(course, repo['path'], "Changes from cms import of revision %s" % revision)
+    sync_with_github(repo)
 
     return HttpResponse('Push received')

cms/envs/aws.py

+"""
+This is the default template for our main set of AWS servers.
+"""
+import json
+
+from .logsettings import get_logger_config
+from .common import *
+
+############################### ALWAYS THE SAME ################################
+DEBUG = False
+TEMPLATE_DEBUG = False
+
+EMAIL_BACKEND = 'django_ses.SESBackend'
+SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
+DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
+
+########################### NON-SECURE ENV CONFIG ##############################
+# Things like server locations, ports, etc.
+with open(ENV_ROOT / "cms.env.json") as env_file:
+    ENV_TOKENS = json.load(env_file)
+
+SITE_NAME = ENV_TOKENS['SITE_NAME']
+
+LOG_DIR = ENV_TOKENS['LOG_DIR']
+
+CACHES = ENV_TOKENS['CACHES']
+
+for feature, value in ENV_TOKENS.get('MITX_FEATURES', {}).items():
+    MITX_FEATURES[feature] = value
+
+LOGGING = get_logger_config(LOG_DIR,
+                            logging_env=ENV_TOKENS['LOGGING_ENV'],
+                            syslog_addr=(ENV_TOKENS['SYSLOG_SERVER'], 514),
+                            debug=False)
+
+with open(ENV_ROOT / "repos.json") as repos_file:
+    REPOS = json.load(repos_file)
+
+
+############################## SECURE AUTH ITEMS ###############################
+# Secret things: passwords, access keys, etc.
+with open(ENV_ROOT / "cms.auth.json") as auth_file:
+    AUTH_TOKENS = json.load(auth_file)
+
+AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"]
+AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"]
+DATABASES = AUTH_TOKENS['DATABASES']
+MODULESTORE = AUTH_TOKENS['MODULESTORE']

cms/envs/common.py

@@ -243,7 +243,7 @@ with open(module_styles_path, 'w') as module_styles:
 PIPELINE_CSS = {
     'base-style': {
         'source_filenames': ['sass/base-style.scss'],
-        'output_filename': 'css/base-style.css',
+        'output_filename': 'css/cms-base-style.css',
     },
 }
 
@@ -260,15 +260,15 @@ PIPELINE_JS = {
             for pth
             in glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee')
         ],
-        'output_filename': 'js/application.js',
+        'output_filename': 'js/cms-application.js',
     },
     'module-js': {
         'source_filenames': module_js_sources,
-        'output_filename': 'js/modules.js',
+        'output_filename': 'js/cms-modules.js',
     },
     'spec': {
         'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '') for pth in glob2.glob(PROJECT_ROOT / 'static/coffee/spec/**/*.coffee')],
-        'output_filename': 'js/spec.js'
+        'output_filename': 'js/cms-spec.js'
     }
 }
 
@@ -309,6 +309,7 @@ INSTALLED_APPS = (
 
     # For CMS
     'contentstore',
+    'github_sync',
     'student',  # misleading name due to sharing with lms
 
     # For asset pipelining

cms/envs/dev.py

@@ -32,38 +32,23 @@ DATABASES = {
 
 REPOS = {
     'edx4edx': {
-        'path': "edx4edx",
-        'org': 'edx',
-        'course': 'edx4edx',
-        'branch': 'for_cms',
+        'branch': 'master',
         'origin': 'git@github.com:MITx/edx4edx.git',
     },
-    '6002x-fall-2012': {
-        'path': '6002x-fall-2012',
-        'org': 'mit.edu',
-        'course': '6.002x',
-        'branch': 'for_cms',
+    'content-mit-6002x': {
+        'branch': 'master',
         'origin': 'git@github.com:MITx/6002x-fall-2012.git',
     },
     '6.00x': {
-        'path': '6.00x',
-        'org': 'mit.edu',
-        'course': '6.00x',
-        'branch': 'for_cms',
+        'branch': 'master',
         'origin': 'git@github.com:MITx/6.00x.git',
     },
     '7.00x': {
-        'path': '7.00x',
-        'org': 'mit.edu',
-        'course': '7.00x',
-        'branch': 'for_cms',
+        'branch': 'master',
         'origin': 'git@github.com:MITx/7.00x.git',
     },
     '3.091x': {
-        'path': '3.091x',
-        'org': 'mit.edu',
-        'course': '3.091x',
-        'branch': 'for_cms',
+        'branch': 'master',
         'origin': 'git@github.com:MITx/3.091x.git',
     },
 }

cms/envs/logsettings.py

+import os
+import os.path
+import platform
+import sys
+
+def get_logger_config(log_dir, 
+                      logging_env="no_env", 
+                      tracking_filename=None,
+                      syslog_addr=None,
+                      debug=False):
+    """Return the appropriate logging config dictionary. You should assign the
+    result of this to the LOGGING var in your settings. The reason it's done 
+    this way instead of registering directly is because I didn't want to worry
+    about resetting the logging state if this is called multiple times when 
+    settings are extended."""
+
+    # If we're given an explicit place to put tracking logs, we do that (say for
+    # debugging). However, logging is not safe for multiple processes hitting 
+    # the same file. So if it's left blank, we dynamically create the filename
+    # based on the PID of this worker process.
+    if tracking_filename:
+        tracking_file_loc = os.path.join(log_dir, tracking_filename)
+    else:
+        pid = os.getpid() # So we can log which process is creating the log
+        tracking_file_loc = os.path.join(log_dir, "tracking_{0}.log".format(pid))
+
+    hostname = platform.node().split(".")[0]
+    syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s [{hostname} " +
+                     " %(process)d] [%(filename)s:%(lineno)d] - %(message)s").format(
+                        logging_env=logging_env, hostname=hostname)
+
+    handlers = ['console'] if debug else ['console', 'syslogger', 'newrelic']
+
+    return {
+        'version': 1,
+        'formatters' : {
+            'standard' : {
+                'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s',
+            },
+            'syslog_format' : { 'format' : syslog_format },
+            'raw' : { 'format' : '%(message)s' },
+        },
+        'handlers' : {
+            'console' : {
+                'level' : 'DEBUG' if debug else 'INFO',
+                'class' : 'logging.StreamHandler',
+                'formatter' : 'standard',
+                'stream' : sys.stdout,
+            },
+            'syslogger' : {
+                'level' : 'INFO',
+                'class' : 'logging.handlers.SysLogHandler',
+                'address' : syslog_addr,
+                'formatter' : 'syslog_format',
+            },
+            'tracking' : {
+                'level' : 'DEBUG',
+                'class' : 'logging.handlers.WatchedFileHandler',
+                'filename' : tracking_file_loc,
+                'formatter' : 'raw',
+            },
+            'newrelic' : {
+                'level': 'ERROR',
+                'class': 'newrelic_logging.NewRelicHandler',
+                'formatter': 'raw',
+            }
+        },
+        'loggers' : {
+            'django' : {
+                'handlers' : handlers,
+                'propagate' : True,
+                'level' : 'INFO'
+            },
+            'tracking' : {
+                'handlers' : ['tracking'],
+                'level' : 'DEBUG',
+                'propagate' : False,
+            },
+            '' : {
+                'handlers' : handlers,
+                'level' : 'DEBUG',
+                'propagate' : False
+            },
+            'mitx' : {
+                'handlers' : handlers,
+                'level' : 'DEBUG',
+                'propagate' : False
+            },
+            'keyedcache' : {
+                'handlers' : handlers,
+                'level' : 'DEBUG',
+                'propagate' : False
+            },
+        }
+    }

cms/static/sass/_base.scss

@@ -5,7 +5,7 @@ $body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida
 $body-font-size: 14px;
 $body-line-height: 20px;
 
-$light-blue: #f0f8fa;
+$light-blue: #f0f7fd;
 $dark-blue: #50545c;
 $bright-blue: #3c8ebf;
 $orange: #f96e5b;
@@ -13,6 +13,14 @@ $yellow: #fff8af;
 $cream: #F6EFD4;
 $mit-red: #933;
 
+@mixin hide-text {
+  background-color: transparent;
+  border:           0;
+  color:            transparent;
+  font:             0/0 a;
+  text-shadow:      none;
+}
+
 // Base html styles
 html {
   height: 100%;
@@ -34,14 +42,18 @@ input {
 
 button, input[type="submit"], .button {
   background-color: $orange;
-  border: 0;
+  border: 1px solid darken($orange, 15%);
+  @include border-radius(4px);
+  @include box-shadow(inset 0 0 0 1px adjust-hue($orange, 20%), 0 1px 0 #fff);
   color: #fff;
   font-weight: bold;
-  padding: 8px 10px;
+  @include linear-gradient(adjust-hue($orange, 8%), $orange);
+  padding: 6px 20px;
+  text-shadow: 0 1px 0 darken($orange, 10%);
   -webkit-font-smoothing: antialiased;
 
-  &:hover {
-    background-color: shade($orange, 10%);
+  &:hover, &:focus {
+    @include box-shadow(inset 0 0 6px 1px adjust-hue($orange, 30%));
   }
 }
 
@@ -122,10 +134,10 @@ textarea {
   }
 }
 
-.wip {
-  outline: 1px solid #f00 !important;
-  position: relative;
-}
+// .wip {
+//   outline: 1px solid #f00 !important;
+//   position: relative;
+// }
 
 .hidden {
   display: none;

cms/static/sass/_calendar.scss

 section.cal {
   @include box-sizing(border-box);
   @include clearfix;
-  padding: 25px;
+  padding: 20px;
 
   > header {
+    display: none;
     @include clearfix;
     margin-bottom: 10px;
     opacity: .4;
     @include transition;
+    text-shadow: 0 1px 0 #fff;
 
     &:hover {
       opacity: 1;
@@ -70,12 +72,15 @@ section.cal {
   ol {
     list-style: none;
     @include clearfix;
-    border-left: 1px solid lighten($dark-blue, 40%);
-    border-top: 1px solid lighten($dark-blue, 40%);
+    border: 1px solid lighten( $dark-blue , 30% );
+    background: #FFF;
     width: 100%;
     @include box-sizing(border-box);
     margin: 0;
     padding: 0;
+    @include box-shadow(0 0 5px lighten($dark-blue, 45%));
+    @include border-radius(3px);
+    overflow: hidden;
 
     > li {
       border-right: 1px solid lighten($dark-blue, 40%);
@@ -84,6 +89,7 @@ section.cal {
       float: left;
       width: flex-grid(3) + ((flex-gutter() * 3) / 4);
       background-color: $light-blue;
+      @include box-shadow(inset 0 0 0 1px lighten($light-blue, 8%));
 
       &:hover {
         li.create-module {
@@ -91,6 +97,10 @@ section.cal {
         }
       }
 
+      &:nth-child(4n) {
+        border-right: 0;
+      }
+
       header {
         border-bottom: 1px solid lighten($dark-blue, 40%);
         @include box-shadow(0 2px 2px $light-blue);
@@ -128,6 +138,7 @@ section.cal {
             color: #888;
             border-bottom: 0;
             font-size: 12px;
+            @include box-shadow(none);
           }
         }
       }
@@ -138,9 +149,11 @@ section.cal {
         padding: 0;
 
         li {
-          border-bottom: 1px solid darken($light-blue, 8%);
-          position: relative;
+          border-bottom: 1px solid darken($light-blue, 6%);
+          // @include box-shadow(0 1px 0 lighten($light-blue, 4%));
           overflow: hidden;
+          position: relative;
+          text-shadow: 0 1px 0 #fff;
 
           &:hover {
             background-color: lighten($yellow, 14%);
@@ -314,16 +327,13 @@ section.cal {
   @include box-sizing(border-box);
   opacity: .4;
   @include transition();
-  background: darken($light-blue, 2%);
 
   &:hover {
     opacity: 1;
     width: flex-grid(5) + flex-gutter();
-    background-color: transparent;
 
     + section.main-content {
       width: flex-grid(7);
-      opacity: .6;
     }
   }
 
@@ -340,6 +350,7 @@ section.cal {
       display: block;
 
       li {
+
         ul {
           display: inline;
         }
@@ -351,6 +362,7 @@ section.cal {
     li {
       @include box-sizing(border-box);
       width: 100%;
+      border-right: 0;
 
       &.create-module {
         display: none;

cms/static/sass/_content-types.scss

@@ -53,3 +53,13 @@
   @extend .content-type;
   background-image: url('../img/content-types/chapter.png');
 }
+
+.module a:first-child {
+  @extend .content-type;
+  background-image: url('/static/img/content-types/module.png');
+}
+
+.module a:first-child {
+  @extend .content-type;
+  background-image: url('/static/img/content-types/module.png');
+}

cms/static/sass/_index.scss

+body.index {
+  > header {
+    display: none;
+  }
+
+  > h1 {
+    font-weight: 300;
+    color: lighten($dark-blue, 40%);
+    text-shadow: 0 1px 0 #fff;
+    -webkit-font-smoothing: antialiased;
+    max-width: 600px;
+    text-align: center;
+    margin: 80px auto 30px;
+  }
+
+  section.main-container {
+    border-right: 3px;
+    background: #FFF;
+    max-width: 600px;
+    margin: 0 auto;
+    display: block;
+    @include box-sizing(border-box);
+    border: 1px solid lighten( $dark-blue , 30% );
+    @include border-radius(3px);
+    overflow: hidden;
+    @include bounce-in-animation(.8s);
+
+    header {
+      border-bottom: 1px solid lighten($dark-blue, 50%);
+      @include linear-gradient(#fff, lighten($dark-blue, 62%));
+      @include clearfix();
+      @include box-shadow( 0 2px 0 $light-blue, inset 0 -1px 0 #fff);
+      text-shadow: 0 1px 0 #fff;
+
+      h1 {
+        font-size: 14px;
+        padding: 8px 20px;
+        float: left;
+        color: $dark-blue;
+        margin: 0;
+      }
+
+      a {
+        float: right;
+        padding: 8px 20px;
+        border-left: 1px solid lighten($dark-blue, 50%);
+        @include box-shadow( inset -1px 0 0 #fff);
+        font-weight: bold;
+        font-size: 22px;
+        line-height: 1;
+        color: $dark-blue;
+      }
+    }
+
+    ol {
+      list-style: none;
+      margin: 0;
+      padding: 0;
+
+      li {
+        border-bottom: 1px solid lighten($dark-blue, 50%);
+
+        a {
+          display: block;
+          padding: 10px 20px;
+
+          &:hover {
+            color: $dark-blue;
+            background: lighten($yellow, 10%);
+            text-shadow: 0 1px 0 #fff;
+          }
+        }
+
+        &:last-child {
+          border-bottom: none;
+        }
+      }
+    }
+  }
+}

cms/static/sass/_keyframes.scss

+@mixin bounce-in {
+  0% {
+    opacity: 0;
+    @include transform(scale(.3));
+  }
+
+  50% {
+    opacity: 1;
+    @include transform(scale(1.05));
+  }
+
+  100% {
+    @include transform(scale(1));
+  }
+}
+
+@-moz-keyframes bounce-in { @include bounce-in(); }
+@-webkit-keyframes bounce-in { @include bounce-in(); }
+@-o-keyframes bounce-in { @include bounce-in(); }
+@keyframes bounce-in { @include bounce-in();}
+
+@mixin bounce-in-animation($duration, $timing: ease-in-out) {
+  @include animation-name(bounce-in);
+  @include animation-duration($duration);
+  @include animation-timing-function($timing);
+  @include animation-fill-mode(both);
+}

cms/static/sass/_layout.scss

@@ -2,6 +2,8 @@ body {
   @include clearfix();
   height: 100%;
   font: 14px $body-font-family;
+  background-color: lighten($dark-blue, 62%);
+  background-image: url('/static/img/noise.png');
 
   > section {
     display: table;
@@ -11,28 +13,53 @@ body {
 
   > header {
     background: $dark-blue;
+    @include background-image(url('/static/img/noise.png'), linear-gradient(lighten($dark-blue, 10%), $dark-blue));
+    border-bottom: 1px solid darken($dark-blue, 15%);
+    @include box-shadow(inset 0 -1px 0 lighten($dark-blue, 10%));
+    @include box-sizing(border-box);
     color: #fff;
     display: block;
     float: none;
-    padding: 8px 25px;
+    padding: 0 20px;
+    text-shadow: 0 -1px 0 darken($dark-blue, 15%);
     width: 100%;
-    @include box-sizing(border-box);
-    -webkit-font-smoothing: antialiased;
 
     nav {
       @include clearfix;
 
+      > a {
+        @include hide-text;
+        background: url('/static/img/menu.png') 0 center no-repeat;
+        border-right: 1px solid darken($dark-blue, 10%);
+        @include box-shadow(1px 0 0 lighten($dark-blue, 10%));
+        display: block;
+        float: left;
+        height: 19px;
+        padding: 8px 10px 8px 0;
+        width: 14px;
+
+        &:hover, &:focus {
+          opacity: .7;
+        }
+      }
+
       h2 {
+        border-right: 1px solid darken($dark-blue, 10%);
+        @include box-shadow(1px 0 0 lighten($dark-blue, 10%));
+        float: left;
         font-size: 14px;
+        margin: 0;
         text-transform: uppercase;
-        float: left;
-        margin: 0 15px 0 0;
+        -webkit-font-smoothing: antialiased;
 
         a {
           color: #fff;
+          padding: 8px 20px;
+          display: block;
 
           &:hover {
-            color: rgba(#fff, .6);
+            background-color: rgba(darken($dark-blue, 15%), .5);
+            color: $yellow;
           }
         }
       }
@@ -48,21 +75,35 @@ body {
       ul {
         float: left;
         margin: 0;
+        padding: 0;
+        @include clearfix;
 
         &.user-nav {
           float: right;
+          border-left: 1px solid darken($dark-blue, 10%);
         }
 
         li {
-          @include inline-block();
+          border-right: 1px solid darken($dark-blue, 10%);
+          float: left;
+          @include box-shadow(1px 0 0 lighten($dark-blue, 10%));
 
           a {
-            padding: 8px 10px;
+            padding: 8px 20px;
             display: block;
-            margin: -8px 0;
 
             &:hover {
-              background-color: darken($dark-blue, 15%);
+              background-color: rgba(darken($dark-blue, 15%), .5);
+              color: $yellow;
+            }
+
+            &.new-module {
+              &:before {
+                @include inline-block;
+                content: "+";
+                font-weight: bold;
+                margin-right: 10px;
+              }
             }
           }
         }
@@ -76,8 +117,9 @@ body {
       @include box-sizing(border-box);
       width: flex-grid(9) + flex-gutter();
       float: left;
-      @include box-shadow( -2px 0 0 darken($light-blue, 3%));
+      @include box-shadow( -2px 0 0 lighten($dark-blue, 55%));
       @include transition();
+      background: #FFF;
     }
   }
 }

cms/static/sass/_section.scss

 section#unit-wrapper {
   section.filters {
     @include clearfix;
+    display: none;
     opacity: .4;
     margin-bottom: 10px;
     @include transition;
@@ -52,22 +53,22 @@ section#unit-wrapper {
     display: table;
     border: 1px solid lighten($dark-blue, 40%);
     width: 100%;
+    @include border-radius(3px);
+    @include box-shadow(0 0 4px lighten($dark-blue, 50%));
 
     section {
       header {
         background: #fff;
         padding: 6px;
         border-bottom: 1px solid lighten($dark-blue, 60%);
-        border-top: 1px solid lighten($dark-blue, 60%);
-        margin-top: -1px;
         @include clearfix;
 
         h2 {
           color: $bright-blue;
-          float: left;
-          font-size: 12px;
+          // float: left;
+          font-size: 14px;
           letter-spacing: 1px;
-          line-height: 19px;
+          // line-height: 20px;
           text-transform: uppercase;
           margin: 0;
         }
@@ -172,7 +173,6 @@ section#unit-wrapper {
           padding: 0;
 
           li {
-            border-bottom: 1px solid darken($light-blue, 8%);
             background: $light-blue;
 
             &:last-child {
@@ -181,6 +181,7 @@ section#unit-wrapper {
 
             &.new-module a {
               background-color: darken($light-blue, 2%);
+              border-bottom: 1px solid darken($light-blue, 8%);
 
               &:hover {
                 background-color: lighten($yellow, 10%);
@@ -199,6 +200,7 @@ section#unit-wrapper {
               li {
                 padding: 6px;
                 border-collapse: collapse;
+                border-bottom: 1px solid darken($light-blue, 8%);
                 position: relative;
 
                 &:last-child {

cms/static/sass/_unit.scss

 section#unit-wrapper {
   > header {
-    border-bottom: 2px solid $dark-blue;
+    border-bottom: 1px solid lighten($dark-blue, 50%);
+    @include linear-gradient(#fff, lighten($dark-blue, 62%));
     @include clearfix();
-    @include box-shadow( 0 2px 0 darken($light-blue, 3%));
-    padding: 6px 20px;
+    @include box-shadow( 0 2px 0 $light-blue, inset 0 -1px 0 #fff);
+    text-shadow: 0 1px 0 #fff;
 
     section {
       float: left;
+      padding: 10px 20px;
 
       h1 {
-        font-size: 16px;
-        text-transform: uppercase;
-        letter-spacing: 1px;
+        font-size: 18px;
         @include inline-block();
-        color: $bright-blue;
+        color: $dark-blue;
         margin: 0;
       }
 
@@ -22,32 +22,41 @@ section#unit-wrapper {
         margin: 0;
 
         a {
-          text-indent: -9999px;
           @include inline-block();
-          width: 1px;
-          height: 100%;
+          font-size: 12px;
        }
       }
     }
 
     div {
-      float: right;
+      @include clearfix;
       color: #666;
+      float: right;
+      padding: 6px 20px;
 
       a {
-        display: block;
         @include inline-block;
 
-       &.cancel {
-         margin-right: 20px;
-         font-style: italic;
-         font-size: 12px;
-       }
+        &.cancel {
+          margin-right: 20px;
+          font-style: italic;
+          font-size: 12px;
+          padding: 6px 0;
+        }
 
-       &.save-update {
-         @extend .button;
-         margin: -6px -21px -6px 0;
-       }
+        &.save-update {
+          padding: 6px 20px;
+          @include border-radius(3px);
+          border: 1px solid lighten($dark-blue, 40%);
+          @include box-shadow(inset 0 0 0 1px #fff);
+          color: $dark-blue;
+          @include linear-gradient(lighten($dark-blue, 60%), lighten($dark-blue, 55%));
+
+          &:hover, &:focus {
+            @include linear-gradient(lighten($dark-blue, 58%), lighten($dark-blue, 53%));
+            @include box-shadow(inset 0 0 6px 1px #fff);
+          }
+        }
       }
     }
   }

cms/static/sass/base-style.scss

 @import 'bourbon/bourbon';
 @import 'vendor/normalize';
+@import 'keyframes';
 
 @import 'base', 'layout', 'content-types';
 @import 'calendar';
-@import 'section', 'unit';
+@import 'section', 'unit', 'index';
 
 @import 'module/module-styles.scss';

cms/templates/base.html

@@ -15,7 +15,7 @@
     <meta name="path_prefix" content="${MITX_ROOT_URL}">
   </head>
 
-  <body>
+  <body class="<%block name='bodyclass'></%block>">
 
     <%include file="widgets/header.html"/>
     <%include file="courseware_vendor_js.html"/>

cms/templates/course_index.html

 <%inherit file="base.html" />
 <%block name="title">Course Manager</%block>
+<%include file="widgets/header.html"/>
 
 <%block name="content">
 <section class="main-container">

cms/templates/index.html

 <%inherit file="base.html" />
+<%block name="bodyclass">index</%block>
 <%block name="title">Courses</%block>
 
 <%block name="content">
+<h1>edX Course Management</h1>
+
 <section class="main-container">
+  <header>
+    <h1>Courses</h1>
+    <a href="#" class="wip">+</a>
+  </header>
+
   <ol>
     %for course, url in courses:
     <li><a href="${url}">${course}</a></li>

cms/templates/widgets/header.html

 <%! from django.core.urlresolvers import reverse %>
 <header>
   <nav>
-    <h2><a href="/">edX CMS: TODO:-course-name-here</a></h2>
+    <a href="/">Home</a>
+    <h2><a href="#">edX CMS: TODO:-course-name-here</a></h2>
     <ul>
       <li>
-        <a href="#" class="new-module wip">New Module</a>
+        <a href="#" class="new-module wip">Module</a>
       </li>
       <li>
-        <a href="#" class="new-module wip">New Unit</a>
+        <a href="#" class="new-module wip">Unit</a>
       </li>
     </ul>
 

common/djangoapps/student/views.py

@@ -30,6 +30,7 @@ from django_future.csrf import ensure_csrf_cookie
 from student.models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment
 from util.cache import cache_if_anonymous
 from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 

common/lib/xmodule/xmodule/backcompat_module.py

@@ -5,6 +5,7 @@ from x_module import XModuleDescriptor
 from lxml import etree
 from functools import wraps
 import logging
+import traceback
 
 log = logging.getLogger(__name__)
 
@@ -12,8 +13,8 @@ log = logging.getLogger(__name__)
 def process_includes(fn):
     """
     Wraps a XModuleDescriptor.from_xml method, and modifies xml_data to replace
-    any immediate child <include> items with the contents of the file that they are
-    supposed to include
+    any immediate child <include> items with the contents of the file that they
+    are supposed to include
     """
     @wraps(fn)
     def from_xml(cls, xml_data, system, org=None, course=None):
@@ -21,23 +22,31 @@ def process_includes(fn):
         next_include = xml_object.find('include')
         while next_include is not None:
             file = next_include.get('file')
-            if file is not None:
-                try:
-                    ifp = system.resources_fs.open(file)
-                except Exception:
-                    log.exception('Error in problem xml include: %s' % (etree.tostring(next_include, pretty_print=True)))
-                    log.exception('Cannot find file %s in %s' % (file, dir))
-                    raise
-                try:
-                    # read in and convert to XML
-                    incxml = etree.XML(ifp.read())
-                except Exception:
-                    log.exception('Error in problem xml include: %s' % (etree.tostring(next_include, pretty_print=True)))
-                    log.exception('Cannot parse XML in %s' % (file))
-                    raise
+            parent = next_include.getparent()
+
+            if file is None:
+                continue
+
+            try:
+                ifp = system.resources_fs.open(file)
+                # read in and convert to XML
+                incxml = etree.XML(ifp.read())
+
                 # insert  new XML into tree in place of inlcude
-                parent = next_include.getparent()
                 parent.insert(parent.index(next_include), incxml)
+            except Exception:
+                msg = "Error in problem xml include: %s" % (etree.tostring(next_include, pretty_print=True))
+                log.exception(msg)
+                parent = next_include.getparent()
+
+                errorxml = etree.Element('error')
+                messagexml = etree.SubElement(errorxml, 'message')
+                messagexml.text = msg
+                stackxml = etree.SubElement(errorxml, 'stacktrace')
+                stackxml.text = traceback.format_exc()
+
+                # insert error XML in place of include
+                parent.insert(parent.index(next_include), errorxml)
             parent.remove(next_include)
 
             next_include = xml_object.find('include')
@@ -50,8 +59,8 @@ class SemanticSectionDescriptor(XModuleDescriptor):
     @process_includes
     def from_xml(cls, xml_data, system, org=None, course=None):
         """
-        Removes sections single child elements in favor of just embedding the child element
-
+        Removes sections with single child elements in favor of just embedding
+        the child element
         """
         xml_object = etree.fromstring(xml_data)
 
@@ -76,7 +85,6 @@ class TranslateCustomTagDescriptor(XModuleDescriptor):
         xml_object = etree.fromstring(xml_data)
         tag = xml_object.tag
         xml_object.tag = 'customtag'
-        impl = etree.SubElement(xml_object, 'impl')
-        impl.text = tag
+        xml_object.attrib['impl'] = tag
 
         return system.process_xml(etree.tostring(xml_object))

common/lib/xmodule/xmodule/capa_module.py

@@ -67,7 +67,8 @@ class ComplexEncoder(json.JSONEncoder):
 
 class CapaModule(XModule):
     '''
-    An XModule implementing LonCapa format problems, implemented by way of capa.capa_problem.LoncapaProblem
+    An XModule implementing LonCapa format problems, implemented by way of
+    capa.capa_problem.LoncapaProblem
     '''
     icon_class = 'problem'
 
@@ -77,8 +78,10 @@ class CapaModule(XModule):
     js_module_name = "Problem"
     css = {'scss': [resource_string(__name__, 'css/capa/display.scss')]}
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition, instance_state=None,
+                 shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, instance_state,
+                         shared_state, **kwargs)
 
         self.attempts = 0
         self.max_attempts = None
@@ -133,7 +136,8 @@ class CapaModule(XModule):
             seed = None
 
         try:
-            self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(), instance_state, seed=seed, system=self.system)
+            self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(),
+                                      instance_state, seed=seed, system=self.system)
         except Exception:
             msg = 'cannot create LoncapaProblem %s' % self.location.url()
             log.exception(msg)
@@ -141,15 +145,20 @@ class CapaModule(XModule):
                 msg = '<p>%s</p>' % msg.replace('<', '&lt;')
                 msg += '<p><pre>%s</pre></p>' % traceback.format_exc().replace('<', '&lt;')
                 # create a dummy problem with error message instead of failing
-                problem_text = '<problem><text><font color="red" size="+2">Problem %s has an error:</font>%s</text></problem>' % (self.location.url(), msg)
-                self.lcp = LoncapaProblem(problem_text, self.location.html_id(), instance_state, seed=seed, system=self.system)
+                problem_text = ('<problem><text><font color="red" size="+2">'
+                                'Problem %s has an error:</font>%s</text></problem>' %
+                                (self.location.url(), msg))
+                self.lcp = LoncapaProblem(
+                    problem_text, self.location.html_id(),
+                    instance_state, seed=seed, system=self.system)
             else:
                 raise
 
     @property
     def rerandomize(self):
         """
-        Property accessor that returns self.metadata['rerandomize'] in a canonical form
+        Property accessor that returns self.metadata['rerandomize'] in a
+        canonical form
         """
         rerandomize = self.metadata.get('rerandomize', 'always')
         if rerandomize in ("", "always", "true"):
@@ -203,7 +212,10 @@ class CapaModule(XModule):
         except Exception, err:
             if self.system.DEBUG:
                 log.exception(err)
-                msg = '[courseware.capa.capa_module] <font size="+1" color="red">Failed to generate HTML for problem %s</font>' % (self.location.url())
+                msg = (
+                    '[courseware.capa.capa_module] <font size="+1" color="red">'
+                    'Failed to generate HTML for problem %s</font>' %
+                    (self.location.url()))
                 msg += '<p>Error:</p><p><pre>%s</pre></p>' % str(err).replace('<', '&lt;')
                 msg += '<p><pre>%s</pre></p>' % traceback.format_exc().replace('<', '&lt;')
                 html = msg
@@ -215,8 +227,8 @@ class CapaModule(XModule):
                    'weight': self.weight,
                   }
 
-        # We using strings as truthy values, because the terminology of the check button
-        # is context-specific.
+        # We using strings as truthy values, because the terminology of the
+        # check button is context-specific.
         check_button = "Grade" if self.max_attempts else "Check"
         reset_button = True
         save_button = True
@@ -242,7 +254,8 @@ class CapaModule(XModule):
         if not self.lcp.done:
             reset_button = False
 
-        # We don't need a "save" button if infinite number of attempts and non-randomized
+        # We don't need a "save" button if infinite number of attempts and
+        # non-randomized
         if self.max_attempts is None and self.rerandomize != "always":
             save_button = False
 
@@ -517,11 +530,13 @@ class CapaModule(XModule):
 
         self.lcp.do_reset()
         if self.rerandomize == "always":
-            # reset random number generator seed (note the self.lcp.get_state() in next line)
+            # reset random number generator seed (note the self.lcp.get_state()
+            # in next line)
             self.lcp.seed = None
 
         self.lcp = LoncapaProblem(self.definition['data'],
-                                  self.location.html_id(), self.lcp.get_state(), system=self.system)
+                                  self.location.html_id(), self.lcp.get_state(),
+                                  system=self.system)
 
         event_info['new_state'] = self.lcp.get_state()
         self.system.track_function('reset_problem', event_info)
@@ -537,6 +552,7 @@ class CapaDescriptor(RawDescriptor):
 
     module_class = CapaModule
 
+    # VS[compat]
     # TODO (cpennington): Delete this method once all fall 2012 course are being
     # edited in the cms
     @classmethod
@@ -545,3 +561,7 @@ class CapaDescriptor(RawDescriptor):
             'problems/' + path[8:],
             path[8:],
         ]
+    @classmethod
+    def split_to_file(cls, xml_object):
+        '''Problems always written in their own files'''
+        return True

common/lib/xmodule/xmodule/course_module.py

@@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
 
 class CourseDescriptor(SequenceDescriptor):
     module_class = SequenceModule
+    metadata_attributes = SequenceDescriptor.metadata_attributes + ('org', 'course')
 
     def __init__(self, system, definition=None, **kwargs):
         super(CourseDescriptor, self).__init__(system, definition, **kwargs)
@@ -17,23 +18,40 @@ class CourseDescriptor(SequenceDescriptor):
         try:
             self.start = time.strptime(self.metadata["start"], "%Y-%m-%dT%H:%M")
         except KeyError:
-            self.start = time.gmtime(0)  # The epoch
-            log.critical("Course loaded without a start date. " + str(self.id))
-        except ValueError, e:
-            self.start = time.gmtime(0)  # The epoch
-            log.critical("Course loaded with a bad start date. " + str(self.id) + " '" + str(e) + "'")
+            self.start = time.gmtime(0) #The epoch
+            log.critical("Course loaded without a start date. %s", self.id)
+        except ValueError as e:
+            self.start = time.gmtime(0) #The epoch
+            log.critical("Course loaded with a bad start date. %s '%s'",
+                         self.id, e)
 
     def has_started(self):
         return time.gmtime() > self.start
 
-    @classmethod
-    def id_to_location(cls, course_id):
+    @staticmethod
+    def id_to_location(course_id):
+        '''Convert the given course_id (org/course/name) to a location object.
+        Throws ValueError if course_id is of the wrong format.
+        '''
         org, course, name = course_id.split('/')
         return Location('i4x', org, course, 'course', name)
 
+    @staticmethod
+    def location_to_id(location):
+        '''Convert a location of a course to a course_id.  If location category
+        is not "course", raise a ValueError.
+
+        location: something that can be passed to Location
+        '''
+        loc = Location(location)
+        if loc.category != "course":
+            raise ValueError("{0} is not a course location".format(loc))
+        return "/".join([loc.org, loc.course, loc.name])
+
+
     @property
     def id(self):
-        return "/".join([self.location.org, self.location.course, self.location.name])
+        return self.location_to_id(self.location)
 
     @property
     def start_date_text(self):

common/lib/xmodule/xmodule/errorhandlers.py

+import logging
+import sys
+
+log = logging.getLogger(__name__)
+
+def in_exception_handler():
+    '''Is there an active exception?'''
+    return sys.exc_info() != (None, None, None)
+
+def strict_error_handler(msg, exc_info=None):
+    '''
+    Do not let errors pass.  If exc_info is not None, ignore msg, and just
+    re-raise.  Otherwise, check if we are in an exception-handling context.
+    If so, re-raise.  Otherwise, raise Exception(msg).
+
+    Meant for use in validation, where any errors should trap.
+    '''
+    if exc_info is not None:
+        raise exc_info[0], exc_info[1], exc_info[2]
+
+    if in_exception_handler():
+        raise
+
+    raise Exception(msg)
+
+
+def logging_error_handler(msg, exc_info=None):
+    '''Log all errors, but otherwise let them pass, relying on the caller to
+    workaround.'''
+    if exc_info is not None:
+        log.exception(msg, exc_info=exc_info)
+        return
+
+    if in_exception_handler():
+        log.exception(msg)
+        return
+
+    log.error(msg)
+
+
+def ignore_errors_handler(msg, exc_info=None):
+    '''Ignore all errors, relying on the caller to workaround.
+    Meant for use in the LMS, where an error in one part of the course
+    shouldn't bring down the whole system'''
+    pass

common/lib/xmodule/xmodule/exceptions.py

 class InvalidDefinitionError(Exception):
     pass
 
+
 class NotFoundError(Exception):
     pass

common/lib/xmodule/xmodule/html_module.py

@@ -12,8 +12,10 @@ class HtmlModule(XModule):
     def get_html(self):
         return self.html
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition,
+                 instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition,
+                         instance_state, shared_state, **kwargs)
         self.html = self.definition['data']
 
 
@@ -42,3 +44,8 @@ class HtmlDescriptor(RawDescriptor):
     def file_to_xml(cls, file_object):
         parser = etree.HTMLParser()
         return etree.parse(file_object, parser).getroot()
+
+    @classmethod
+    def split_to_file(cls, xml_object):
+        # never include inline html
+        return True

common/lib/xmodule/xmodule/js/src/capa/schematic.js

@@ -52,6 +52,7 @@ function update_schematics() {
 	    schematics[i].setAttribute("loaded","true");
 	}
 }
+window.update_schematics = update_schematics;
 
 // add ourselves to the tasks that get performed when window is loaded
 function add_schematic_handler(other_onload) {

common/lib/xmodule/xmodule/mako_module.py

@@ -2,9 +2,12 @@ from x_module import XModuleDescriptor, DescriptorSystem
 
 
 class MakoDescriptorSystem(DescriptorSystem):
-    def __init__(self, render_template, *args, **kwargs):
+    def __init__(self, load_item, resources_fs, error_handler,
+                 render_template):
+        super(MakoDescriptorSystem, self).__init__(
+            load_item, resources_fs, error_handler)
+
         self.render_template = render_template
-        super(MakoDescriptorSystem, self).__init__(*args, **kwargs)
 
 
 class MakoModuleDescriptor(XModuleDescriptor):
@@ -19,7 +22,9 @@ class MakoModuleDescriptor(XModuleDescriptor):
 
     def __init__(self, system, definition=None, **kwargs):
         if getattr(system, 'render_template', None) is None:
-            raise TypeError('{system} must have a render_template function in order to use a MakoDescriptor'.format(system=system))
+            raise TypeError('{system} must have a render_template function'
+                            ' in order to use a MakoDescriptor'.format(
+                    system=system))
         super(MakoModuleDescriptor, self).__init__(system, definition, **kwargs)
 
     def get_context(self):
@@ -29,4 +34,5 @@ class MakoModuleDescriptor(XModuleDescriptor):
         return {'module': self}
 
     def get_html(self):
-        return self.system.render_template(self.mako_template, self.get_context())
+        return self.system.render_template(
+            self.mako_template, self.get_context())

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -45,13 +45,28 @@ class Location(_LocationBase):
         """
         return re.sub('_+', '_', INVALID_CHARS.sub('_', value))
 
-    def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None, name=None, revision=None):
+    @classmethod
+    def is_valid(cls, value):
+        '''
+        Check if the value is a valid location, in any acceptable format.
+        '''
+        try:
+            Location(value)
+        except InvalidLocationError:
+            return False
+        return True
+
+    def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None,
+                name=None, revision=None):
         """
         Create a new location that is a clone of the specifed one.
 
         location - Can be any of the following types:
-            string: should be of the form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+            string: should be of the form
+                    {tag}://{org}/{course}/{category}/{name}[/{revision}]
+
             list: should be of the form [tag, org, course, category, name, revision]
+
             dict: should be of the form {
                 'tag': tag,
                 'org': org,
@@ -62,16 +77,19 @@ class Location(_LocationBase):
             }
             Location: another Location object
 
-        In both the dict and list forms, the revision is optional, and can be ommitted.
+        In both the dict and list forms, the revision is optional, and can be
+        ommitted.
 
-        Components must be composed of alphanumeric characters, or the characters '_', '-', and '.'
+        Components must be composed of alphanumeric characters, or the
+        characters '_', '-', and '.'
 
-        Components may be set to None, which may be interpreted by some contexts to mean
-        wildcard selection
+        Components may be set to None, which may be interpreted by some contexts
+        to mean wildcard selection
         """
 
 
-        if org is None and course is None and category is None and name is None and revision is None:
+        if (org is None and course is None and category is None and
+            name is None and revision is None):
             location = loc_or_tag
         else:
             location = (loc_or_tag, org, course, category, name, revision)
@@ -131,9 +149,11 @@ class Location(_LocationBase):
 
     def html_id(self):
         """
-        Return a string with a version of the location that is safe for use in html id attributes
+        Return a string with a version of the location that is safe for use in
+        html id attributes
         """
-        return "-".join(str(v) for v in self.list() if v is not None).replace('.', '_')
+        return "-".join(str(v) for v in self.list()
+                        if v is not None).replace('.', '_')
 
     def dict(self):
         """
@@ -154,7 +174,8 @@ class Location(_LocationBase):
 
 class ModuleStore(object):
     """
-    An abstract interface for a database backend that stores XModuleDescriptor instances
+    An abstract interface for a database backend that stores XModuleDescriptor
+    instances
     """
     def get_item(self, location, depth=0):
         """
@@ -164,13 +185,16 @@ class ModuleStore(object):
 
         If any segment of the location is None except revision, raises
             xmodule.modulestore.exceptions.InsufficientSpecificationError
-        If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError
+
+        If no object is found at that location, raises
+            xmodule.modulestore.exceptions.ItemNotFoundError
 
         location: Something that can be passed to Location
 
-        depth (int): An argument that some module stores may use to prefetch descendents of the queried modules
-            for more efficient results later in the request. The depth is counted in the number of
-            calls to get_children() to cache. None indicates to cache all descendents
+        depth (int): An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of calls to
+            get_children() to cache. None indicates to cache all descendents
         """
         raise NotImplementedError
 
@@ -182,9 +206,10 @@ class ModuleStore(object):
 
         location: Something that can be passed to Location
 
-        depth: An argument that some module stores may use to prefetch descendents of the queried modules
-            for more efficient results later in the request. The depth is counted in the number of calls
-            to get_children() to cache. None indicates to cache all descendents
+        depth: An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of calls to
+            get_children() to cache. None indicates to cache all descendents
         """
         raise NotImplementedError
 
@@ -229,3 +254,25 @@ class ModuleStore(object):
         '''
         raise NotImplementedError
 
+    def path_to_location(self, location, course=None, chapter=None, section=None):
+        '''
+        Try to find a course/chapter/section[/position] path to this location.
+
+        raise ItemNotFoundError if the location doesn't exist.
+
+        If course, chapter, section are not None, restrict search to paths with those
+            components as specified.
+            
+        raise NoPathToItem if the location exists, but isn't accessible via
+        a path that matches the course/chapter/section restrictions.
+
+        In general, a location may be accessible via many paths. This method may
+        return any valid path.
+
+        Return a tuple (course, chapter, section, position).
+
+        If the section a sequence, position should be the position of this location
+        in that sequence.  Otherwise, position should be None.
+        '''
+        raise NotImplementedError
+

common/lib/xmodule/xmodule/modulestore/exceptions.py

@@ -13,3 +13,11 @@ class InsufficientSpecificationError(Exception):
 
 class InvalidLocationError(Exception):
     pass
+
+
+class NoPathToItem(Exception):
+    pass
+
+
+class DuplicateItemError(Exception):
+    pass

common/lib/xmodule/xmodule/modulestore/tests/test_location.py

@@ -13,14 +13,51 @@ def test_string_roundtrip():
     check_string_roundtrip("tag://org/course/category/name/revision")
 
 
+input_dict = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name',
+    'org': 'org'
+}
+
+input_list = ['tag', 'org', 'course', 'category', 'name']
+
+input_str = "tag://org/course/category/name"
+input_str_rev = "tag://org/course/category/name/revision"
+
+valid = (input_list, input_dict, input_str, input_str_rev)
+
+invalid_dict = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name/more_name',
+    'org': 'org'
+}
+
+invalid_dict2 = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name ',   # extra space
+    'org': 'org'
+}
+
+invalid = ("foo", ["foo"], ["foo", "bar"],
+           ["foo", "bar", "baz", "blat", "foo/bar"],
+           "tag://org/course/category/name with spaces/revision",
+           invalid_dict,
+           invalid_dict2)
+
+def test_is_valid():
+    for v in valid:
+        assert_equals(Location.is_valid(v), True)
+
+    for v in invalid:
+        assert_equals(Location.is_valid(v), False)
+
 def test_dict():
-    input_dict = {
-        'tag': 'tag',
-        'course': 'course',
-        'category': 'category',
-        'name': 'name',
-        'org': 'org'
-    }
     assert_equals("tag://org/course/category/name", Location(input_dict).url())
     assert_equals(dict(revision=None, **input_dict), Location(input_dict).dict())
 
@@ -30,7 +67,6 @@ def test_dict():
 
 
 def test_list():
-    input_list = ['tag', 'org', 'course', 'category', 'name']
     assert_equals("tag://org/course/category/name", Location(input_list).url())
     assert_equals(input_list + [None], Location(input_list).list())
 
@@ -65,3 +101,13 @@ def test_equality():
         Location('tag', 'org', 'course', 'category', 'name1'),
         Location('tag', 'org', 'course', 'category', 'name')
     )
+
+def test_clean():
+    pairs = [ ('',''),
+              (' ', '_'),
+              ('abc,', 'abc_'),
+              ('ab    fg!@//\\aj', 'ab_fg_aj'),
+              (u"ab\xA9", "ab_"),  # no unicode allowed for now
+              ]
+    for input, output in pairs:
+        assert_equals(Location.clean(input), output)

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

+import pymongo
+
+from nose.tools import assert_equals, assert_raises, assert_not_equals, with_setup
+from path import path
+from pprint import pprint
+
+from xmodule.modulestore import Location
+from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
+from xmodule.modulestore.mongo import MongoModuleStore
+from xmodule.modulestore.xml_importer import import_from_xml
+
+# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
+# to   ~/mitx_all/mitx/common/test
+TEST_DIR = path(__file__).abspath().dirname()
+for i in range(5):
+    TEST_DIR = TEST_DIR.dirname()
+TEST_DIR = TEST_DIR / 'test'
+
+DATA_DIR = TEST_DIR / 'data'
+
+
+HOST = 'localhost'
+PORT = 27017
+DB = 'test'
+COLLECTION = 'modulestore'
+FS_ROOT = DATA_DIR  # TODO (vshnayder): will need a real fs_root for testing load_item
+DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
+
+
+class TestMongoModuleStore(object):
+
+    @classmethod
+    def setupClass(cls):
+        cls.connection = pymongo.connection.Connection(HOST, PORT)
+        cls.connection.drop_database(DB)
+
+        # NOTE: Creating a single db for all the tests to save time.  This
+        # is ok only as long as none of the tests modify the db.
+        # If (when!) that changes, need to either reload the db, or load
+        # once and copy over to a tmp db for each test.
+        cls.store = cls.initdb()
+
+    @classmethod
+    def teardownClass(cls):
+        pass
+
+    @staticmethod
+    def initdb():
+        # connect to the db
+        store = MongoModuleStore(HOST, DB, COLLECTION, FS_ROOT, default_class=DEFAULT_CLASS)
+        # Explicitly list the courses to load (don't want the big one)
+        courses = ['toy', 'simple']
+        import_from_xml(store, DATA_DIR, courses)
+        return store
+
+    @staticmethod
+    def destroy_db(connection):
+        # Destroy the test db.
+        connection.drop_database(DB)
+
+    def setUp(self):
+        # make a copy for convenience
+        self.connection = TestMongoModuleStore.connection
+
+    def tearDown(self):
+        pass
+
+    def test_init(self):
+        '''Make sure the db loads, and print all the locations in the db.
+        Call this directly from failing tests to see what's loaded'''
+        ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True}))
+
+        pprint([Location(i['_id']).url() for i in ids])
+
+    def test_get_courses(self):
+        '''Make sure the course objects loaded properly'''
+        courses = self.store.get_courses()
+        assert_equals(len(courses), 2)
+        courses.sort(key=lambda c: c.id)
+        assert_equals(courses[0].id, 'edX/simple/2012_Fall')
+        assert_equals(courses[1].id, 'edX/toy/2012_Fall')
+
+    def test_loads(self):
+        assert_not_equals(
+            self.store.get_item("i4x://edX/toy/course/2012_Fall"),
+            None)
+
+        assert_not_equals(
+            self.store.get_item("i4x://edX/simple/course/2012_Fall"),
+            None)
+
+        assert_not_equals(
+            self.store.get_item("i4x://edX/toy/video/Welcome"),
+            None)
+
+
+
+    def test_find_one(self):
+        assert_not_equals(
+            self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
+            None)
+
+        assert_not_equals(
+            self.store._find_one(Location("i4x://edX/simple/course/2012_Fall")),
+            None)
+
+        assert_not_equals(
+            self.store._find_one(Location("i4x://edX/toy/video/Welcome")),
+            None)
+
+    def test_path_to_location(self):
+        '''Make sure that path_to_location works'''
+        should_work = (
+            ("i4x://edX/toy/video/Welcome",
+             ("edX/toy/2012_Fall", "Overview", "Welcome", None)),
+            ("i4x://edX/toy/html/toylab",
+             ("edX/toy/2012_Fall", "Overview", "Toy_Videos", None)),
+            )
+        for location, expected in should_work:
+            assert_equals(self.store.path_to_location(location), expected)
+
+        not_found = (
+            "i4x://edX/toy/video/WelcomeX",
+            )
+        for location in not_found:
+            assert_raises(ItemNotFoundError, self.store.path_to_location, location)
+
+        # Since our test files are valid, there shouldn't be any
+        # elements with no path to them.  But we can look for them in
+        # another course.
+        no_path = (
+            "i4x://edX/simple/video/Lost_Video",
+            )
+        for location in no_path:
+            assert_raises(NoPathToItem, self.store.path_to_location, location, "toy")
+

cms/djangoapps/contentstore/__init__.py → common/lib/xmodule/xmodule/modulestore/xml_importer.py

-from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.xml import XMLModuleStore
 import logging
 
+from .xml import XMLModuleStore
+from .exceptions import DuplicateItemError
+
 log = logging.getLogger(__name__)
 
 
-def import_from_xml(data_dir, course_dirs=None):
+def import_from_xml(store, data_dir, course_dirs=None, eager=True,
+                    default_class='xmodule.raw_module.RawDescriptor'):
     """
-    Import the specified xml data_dir into the django defined modulestore,
+    Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
+
+    course_dirs: If specified, the list of course_dirs to load. Otherwise, load
+    all course dirs
+
     """
     module_store = XMLModuleStore(
         data_dir,
-        default_class='xmodule.raw_module.RawDescriptor',
-        eager=True,
+        default_class=default_class,
+        eager=eager,
         course_dirs=course_dirs
     )
     for module in module_store.modules.itervalues():
@@ -21,14 +27,14 @@ def import_from_xml(data_dir, course_dirs=None):
         # TODO (cpennington): This forces import to overrite the same items.
         # This should in the future create new revisions of the items on import
         try:
-            modulestore().create_item(module.location)
-        except:
+            store.create_item(module.location)
+        except DuplicateItemError:
             log.exception('Item already exists at %s' % module.location.url())
             pass
         if 'data' in module.definition:
-            modulestore().update_item(module.location, module.definition['data'])
+            store.update_item(module.location, module.definition['data'])
         if 'children' in module.definition:
-            modulestore().update_children(module.location, module.definition['children'])
-        modulestore().update_metadata(module.location, dict(module.metadata))
+            store.update_children(module.location, module.definition['children'])
+        store.update_metadata(module.location, dict(module.metadata))
 
     return module_store

common/lib/xmodule/xmodule/raw_module.py

@@ -6,9 +6,10 @@ import logging
 
 log = logging.getLogger(__name__)
 
+
 class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
     """
-    Module that provides a raw editing view of it's data and children
+    Module that provides a raw editing view of its data and children
     """
     mako_template = "widgets/raw-edit.html"
 
@@ -31,8 +32,11 @@ class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
         except etree.XMLSyntaxError as err:
             lines = self.definition['data'].split('\n')
             line, offset = err.position
-            log.exception("Unable to create xml for problem {loc}. Context: '{context}'".format(
-                context=lines[line-1][offset - 40:offset + 40],
-                loc=self.location
-            ))
+            msg = ("Unable to create xml for problem {loc}. "
+                   "Context: '{context}'".format(
+                    context=lines[line - 1][offset - 40:offset + 40],
+                    loc=self.location))
+            log.exception(msg)
+            self.system.error_handler(msg)
+            # no workaround possible, so just re-raise
             raise

common/lib/xmodule/xmodule/seq_module.py

@@ -20,12 +20,15 @@ class_priority = ['video', 'problem']
 class SequenceModule(XModule):
     ''' Layout module which lays out content in a temporal sequence
     '''
-    js = {'coffee': [resource_string(__name__, 'js/src/sequence/display.coffee')]}
+    js = {'coffee': [resource_string(__name__,
+                                     'js/src/sequence/display.coffee')]}
     css = {'scss': [resource_string(__name__, 'css/sequence/display.scss')]}
     js_module_name = "Sequence"
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition, instance_state=None,
+                 shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition,
+                         instance_state, shared_state, **kwargs)
         self.position = 1
 
         if instance_state is not None:
@@ -92,7 +95,8 @@ class SequenceModule(XModule):
         self.rendered = True
 
     def get_icon_class(self):
-        child_classes = set(child.get_icon_class() for child in self.get_children())
+        child_classes = set(child.get_icon_class()
+                            for child in self.get_children())
         new_class = 'other'
         for c in class_priority:
             if c in child_classes:
@@ -114,5 +118,20 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
     def definition_to_xml(self, resource_fs):
         xml_object = etree.Element('sequential')
         for child in self.get_children():
-            xml_object.append(etree.fromstring(child.export_to_xml(resource_fs)))
+            xml_object.append(
+                etree.fromstring(child.export_to_xml(resource_fs)))
         return xml_object
+
+    @classmethod
+    def split_to_file(cls, xml_object):
+        # Note: if we end up needing subclasses, can port this logic there.
+        yes = ('chapter',)
+        no = ('course',)
+
+        if xml_object.tag in yes:
+            return True
+        elif xml_object.tag in no:
+            return False
+
+        # otherwise maybe--delegate to superclass.
+        return XmlDescriptor.split_to_file(xml_object)

common/lib/xmodule/xmodule/template_module.py

@@ -21,19 +21,23 @@ class CustomTagModule(XModule):
 
     course.xml::
         ...
-        <customtag page="234"><impl>book</impl></customtag>
+        <customtag page="234" impl="book"/>
         ...
 
     Renders to::
         More information given in <a href="/book/234">the text</a>
     """
 
-    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
-        XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
+    def __init__(self, system, location, definition,
+                 instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition,
+                         instance_state, shared_state, **kwargs)
+
         xmltree = etree.fromstring(self.definition['data'])
-        template_name = xmltree.find('impl').text
+        template_name = xmltree.attrib['impl']
         params = dict(xmltree.items())
-        with self.system.filestore.open('custom_tags/{name}'.format(name=template_name)) as template:
+        with self.system.filestore.open(
+                'custom_tags/{name}'.format(name=template_name)) as template:
             self.html = Template(template.read()).render(**params)
 
     def get_html(self):

common/lib/xmodule/xmodule/video_module.py

@@ -60,7 +60,7 @@ class VideoModule(XModule):
         return None
 
     def get_instance_state(self):
-        log.debug(u"STATE POSITION {0}".format(self.position))
+        #log.debug(u"STATE POSITION {0}".format(self.position))
         return json.dumps({'position': self.position})
 
     def video_list(self):

common/test/data/full/chapter/Overview.xml

@@ -3,6 +3,7 @@
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
     <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_5555" filename="html_5555"/>
     <problem filename="Lab_0_Using_the_Tools" slug="Lab_0_Using_the_Tools" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Lab 0: Using the Tools"/>
   </vertical>
   <problem filename="Circuit_Sandbox" slug="Circuit_Sandbox" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Circuit Sandbox"/>

common/test/data/full/course.xml

-<course filename="6.002_Spring_2012" slug="6.002_Spring_2012" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="6.002 Spring 2012"/>
+<course filename="6.002_Spring_2012" slug="6.002_Spring_2012" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="6.002 Spring 2012" start="2015-07-17T12:00" course="full" org="edx"/>

common/test/data/full/course/6.002_Spring_2012.xml

@@ -6,7 +6,7 @@
       <vertical slug="vertical_1122" graceperiod="0 day 0 hours 5 minutes 0 seconds" showanswer="attempted" rerandomize="per_student" due="April 30, 12:00" graded="true">
         <html filename="Midterm_Exam_1123" slug="Midterm_Exam_1123" graceperiod="0 day 0 hours 5 minutes 0 seconds" rerandomize="per_student" due="April 30, 12:00" graded="true" name="Midterm Exam"/>
       </vertical>
-      <vertical filename="vertical_1124" slug="vertical_1124" graceperiod="0 day 0 hours 5 minutes 0 seconds" showanswer="attempted" rerandomize="per_student" due="April 30, 12:00" graded="true"/>
+      <vertical filename="vertical_98" slug="vertical_1124" graceperiod="0 day 0 hours 5 minutes 0 seconds" showanswer="attempted" rerandomize="per_student" due="April 30, 12:00" graded="true"/>
     </sequential>
   </chapter>
 </sequential>

common/test/data/full/html/html_5555.html

+    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>

common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml

@@ -2,17 +2,13 @@
   <vertical filename="vertical_58" slug="vertical_58" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
-    <customtag tag="S1E3" slug="discuss_67">
-      <impl>discuss</impl>
-    </customtag>
+    <customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
     <html slug="html_68"> S1E4 has been removed. </html>
   </vertical>
   <vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <video youtube="0.75:XNh13VZhThQ,1.0:XbDRmF6J0K0,1.25:JDty12WEQWk,1.50:wELKGj-5iyM" slug="What_s_next" name="What's next"/>
     <html slug="html_95">Minor correction: Six elements (five resistors)</html>
-    <customtag tag="S1" slug="discuss_96">
-      <impl>discuss</impl>
-    </customtag>
+    <customtag tag="S1" slug="discuss_96" impl="discuss"/>
   </vertical>
 </sequential>

common/test/data/full/sequential/System_Usage_Sequence.xml

@@ -3,5 +3,4 @@
     <problem filename="Sample_Numeric_Problem" slug="Sample_Numeric_Problem" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Sample Numeric Problem"/>
     <problem filename="Sample_Algebraic_Problem" slug="Sample_Algebraic_Problem" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Sample Algebraic Problem"/>
   </vertical>
-  <vertical filename="vertical_16" slug="vertical_16" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
 </sequential>

common/test/data/full/vertical/vertical_58.xml

 <sequential>
   <video youtube="1.50:8kARlsUt9lM,1.25:4cLA-IME32w,1.0:pFOrD8k9_p4,0.75:CcgAYu0n0bg" slug="S1V9_Demo_Setup_-_Lumped_Elements" name="S1V9: Demo Setup - Lumped Elements"/>
-  <customtag tag="S1" slug="discuss_59">
-    <impl>discuss</impl>
-  </customtag>
-  <customtag page="29" slug="book_60">
-    <impl>book</impl>
-  </customtag>
-  <customtag lecnum="1" slug="slides_61">
-    <impl>slides</impl>
-  </customtag>
+  <customtag tag="S1" slug="discuss_59" impl="discuss"/>
+  <customtag page="29" slug="book_60" impl="book"/>
+  <customtag lecnum="1" slug="slides_61" impl="slides"/>
 </sequential>

common/test/data/full/vertical/vertical_89.xml

@@ -3,13 +3,7 @@
     <h1> </h1>
   </html>
   <video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>
-  <customtag tag="S1" slug="discuss_91">
-    <impl>discuss</impl>
-  </customtag>
-  <customtag page="70" slug="book_92">
-    <impl>book</impl>
-  </customtag>
-  <customtag lecnum="1" slug="slides_93">
-    <impl>slides</impl>
-  </customtag>
+  <customtag tag="S1" slug="discuss_91" impl="discuss"/>
+  <customtag page="70" slug="book_92" impl="book"/>
+  <customtag lecnum="1" slug="slides_93" impl="slides"/>
 </sequential>

common/test/data/full/vertical/vertical_94.xml

 <sequential>
   <video youtube="0.75:3NIegrCmA5k,1.0:eLAyO33baQ8,1.25:m1zWi_sh4Aw,1.50:EG-fRTJln_E" slug="S2V1_Review_KVL_KCL" name="S2V1: Review KVL, KCL"/>
-  <customtag tag="S2" slug="discuss_95">
-    <impl>discuss</impl>
-  </customtag>
-  <customtag page="54" slug="book_96">
-    <impl>book</impl>
-  </customtag>
-  <customtag lecnum="2" slug="slides_97">
-    <impl>slides</impl>
-  </customtag>
+  <customtag tag="S2" slug="discuss_95" impl="discuss"/>
+  <customtag page="54" slug="book_96" impl="book"/>
+  <customtag lecnum="2" slug="slides_97" impl="slides"/>
 </sequential>

common/test/data/full/vertical/vertical_98.xml

 <sequential>
   <video youtube="0.75:S_1NaY5te8Q,1.0:G_2F9wivspM,1.25:b-r7dISY-Uc,1.50:jjxHom0oXWk" slug="S2V2_Demo-_KVL_KCL" name="S2V2: Demo- KVL, KCL"/>
-  <customtag tag="S2" slug="discuss_99">
-    <impl>discuss</impl>
-  </customtag>
-  <customtag page="56" slug="book_100">
-    <impl>book</impl>
-  </customtag>
-  <customtag lecnum="2" slug="slides_101">
-    <impl>slides</impl>
-  </customtag>
+  <customtag tag="S2" slug="discuss_99" impl="discuss"/>
+  <customtag page="56" slug="book_100" impl="book"/>
+  <customtag lecnum="2" slug="slides_101" impl="slides"/>
 </sequential>

common/test/data/simple/course.xml

+<course name="A Simple Course" org="edX" course="simple" graceperiod="1 day 5 hours 59 minutes 59 seconds" slug="2012_Fall">
+  <chapter name="Overview">
+    <video name="Welcome" youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8"/>
+    <videosequence format="Lecture Sequence" name="A simple sequence">
+      <html id="toylab" filename="toylab"/>
+      <video name="S0V1: Video Resources" youtube="0.75:EuzkdzfR0i8,1.0:1bK-WdDi6Qw,1.25:0v1VzoDVUTM,1.50:Bxk_-ZJb240"/>
+    </videosequence>
+    <section name="Lecture 2">
+    <sequential>
+        <video youtube="1.0:TBvX7HzxexQ"/>
+        <problem name="L1 Problem 1" points="1" type="lecture" showanswer="attempted" filename="L1_Problem_1" rerandomize="never"/>
+      </sequential>
+    </section>
+  </chapter>
+  <chapter name="Chapter 2">
+    <section name="Problem Set 1">
+      <sequential>
+        <problem type="lecture" showanswer="attempted" rerandomize="true" title="A simple coding problem" name="Simple coding problem" filename="ps01-simple"/>
+      </sequential>
+    </section>
+    <video name="Lost Video" youtube="1.0:TBvX7HzxexQ"/>
+  </chapter>
+
+</course>

common/test/data/simple/html/toylab.html

+<b>Lab 2A: Superposition Experiment</b>
+
+<p>Isn't the toy course great?</p>

common/test/data/simple/problems/L1_Problem_1.xml

+<?xml version="1.0"?>
+<problem>
+  <p>
+    <h1>Finger Exercise 1</h1>
+  </p>
+  <p>
+Here are two definitions: </p>
+  <ol class="enumerate">
+    <li>
+      <p>
+Declarative knowledge refers to statements of fact. </p>
+    </li>
+    <li>
+      <p>
+Imperative knowledge refers to 'how to' methods. </p>
+    </li>
+  </ol>
+  <p>
+Which of the following choices is correct? </p>
+  <ol class="enumerate">
+    <li>
+      <p>
+Statement 1 is true, Statement 2 is false </p>
+    </li>
+    <li>
+      <p>
+Statement 1 is false, Statement 2 is true </p>
+    </li>
+    <li>
+      <p>
+Statement 1 and Statement 2 are both false </p>
+    </li>
+    <li>
+      <p>
+Statement 1 and Statement 2 are both true </p>
+    </li>
+  </ol>
+  <p>
+    <symbolicresponse answer="4">
+      <textline size="90" math="1"/>
+    </symbolicresponse>
+  </p>
+</problem>

common/test/data/simple/problems/ps01-simple.xml

+<problem><style media="all" type="text/css"/>
+<text><h2>Paying Off Credit Card Debt</h2>
+<p> Each month, a credit
+      card statement will come with the option for you to pay a
+      minimum amount of your charge, usually 2% of the balance due.
+      However, the credit card company earns money by charging
+      interest on the balance that you don't pay. So even if you
+      pay credit card payments on time, interest is still accruing
+      on the outstanding balance.</p>
+<p >Say you've made a
+      $5,000 purchase on a credit card with 18% annual interest
+      rate and 2% minimum monthly payment rate. After a year, how
+      much is the remaining balance? Use the following
+      equations.</p>
+<blockquote>
+<p><strong>Minimum monthly payment</strong>
+= (Minimum monthly payment rate) x (Balance)<br/>
+        (Minimum monthly payment gets split into interest paid and
+        principal paid)<br/>
+<strong>Interest Paid</strong> = (Annual interest rate) / (12
+        months) x (Balance)<br/>
+<strong>Principal paid</strong> = (Minimum monthly payment) -
+        (Interest paid)<br/>
+<strong>Remaining balance</strong> = Balance - (Principal
+        paid)</p>
+</blockquote>
+<p >For month 1, compute the minimum monthly payment by taking 2% of the balance.</p>
+<blockquote xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">
+<p><strong>Minimum monthly payment</strong>
+= .02 x $5000 = $100</p>
+<p>We can't simply deduct this from the balance because
+        there is compounding interest. Of this $100 monthly
+        payment, compute how much will go to paying off interest
+        and how much will go to paying off the principal. Remember
+        that it's the annual interest rate that is given, so we
+        need to divide it by 12 to get the monthly interest
+        rate.</p>
+<p><strong>Interest paid</strong> = .18/12 x $5000 =
+        $75<br/>
+<strong>Principal paid</strong> = $100 - $75 = $25</p>
+<p>The remaining balance at the end of the first month will
+        be the principal paid this month subtracted from the
+        balance at the start of the month.</p>
+<p><strong>Remaining balance</strong> = $5000 - $25 =
+        $4975</p>
+</blockquote>
+<p xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">For month 2, we
+      repeat the same steps.</p>
+<blockquote xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">
+<p><strong>Minimum monthly payment</strong>
+= .02 x $4975 = $99.50<br/>
+<strong>Interest Paid</strong> = .18/12 x $4975 =
+        $74.63<br/>
+<strong>Principal Paid</strong> = $99.50 - $74.63 =
+        $24.87<br/>
+<strong>Remaining Balance</strong> = $4975 - $24.87 =
+        $4950.13</p>
+</blockquote>
+<p xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">After 12 months, the
+      total amount paid is $1167.55, leaving an outstanding balance
+      of $4708.10. Pretty depressing!</p>
+</text></problem>

common/test/data/toy/course.xml

-<course name="Toy Course" graceperiod="1 day 5 hours 59 minutes 59 seconds" showanswer="always" rerandomize="never">
+<course name="Toy Course" org="edX" course="toy" graceperiod="1 day 5 hours 59 minutes 59 seconds" slug="2012_Fall" start="2015-07-17T12:00">
   <chapter name="Overview">
-    <video name="Welcome" youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8"/>
-    <videosequence format="Lecture Sequence" name="System Usage Sequence">
-      <html id="Lab2A" filename="Lab2A"/>
-      <video name="S0V1: Video Resources" youtube="0.75:EuzkdzfR0i8,1.0:1bK-WdDi6Qw,1.25:0v1VzoDVUTM,1.50:Bxk_-ZJb240"/>
+    <videosequence format="Lecture Sequence" name="Toy Videos">
+      <html name="toylab" filename="toylab"/>
+      <video name="Video Resources" youtube="1.0:1bK-WdDi6Qw"/>
     </videosequence>
+    <video name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
   </chapter>
 </course>

common/test/data/toy/html/Lab2A.html

-<script type="text/javascript">
-$(document).ready(function() { 
-$("#r1_slider").slider({
-value: 1, min: 1, max: 10, step: 1, slide: schematic.component_slider,
-schematic: "ctrls", component: "R1", property: "r", analysis: "dc",
-})
-
-$("#r2_slider").slider({
-value: 1, min: 1, max: 10, step: 1, slide: schematic.component_slider,
-schematic: "ctrls", component: "R2", property: "r", analysis: "dc",
-})
-
-$("#r3_slider").slider({
-value: 1, min: 1, max: 10, step: 1, slide: schematic.component_slider,
-schematic: "ctrls", component: "R3", property: "r", analysis: "dc",
-})
-
-$("#r4_slider").slider({
-value: 1, min: 1, max: 10, step: 1, slide: schematic.component_slider,
-schematic: "ctrls", component: "R4", property: "r", analysis: "dc",
-})
-
-$("#slider").slider(); });
-</script>
-
-<b>Lab 2A: Superposition Experiment</b>
-
-<br><br><i>Note: This part of the lab is just to develop your intuition about
-superposition.  There are no responses that need to be checked.</i>
-
-<br/><br/>Circuits with multiple sources can be hard to analyze as-is.  For example, what is the voltage
-between the two terminals on the right of Figure 1?
-
-<center>
-<input width="425" type="hidden" height="150" id="schematic1" parts="" analyses="" class="schematic ctrls" name="test2" value="[[&quot;w&quot;,[160,64,184,64]],[&quot;w&quot;,[160,16,184,16]],[&quot;w&quot;,[64,16,112,16]],[&quot;w&quot;,[112,64,88,64]],[&quot;w&quot;,[64,64,88,64]],[&quot;g&quot;,[88,64,0],{},[&quot;0&quot;]],[&quot;w&quot;,[112,64,160,64]],[&quot;w&quot;,[16,64,64,64]],[&quot;r&quot;,[160,16,0],{&quot;name&quot;:&quot;R4&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;1&quot;,&quot;0&quot;]],[&quot;r&quot;,[160,16,1],{&quot;name&quot;:&quot;R3&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;1&quot;,&quot;2&quot;]],[&quot;i&quot;,[112,64,6],{&quot;name&quot;:&quot;&quot;,&quot;value&quot;:&quot;6A&quot;},[&quot;0&quot;,&quot;2&quot;]],[&quot;r&quot;,[64,16,0],{&quot;name&quot;:&quot;R2&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;2&quot;,&quot;0&quot;]],[&quot;r&quot;,[64,16,1],{&quot;name&quot;:&quot;R1&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;2&quot;,&quot;3&quot;]],[&quot;v&quot;,[16,16,0],{&quot;name&quot;:&quot;&quot;,&quot;value&quot;:&quot;8V&quot;},[&quot;3&quot;,&quot;0&quot;]],[&quot;view&quot;,-24,0,2]]"/>
-Figure 1.  Example multi-source circuit
-</center>
-
-<br/><br/>We can use superposition to make the analysis much easier.
-The circuit in Figure 1 can be decomposed into two separate
-subcircuits: one involving only the voltage source and one involving only the
-current source.  We'll analyze each circuit separately and combine the
-results using superposition.  Recall that to decompose a circuit for
-analysis, we'll pick each source in turn and set all the other sources
-to zero (i.e., voltage sources become short circuits and current
-sources become open circuits).  The circuit above has two sources, so
-the decomposition produces two subcircuits, as shown in Figure 2.
-
-<center>
-<table><tr><td>
-<input style="display:inline;" width="425" type="hidden" height="150" id="schematic2" parts="" analyses="" class="schematic ctrls" name="test2" value="[[&quot;w&quot;,[160,64,184,64]],[&quot;w&quot;,[160,16,184,16]],[&quot;w&quot;,[64,16,112,16]],[&quot;w&quot;,[112,64,88,64]],[&quot;w&quot;,[64,64,88,64]],[&quot;g&quot;,[88,64,0],{},[&quot;0&quot;]],[&quot;w&quot;,[112,64,160,64]],[&quot;w&quot;,[16,64,64,64]],[&quot;r&quot;,[160,16,0],{&quot;name&quot;:&quot;R4&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;1&quot;,&quot;0&quot;]],[&quot;r&quot;,[160,16,1],{&quot;name&quot;:&quot;R3&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;1&quot;,&quot;2&quot;]],[&quot;r&quot;,[64,16,0],{&quot;name&quot;:&quot;R2&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;2&quot;,&quot;0&quot;]],[&quot;r&quot;,[64,16,1],{&quot;name&quot;:&quot;R1&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;2&quot;,&quot;3&quot;]],[&quot;v&quot;,[16,16,0],{&quot;name&quot;:&quot;&quot;,&quot;value&quot;:&quot;8V&quot;},[&quot;3&quot;,&quot;0&quot;]],[&quot;view&quot;,-24,0,2]]"/>
-(a) Subcircuit for analyzing contribution of voltage source
-</td><td>
-<input width="425" type="hidden" height="150" id="schematic3" parts="" analyses="" class="schematic ctrls" name="test2" value="[[&quot;w&quot;,[16,16,16,64]],[&quot;w&quot;,[160,64,184,64]],[&quot;w&quot;,[160,16,184,16]],[&quot;w&quot;,[64,16,112,16]],[&quot;w&quot;,[112,64,88,64]],[&quot;w&quot;,[64,64,88,64]],[&quot;g&quot;,[88,64,0],{},[&quot;0&quot;]],[&quot;w&quot;,[112,64,160,64]],[&quot;w&quot;,[16,64,64,64]],[&quot;r&quot;,[160,16,0],{&quot;name&quot;:&quot;R4&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;1&quot;,&quot;0&quot;]],[&quot;r&quot;,[160,16,1],{&quot;name&quot;:&quot;R3&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;1&quot;,&quot;2&quot;]],[&quot;i&quot;,[112,64,6],{&quot;name&quot;:&quot;&quot;,&quot;value&quot;:&quot;6A&quot;},[&quot;0&quot;,&quot;2&quot;]],[&quot;r&quot;,[64,16,0],{&quot;name&quot;:&quot;R2&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;2&quot;,&quot;0&quot;]],[&quot;r&quot;,[64,16,1],{&quot;name&quot;:&quot;R1&quot;,&quot;r&quot;:&quot;1&quot;},[&quot;2&quot;,&quot;3&quot;]],[&quot;view&quot;,-24,0,2]]"/>
-(b) Subcircuit for analyzing contribution of current source
-</td></tr></table>
-<br>Figure 2.  Decomposition of Figure 1 into subcircuits
-</center>
-
-<br/>Let's use the DC analysis capability of the schematic tool to see superposition
-in action.  The sliders below control the resistances of R1, R2, R3 and R4 in all
-the diagrams.  As you move the sliders, the schematic tool will adjust the appropriate
-resistance, perform a DC analysis and display the node voltages on the diagrams.  Here's
-what you want to observe as you play with the sliders:
-
-<ul style="margin-left:2em;margin-top:1em;margin-right:2em;margin-bottom:1em;">
-<i>The voltage for a node in Figure 1 is the sum of the voltages for
-that node in Figures 2(a) and 2(b), just as predicted by
-superposition. (Note that due to round-off in the display of the
-voltages, the sum of the displayed voltages in Figure 2 may only be within
-.01 of the voltages displayed in Figure 1.)</i>
-</ul>
-
-<br>
-<center>
-<table><tr valign="top">
-<td>
-  <table>
-    <tr valign="top">
-      <td>R1</td>
-      <td>
-	<div id="r1_slider" style="width:200px; height:10px; margin-left:15px"></div>
-      </td>
-    </tr>
-    <tr valign="top">
-      <td>R2</td>
-      <td>
-	<div id="r2_slider" style="width:200px; height:10px; margin-left:15px; margin-top:10px;"></div>
-      </td>
-    </tr>
-    <tr valign="top">
-      <td>R3</td>
-      <td>
-	<div id="r3_slider" style="width:200px; height:10px; margin-left:15px; margin-top:10px;"></div>
-      </td>
-    </tr>
-    <tr valign="top">
-      <td>R4</td>
-      <td>
-	<div id="r4_slider" style="width:200px; height:10px; margin-left:15px; margin-top:10px;"></div>
-      </td>
-    </tr>
-  </table>
-</td></tr></table>
-</center>

common/test/data/toy/html/toylab.html

+<b>Lab 2A: Superposition Experiment</b>
+
+<p>Isn't the toy course great?</p>

doc/overview.md

@@ -13,17 +13,17 @@ You should be familiar with the following.  If you're not, go read some docs...
  - css
  - git
  - mako templates -- we use these instead of django templates, because they support embedding real python.
- 
+
 ## Other relevant terms
 
  - CAPA -- lon-capa.org -- content management system that has defined a standard for online learning and assessment materials.  Many of our materials follow this standard.
-    - TODO: add more details / link to relevant docs.  lon-capa.org is not immediately intuitive.  
+    - TODO: add more details / link to relevant docs.  lon-capa.org is not immediately intuitive.
     - lcp = loncapa problem
 
 
 ## Parts of the system
 
-  - LMS -- Learning Management System.   The student-facing parts of the system.  Handles student accounts, displaying videos, tutorials, exercies, problems, etc. 
+  - LMS -- Learning Management System.   The student-facing parts of the system.  Handles student accounts, displaying videos, tutorials, exercies, problems, etc.
 
   - CMS -- Course Management System.  The instructor-facing parts of the system.  Allows instructors to see and modify their course, add lectures, problems, reorder things, etc.
 
@@ -42,7 +42,7 @@ You should be familiar with the following.  If you're not, go read some docs...
 
 ## High Level Entities in the code
 
-### Common libraries 
+### Common libraries
 
 - xmodule: generic learning modules. *x* can be sequence, video, template, html,
            vertical, capa, etc.  These are the things that one puts inside sections
@@ -51,7 +51,7 @@ You should be familiar with the following.  If you're not, go read some docs...
     - XModuleDescriptor: This defines the problem and all data and UI needed to edit
            that problem. It is unaware of any student data, but can be used to retrieve
            an XModule, which is aware of that student state.
-             
+
     - XModule: The XModule is a problem instance that is particular to a student. It knows
            how to render itself to html to display the problem, how to score itself,
            and how to handle ajax calls from the front end.
@@ -59,19 +59,25 @@ You should be familiar with the following.  If you're not, go read some docs...
     - Both XModule and XModuleDescriptor take system context parameters. These are named
            ModuleSystem and DescriptorSystem respectively. These help isolate the XModules
            from any interactions with external resources that they require.
-           
+
            For instance, the DescriptorSystem has a function to load an XModuleDescriptor
            from a Location object, and the ModuleSystem knows how to render things,
            track events, and complain about 404s
-    - TODO: document the system context interface--it's different in `x_module.XModule.__init__` and in `x_module tests.py` (do this in the code, not here)
+
+    - `course.xml` format.  We use python setuptools to connect supported tags with the descriptors that handle them.  See `common/lib/xmodule/setup.py`.  There are checking and validation tools in `common/validate`.
+
+         - the xml import+export functionality is in `xml_module.py:XmlDescriptor`, which is a mixin class that's used by the actual descriptor classes.
+
+         - There is a distinction between descriptor _definitions_ that stay the same for any use of that descriptor (e.g. here is what a particular problem is), and _metadata_ describing how that descriptor is used (e.g. whether to allow checking of answers, due date, etc).  When reading in `from_xml`, the code pulls out the metadata attributes into a separate structure, and puts it back on export.
+
     - in `common/lib/xmodule`
 
-- capa modules -- defines `LoncapaProblem` and many related things.  
+- capa modules -- defines `LoncapaProblem` and many related things.
     - in `common/lib/capa`
 
-### LMS 
+### LMS
 
-The LMS is a django site, with root in `lms/`.  It runs in many different environments--the settings files are in `lms/envs`. 
+The LMS is a django site, with root in `lms/`.  It runs in many different environments--the settings files are in `lms/envs`.
 
 - We use the Django Auth system, including the is_staff and is_superuser flags.  User profiles and related code lives in `lms/djangoapps/student/`.   There is support for groups of students (e.g. 'want emails about future courses', 'have unenrolled', etc) in `lms/djangoapps/student/models.py`.
 
@@ -79,19 +85,19 @@ The LMS is a django site, with root in `lms/`.  It runs in many different enviro
     - `lms/djangoapps/courseware/models.py`
 
 - Core rendering path:
-  - `lms/urls.py` points to `courseware.views.index`, which gets module info from the course xml file, pulls list of `StudentModule` objects for this user (to avoid multiple db hits).  
+  - `lms/urls.py` points to `courseware.views.index`, which gets module info from the course xml file, pulls list of `StudentModule` objects for this user (to avoid multiple db hits).
 
   - Calls `render_accordion` to render the "accordion"--the display of the course structure.
 
   - To render the current module, calls `module_render.py:render_x_module()`, which gets the `StudentModule` instance, and passes the `StudentModule` state and other system context to the module constructor the get an instance of the appropriate module class for this user.
 
   - calls the module's `.get_html()` method.  If the module has nested submodules, render_x_module() will be called again for each.
-  
+
   - ajax calls go to `module_render.py:modx_dispatch()`, which passes it to the module's `handle_ajax()` function, and then updates the grade and state if they changed.
 
   - [This diagram](https://github.com/MITx/mitx/wiki/MITx-Architecture) visually shows how the clients communicate with problems + modules.
-  
-- See `lms/urls.py` for the wirings of urls to views.  
+
+- See `lms/urls.py` for the wirings of urls to views.
 
 - Tracking: there is support for basic tracking of client-side events in `lms/djangoapps/track`.
 
@@ -110,7 +116,7 @@ environments, defined in `cms/envs`.
 
 - _mako_  -- we use this for templates, and have wrapper called mitxmako that makes mako look like the django templating calls.
 
-We use a fork of django-pipeline to make sure that the js and css always reflect the latest `*.coffee` and `*.sass` files (We're hoping to get our changes merged in the official version soon).  This works differently in development and production.  Test uses the production settings.  
+We use a fork of django-pipeline to make sure that the js and css always reflect the latest `*.coffee` and `*.sass` files (We're hoping to get our changes merged in the official version soon).  This works differently in development and production.  Test uses the production settings.
 
 In production, the django `collectstatic` command recompiles everything and puts all the generated static files in a static/ dir.  A starting point in the code is `django-pipeline/pipeline/packager.py:pack`.
 
@@ -127,8 +133,6 @@ See `testing.md`.
 
 ## TODO:
 
-- update lms/envs/README.txt
-
 - describe our production environment
 
 - describe the front-end architecture, tools, etc.  Starting point: `lms/static`

lms/djangoapps/courseware/management/commands/clean_xml.py

+import os
+import sys
+import traceback
+
+from filecmp import dircmp
+from fs.osfs import OSFS
+from path import path
+from lxml import etree
+
+from django.core.management.base import BaseCommand
+
+from xmodule.modulestore.xml import XMLModuleStore
+
+
+def traverse_tree(course):
+    '''Load every descriptor in course.  Return bool success value.'''
+    queue = [course]
+    while len(queue) > 0:
+        node = queue.pop()
+#        print '{0}:'.format(node.location)
+#        if 'data' in node.definition:
+#            print '{0}'.format(node.definition['data'])
+        queue.extend(node.get_children())
+
+    return True
+
+def make_logging_error_handler():
+    '''Return a tuple (handler, error_list), where
+    the handler appends the message and any exc_info
+    to the error_list on every call.
+    '''
+    errors = []
+
+    def error_handler(msg, exc_info=None):
+        '''Log errors'''
+        if exc_info is None:
+            if sys.exc_info() != (None, None, None):
+                exc_info = sys.exc_info()
+
+        errors.append((msg, exc_info))
+
+    return (error_handler, errors)
+
+
+def export(course, export_dir):
+    """Export the specified course to course_dir.  Creates dir if it doesn't exist.
+    Overwrites files, does not clean out dir beforehand.
+    """
+    fs = OSFS(export_dir, create=True)
+    if not fs.isdirempty('.'):
+        print ('WARNING: Directory {dir} not-empty.'
+               '  May clobber/confuse things'.format(dir=export_dir))
+
+    try:
+        xml = course.export_to_xml(fs)
+        with fs.open('course.xml', mode='w') as f:
+            f.write(xml)
+
+        return True
+    except:
+        print 'Export failed!'
+        traceback.print_exc()
+
+    return False
+
+
+def import_with_checks(course_dir, verbose=True):
+    all_ok = True
+
+    print "Attempting to load '{0}'".format(course_dir)
+
+    course_dir = path(course_dir)
+    data_dir = course_dir.dirname()
+    course_dirs = [course_dir.basename()]
+
+    (error_handler, errors) = make_logging_error_handler()
+    # No default class--want to complain if it doesn't find plugins for any
+    # module.
+    modulestore = XMLModuleStore(data_dir,
+                   default_class=None,
+                   eager=True,
+                   course_dirs=course_dirs,
+                   error_handler=error_handler)
+
+    def str_of_err(tpl):
+        (msg, exc_info) = tpl
+        if exc_info is None:
+            return msg
+
+        exc_str = '\n'.join(traceback.format_exception(*exc_info))
+        return '{msg}\n{exc}'.format(msg=msg, exc=exc_str)
+
+    courses = modulestore.get_courses()
+    if len(errors) != 0:
+        all_ok = False
+        print '\n'
+        print "=" * 40
+        print 'ERRORs during import:'
+        print '\n'.join(map(str_of_err,errors))
+        print "=" * 40
+        print '\n'
+
+    n = len(courses)
+    if n != 1:
+        print 'ERROR: Expect exactly 1 course.  Loaded {n}: {lst}'.format(
+            n=n, lst=courses)
+        return (False, None)
+
+    course = courses[0]
+
+    #print course
+    validators = (
+        traverse_tree,
+        )
+
+    print "=" * 40
+    print "Running validators..."
+
+    for validate in validators:
+        print 'Running {0}'.format(validate.__name__)
+        all_ok = validate(course) and all_ok
+
+
+    if all_ok:
+        print 'Course passes all checks!'
+    else:
+        print "Course fails some checks.  See above for errors."
+    return all_ok, course
+
+
+def check_roundtrip(course_dir):
+    '''Check that import->export leaves the course the same'''
+
+    print "====== Roundtrip import ======="
+    (ok, course) = import_with_checks(course_dir)
+    if not ok:
+        raise Exception("Roundtrip import failed!")
+
+    print "====== Roundtrip export ======="
+    export_dir = course_dir + ".rt"
+    export(course, export_dir)
+
+    # dircmp doesn't do recursive diffs.
+    # diff = dircmp(course_dir, export_dir, ignore=[], hide=[])
+    print "======== Roundtrip diff: ========="
+    os.system("diff -r {0} {1}".format(course_dir, export_dir))
+    print "======== ideally there is no diff above this ======="
+
+
+def clean_xml(course_dir, export_dir):
+    (ok, course) = import_with_checks(course_dir)
+    if ok:
+        export(course, export_dir)
+        check_roundtrip(export_dir)
+    else:
+        print "Did NOT export"
+
+
+
+class Command(BaseCommand):
+    help = """Imports specified course.xml, validate it, then exports in
+    a canonical format.
+
+Usage: clean_xml PATH-TO-COURSE-DIR PATH-TO-OUTPUT-DIR
+"""
+    def handle(self, *args, **options):
+        if len(args) != 2:
+            print Command.help
+            return
+
+        clean_xml(args[0], args[1])

lms/djangoapps/courseware/module_render.py

@@ -119,7 +119,7 @@ def get_module(user, request, location, student_module_cache, position=None):
         instance_module is a StudentModule specific to this module for this student,
             or None if this is an anonymous user
         shared_module is a StudentModule specific to all modules with the same
-            'shared_state_key' attribute, or None if the module doesn't elect to
+            'shared_state_key' attribute, or None if the module does not elect to
             share state
     '''
     descriptor = modulestore().get_item(location)
@@ -131,11 +131,13 @@ def get_module(user, request, location, student_module_cache, position=None):
     if course_id:
         course_id = course_id.group('course_id')
 
-    instance_module = student_module_cache.lookup(descriptor.category, descriptor.location.url())
+    instance_module = student_module_cache.lookup(descriptor.category,
+                                                  descriptor.location.url())
 
     shared_state_key = getattr(descriptor, 'shared_state_key', None)
     if shared_state_key is not None:
-        shared_module = student_module_cache.lookup(descriptor.category, shared_state_key)
+        shared_module = student_module_cache.lookup(descriptor.category,
+                                                    shared_state_key)
     else:
         shared_module = None
 
@@ -150,10 +152,12 @@ def get_module(user, request, location, student_module_cache, position=None):
     # TODO (vshnayder): fix hardcoded urls (use reverse)
     # Setup system context for module instance
     ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/'
-    xqueue_callback_url = settings.MITX_ROOT_URL + '/xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/'
+    xqueue_callback_url = (settings.MITX_ROOT_URL + '/xqueue/' +
+                           str(user.id) + '/' + descriptor.location.url() + '/')
 
     def _get_module(location):
-        (module, _, _, _) = get_module(user, request, location, student_module_cache, position)
+        (module, _, _, _) = get_module(user, request, location,
+                                       student_module_cache, position)
         return module
 
     # TODO (cpennington): When modules are shared between courses, the static

lms/djangoapps/courseware/tests/__init__.py

+

lms/djangoapps/courseware/tests/tests.py

+import copy
+import json
+import os
+
+from pprint import pprint
+
+from django.test import TestCase
+from django.test.client import Client
+from mock import patch, Mock
+from override_settings import override_settings
+from django.conf import settings
+from django.core.urlresolvers import reverse
+from path import path
+
+from student.models import Registration
+from django.contrib.auth.models import User
+
+from xmodule.modulestore.django import modulestore
+import xmodule.modulestore.django
+from xmodule.modulestore import Location
+from xmodule.modulestore.xml_importer import import_from_xml
+
+
+def parse_json(response):
+    """Parse response, which is assumed to be json"""
+    return json.loads(response.content)
+
+
+def user(email):
+    '''look up a user by email'''
+    return User.objects.get(email=email)
+
+
+def registration(email):
+    '''look up registration object by email'''
+    return Registration.objects.get(user__email=email)
+
+
+# A bit of a hack--want mongo modulestore for these tests, until
+# jump_to works with the xmlmodulestore or we have an even better solution
+# NOTE: this means this test requires mongo to be running.
+
+def mongo_store_config(data_dir):
+    return {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': {
+            'default_class': 'xmodule.raw_module.RawDescriptor',
+            'host': 'localhost',
+            'db': 'xmodule',
+            'collection': 'modulestore',
+            'fs_root': data_dir,
+        }
+    }
+}
+
+TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
+TEST_DATA_MODULESTORE = mongo_store_config(TEST_DATA_DIR)
+
+REAL_DATA_DIR = settings.GITHUB_REPO_ROOT
+REAL_DATA_MODULESTORE = mongo_store_config(REAL_DATA_DIR)
+
+class ActivateLoginTestCase(TestCase):
+    '''Check that we can activate and log in'''
+
+    def setUp(self):
+        email = 'view@test.com'
+        password = 'foo'
+        self.create_account('viewtest', email, password)
+        self.activate_user(email)
+        self.login(email, password)
+
+    # ============ User creation and login ==============
+
+    def _login(self, email, pw):
+        '''Login.  View should always return 200.  The success/fail is in the
+        returned json'''
+        resp = self.client.post(reverse('login'),
+                                {'email': email, 'password': pw})
+        self.assertEqual(resp.status_code, 200)
+        return resp
+
+    def login(self, email, pw):
+        '''Login, check that it worked.'''
+        resp = self._login(email, pw)
+        data = parse_json(resp)
+        self.assertTrue(data['success'])
+        return resp
+
+    def _create_account(self, username, email, pw):
+        '''Try to create an account.  No error checking'''
+        resp = self.client.post('/create_account', {
+            'username': username,
+            'email': email,
+            'password': pw,
+            'name': 'Fred Weasley',
+            'terms_of_service': 'true',
+            'honor_code': 'true',
+        })
+        return resp
+
+    def create_account(self, username, email, pw):
+        '''Create the account and check that it worked'''
+        resp = self._create_account(username, email, pw)
+        self.assertEqual(resp.status_code, 200)
+        data = parse_json(resp)
+        self.assertEqual(data['success'], True)
+
+        # Check both that the user is created, and inactive
+        self.assertFalse(user(email).is_active)
+
+        return resp
+
+    def _activate_user(self, email):
+        '''Look up the activation key for the user, then hit the activate view.
+        No error checking'''
+        activation_key = registration(email).activation_key
+
+        # and now we try to activate
+        resp = self.client.get(reverse('activate', kwargs={'key': activation_key}))
+        return resp
+
+    def activate_user(self, email):
+        resp = self._activate_user(email)
+        self.assertEqual(resp.status_code, 200)
+        # Now make sure that the user is now actually activated
+        self.assertTrue(user(email).is_active)
+
+    def test_activate_login(self):
+        '''The setup function does all the work'''
+        pass
+
+
+class PageLoader(ActivateLoginTestCase):
+    ''' Base class that adds a function to load all pages in a modulestore '''
+
+    def check_pages_load(self, course_name, data_dir, modstore):
+        print "Checking course {0} in {1}".format(course_name, data_dir)
+        import_from_xml(modstore, data_dir, [course_name])
+
+        n = 0
+        num_bad = 0
+        all_ok = True
+        for descriptor in modstore.get_items(
+                Location(None, None, None, None, None)):
+            n += 1
+            print "Checking ", descriptor.location.url()
+            #print descriptor.__class__, descriptor.location
+            resp = self.client.get(reverse('jump_to',
+                                   kwargs={'location': descriptor.location.url()}))
+            msg = str(resp.status_code)
+
+            if resp.status_code != 200:
+                msg = "ERROR " + msg
+                all_ok = False
+                num_bad += 1
+            print msg
+            self.assertTrue(all_ok)  # fail fast
+
+        print "{0}/{1} good".format(n - num_bad, n)
+        self.assertTrue(all_ok)
+
+
+@override_settings(MODULESTORE=TEST_DATA_MODULESTORE)
+class TestCoursesLoadTestCase(PageLoader):
+    '''Check that all pages in test courses load properly'''
+
+    def setUp(self):
+        ActivateLoginTestCase.setUp(self)
+        xmodule.modulestore.django._MODULESTORES = {}
+        xmodule.modulestore.django.modulestore().collection.drop()
+
+    def test_toy_course_loads(self):
+        self.check_pages_load('toy', TEST_DATA_DIR, modulestore())
+
+    def test_full_course_loads(self):
+        self.check_pages_load('full', TEST_DATA_DIR, modulestore())
+
+
+    # ========= TODO: check ajax interaction here too?
+
+
+@override_settings(MODULESTORE=REAL_DATA_MODULESTORE)
+class RealCoursesLoadTestCase(PageLoader):
+    '''Check that all pages in real courses load properly'''
+
+    def setUp(self):
+        ActivateLoginTestCase.setUp(self)
+        xmodule.modulestore.django._MODULESTORES = {}
+        xmodule.modulestore.django.modulestore().collection.drop()
+
+    # TODO: Disabled test for now..  Fix once things are cleaned up.
+    def Xtest_real_courses_loads(self):
+        '''See if any real courses are available at the REAL_DATA_DIR.
+        If they are, check them.'''
+
+        # TODO: adjust staticfiles_dirs
+        if not os.path.isdir(REAL_DATA_DIR):
+            # No data present.  Just pass.
+            return
+
+        courses = [course_dir for course_dir in os.listdir(REAL_DATA_DIR)
+                   if os.path.isdir(REAL_DATA_DIR / course_dir)]
+        for course in courses:
+            self.check_pages_load(course, REAL_DATA_DIR, modulestore())
+
+
+    # ========= TODO: check ajax interaction here too?

lms/djangoapps/courseware/views.py

@@ -25,12 +25,16 @@ from models import StudentModuleCache
 from student.models import UserProfile
 from multicourse import multicourse_settings
 from django_comment_client.utils import get_discussion_title
+from xmodule.modulestore import Location
+from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
+from xmodule.modulestore.django import modulestore
+from xmodule.course_module import CourseDescriptor
 
 from util.cache import cache, cache_if_anonymous
 from student.models import UserTestGroup, CourseEnrollment
 from courseware import grades
 from courseware.courses import check_course
-from xmodule.modulestore.django import modulestore
+
 
 import comment_client
 
@@ -206,65 +210,59 @@ def index(request, course_id, chapter=None, section=None,
     if look_for_module:
         # TODO (cpennington): Pass the right course in here
 
-        section = get_section(course, chapter, section)
-        student_module_cache = StudentModuleCache(request.user, section)
-        module, _, _, _ = get_module(request.user, request, section.location, student_module_cache)
-        context['content'] = module.get_html()
+        section_descriptor = get_section(course, chapter, section)
+        if section_descriptor is not None:
+            student_module_cache = StudentModuleCache(request.user,
+                                                      section_descriptor)
+            module, _, _, _ = get_module(request.user, request,
+                                         section_descriptor.location,
+                                         student_module_cache)
+            context['content'] = module.get_html()
+        else:
+            log.warning("Couldn't find a section descriptor for course_id '{0}',"
+                        "chapter '{1}', section '{2}'".format(
+                        course_id, chapter, section))
+
 
     result = render_to_response('courseware.html', context)
     return result
 
-
-def jump_to(request, probname=None):
-    '''
-    Jump to viewing a specific problem.  The problem is specified by a
-    problem name - currently the filename (minus .xml) of the problem.
-    Maybe this should change to a more generic tag, eg "name" given as
-    an attribute in <problem>.
-
-    We do the jump by (1) reading course.xml to find the first
-    instance of <problem> with the given filename, then (2) finding
-    the parent element of the problem, then (3) rendering that parent
-    element with a specific computed position value (if it is
-    <sequential>).
-
+@ensure_csrf_cookie
+def jump_to(request, location):
     '''
-    # get coursename if stored
-    coursename = multicourse_settings.get_coursename_from_request(request)
+    Show the page that contains a specific location.
 
-    # begin by getting course.xml tree
-    xml = content_parser.course_file(request.user, coursename)
+    If the location is invalid, return a 404.
 
-    # look for problem of given name
-    pxml = xml.xpath('//problem[@filename="%s"]' % probname)
-    if pxml:
-        pxml = pxml[0]
+    If the location is valid, but not present in a course, ?
 
-    # get the parent element
-    parent = pxml.getparent()
-
-    # figure out chapter and section names
-    chapter = None
-    section = None
-    branch = parent
-    for k in range(4):  # max depth of recursion
-        if branch.tag == 'section':
-            section = branch.get('name')
-        if branch.tag == 'chapter':
-            chapter = branch.get('name')
-        branch = branch.getparent()
+    If the location is valid, but in a course the current user isn't registered for, ?
+        TODO -- let the index view deal with it?
+    '''
+    # Complain if the location isn't valid
+    try:
+        location = Location(location)
+    except InvalidLocationError:
+        raise Http404("Invalid location")
 
-    position = None
-    if parent.tag == 'sequential':
-        position = parent.index(pxml) + 1  # position in sequence
+    # Complain if there's not data for this location
+    try:
+        (course_id, chapter, section, position) = modulestore().path_to_location(location)
+    except ItemNotFoundError:
+        raise Http404("No data at this location: {0}".format(location))
+    except NoPathToItem:
+        raise Http404("This location is not in any class: {0}".format(location))
 
-    return index(request,
-                 course=coursename, chapter=chapter,
-                 section=section, position=position)
 
+    return index(request, course_id, chapter, section, position)
 
 @ensure_csrf_cookie
 def course_info(request, course_id):
+    '''
+    Display the course's info.html, or 404 if there is no such course.
+
+    Assumes the course_id is in a valid format.
+    '''
     course = check_course(course_id)
 
     return render_to_response('info.html', {'course': course})

lms/envs/README.txt

-Transitional for moving to new settings scheme. 
+Transitional for moving to new settings scheme.
 
-To use: 
-  django-admin.py runserver --settings=envs.dev --pythonpath=.
+To use:
+   rake lms
+   or
+   django-admin.py runserver --settings=lms.envs.dev --pythonpath=.
 
 NOTE: Using manage.py will automatically run mitx/settings.py first, regardless
 of what you send it for an explicit --settings flag. It still works, but might
-have odd side effects. Using django-admin.py avoids that problem. 
+have odd side effects. Using django-admin.py avoids that problem.
 django-admin.py is installed by default when you install Django.
 
 To use with gunicorn_django in debug mode:
 
-  gunicorn_django envs/dev.py
+  gunicorn_django lms/envs/dev.py
 

lms/envs/aws.py

@@ -32,12 +32,12 @@ LOG_DIR = ENV_TOKENS['LOG_DIR']
 
 CACHES = ENV_TOKENS['CACHES']
 
-for feature,value in ENV_TOKENS.get('MITX_FEATURES', {}).items():
+for feature, value in ENV_TOKENS.get('MITX_FEATURES', {}).items():
     MITX_FEATURES[feature] = value
 
-WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED',WIKI_ENABLED)
+WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED', WIKI_ENABLED)
 
-LOGGING = get_logger_config(LOG_DIR, 
+LOGGING = get_logger_config(LOG_DIR,
                             logging_env=ENV_TOKENS['LOGGING_ENV'],
                             syslog_addr=(ENV_TOKENS['SYSLOG_SERVER'], 514),
                             debug=False)

lms/envs/common.py

@@ -6,16 +6,16 @@ MITX_FEATURES[...]. Modules that extend this one can change the feature
 configuration in an environment specific config file and re-calculate those
 values.
 
-We should make a method that calls all these config methods so that you just 
+We should make a method that calls all these config methods so that you just
 make one call at the end of your site-specific dev file to reset all the
 dependent variables (like INSTALLED_APPS) for you.
 
 Longer TODO:
-1. Right now our treatment of static content in general and in particular 
+1. Right now our treatment of static content in general and in particular
    course-specific static content is haphazard.
 2. We should have a more disciplined approach to feature flagging, even if it
    just means that we stick them in a dict called MITX_FEATURES.
-3. We need to handle configuration for multiple courses. This could be as 
+3. We need to handle configuration for multiple courses. This could be as
    multiple sites, but we do need a way to map their data assets.
 """
 import sys
@@ -43,7 +43,7 @@ MITX_FEATURES = {
     'SAMPLE' : False,
     'USE_DJANGO_PIPELINE' : True,
     'DISPLAY_HISTOGRAMS_TO_STAFF' : True,
-    'REROUTE_ACTIVATION_EMAIL' : False,		# nonempty string = address for all activation emails 
+    'REROUTE_ACTIVATION_EMAIL' : False,		# nonempty string = address for all activation emails
     'DEBUG_LEVEL' : 0,				# 0 = lowest level, least verbose, 255 = max level, most verbose
 
     ## DO NOT SET TO True IN THIS FILE
@@ -62,7 +62,7 @@ PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /mitx/lms
 REPO_ROOT = PROJECT_ROOT.dirname()
 COMMON_ROOT = REPO_ROOT / "common"
 ENV_ROOT = REPO_ROOT.dirname()  # virtualenv dir /mitx is in
-ASKBOT_ROOT = ENV_ROOT / "askbot-devel"
+ASKBOT_ROOT = REPO_ROOT / "askbot"
 COURSES_ROOT = ENV_ROOT / "data"
 
 # FIXME: To support multiple courses, we should walk the courses dir at startup
@@ -86,7 +86,7 @@ MAKO_TEMPLATES['main'] = [PROJECT_ROOT / 'templates',
                           COMMON_ROOT / 'lib' / 'capa' / 'capa' / 'templates',
                           COMMON_ROOT / 'djangoapps' / 'pipeline_mako' / 'templates']
 
-# This is where Django Template lookup is defined. There are a few of these 
+# This is where Django Template lookup is defined. There are a few of these
 # still left lying around.
 TEMPLATE_DIRS = (
     PROJECT_ROOT / "templates",
@@ -104,8 +104,8 @@ TEMPLATE_CONTEXT_PROCESSORS = (
 )
 
 
-# FIXME: 
-# We should have separate S3 staged URLs in case we need to make changes to 
+# FIXME:
+# We should have separate S3 staged URLs in case we need to make changes to
 # these assets and test them.
 LIB_URL = '/static/js/'
 
@@ -121,7 +121,7 @@ STATIC_GRAB = False
 DEV_CONTENT = True
 
 # FIXME: Should we be doing this truncation?
-TRACK_MAX_EVENT = 10000 
+TRACK_MAX_EVENT = 10000
 DEBUG_TRACK_LOG = False
 
 MITX_ROOT_URL = ''
@@ -130,7 +130,7 @@ COURSE_NAME = "6.002_Spring_2012"
 COURSE_NUMBER = "6.002x"
 COURSE_TITLE = "Circuits and Electronics"
 
-### Dark code. Should be enabled in local settings for devel. 
+### Dark code. Should be enabled in local settings for devel.
 
 ENABLE_MULTICOURSE = False     # set to False to disable multicourse display (see lib.util.views.mitxhome)
 QUICKEDIT = False
@@ -212,9 +212,9 @@ USE_L10N = True
 MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
 #################################### AWS #######################################
-# S3BotoStorage insists on a timeout for uploaded assets. We should make it 
+# S3BotoStorage insists on a timeout for uploaded assets. We should make it
 # permanent instead, but rather than trying to figure out exactly where that
-# setting is, I'm just bumping the expiration time to something absurd (100 
+# setting is, I'm just bumping the expiration time to something absurd (100
 # years). This is only used if DEFAULT_FILE_STORAGE is overriden to use S3
 # in the global settings.py
 AWS_QUERYSTRING_EXPIRE = 10 * 365 * 24 * 60 * 60 # 10 years
@@ -281,7 +281,7 @@ MIDDLEWARE_CLASSES = (
     # Instead of AuthenticationMiddleware, we use a cached backed version
     #'django.contrib.auth.middleware.AuthenticationMiddleware',
     'cache_toolbox.middleware.CacheBackedAuthenticationMiddleware',
-    
+
     'django.contrib.messages.middleware.MessageMiddleware',
     'track.middleware.TrackMiddleware',
     'mitxmako.middleware.MakoMiddleware',
@@ -303,15 +303,15 @@ STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
 PIPELINE_CSS = {
     'application': {
         'source_filenames': ['sass/application.scss'],
-        'output_filename': 'css/application.css',
+        'output_filename': 'css/lms-application.css',
     },
     'course': {
       'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css'],
-      'output_filename': 'css/course.css',
+      'output_filename': 'css/lms-course.css',
       },
     'ie-fixes': {
         'source_filenames': ['sass/ie.scss'],
-        'output_filename': 'css/ie.css',
+        'output_filename': 'css/lms-ie.css',
     },
 }
 
@@ -412,23 +412,23 @@ PIPELINE_JS = {
             'js/toggle_login_modal.js',
             'js/sticky_filter.js',
         ],
-        'output_filename': 'js/application.js'
+        'output_filename': 'js/lms-application.js'
     },
     'courseware': {
         'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '') for pth in courseware_only_js],
-        'output_filename': 'js/courseware.js'
+        'output_filename': 'js/lms-courseware.js'
     },
     'main_vendor': {
         'source_filenames': main_vendor_js,
-        'output_filename': 'js/main_vendor.js',
+        'output_filename': 'js/lms-main_vendor.js',
     },
     'module-js': {
         'source_filenames': module_js_sources,
-        'output_filename': 'js/modules.js',
+        'output_filename': 'js/lms-modules.js',
     },
     'spec': {
         'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '') for pth in glob2.glob(PROJECT_ROOT / 'static/coffee/spec/**/*.coffee')],
-        'output_filename': 'js/spec.js'
+        'output_filename': 'js/lms-spec.js'
     }
 }
 

lms/envs/test.py

@@ -12,17 +12,21 @@ from .logsettings import get_logger_config
 import os
 from path import path
 
-INSTALLED_APPS = [
-    app
-    for app
-    in INSTALLED_APPS
-    if not app.startswith('askbot')
-]
+# can't test start dates with this True, but on the other hand,
+# can test everything else :)
+MITX_FEATURES['DISABLE_START_DATES'] = True
+
+# Need wiki for courseware views to work. TODO (vshnayder): shouldn't need it.
+WIKI_ENABLED = True
+
+# Makes the tests run much faster...
+SOUTH_TESTS_MIGRATE = False # To disable migrations and use syncdb instead
 
 # Nose Test Runner
-INSTALLED_APPS += ['django_nose']
+INSTALLED_APPS += ('django_nose',)
 NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html',
-             '--cover-inclusive', '--cover-html-dir', os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
+             '--cover-inclusive', '--cover-html-dir',
+             os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
 for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
     NOSE_ARGS += ['--cover-package', app]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
@@ -30,25 +34,23 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 # Local Directories
 TEST_ROOT = path("test_root")
 # Want static files in the same dir for running on jenkins.
-STATIC_ROOT = TEST_ROOT / "staticfiles" 
+STATIC_ROOT = TEST_ROOT / "staticfiles"
 
 COURSES_ROOT = TEST_ROOT / "data"
 DATA_DIR = COURSES_ROOT
-MAKO_TEMPLATES['course'] = [DATA_DIR]
-MAKO_TEMPLATES['sections'] = [DATA_DIR / 'sections']
-MAKO_TEMPLATES['custom_tags'] = [DATA_DIR / 'custom_tags']
-MAKO_TEMPLATES['main'] = [PROJECT_ROOT / 'templates', 
-                          DATA_DIR / 'info',
-                          DATA_DIR / 'problems']
-
-LOGGING = get_logger_config(TEST_ROOT / "log", 
+
+LOGGING = get_logger_config(TEST_ROOT / "log",
                             logging_env="dev",
                             tracking_filename="tracking.log",
                             debug=True)
 
 COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"
+# Where the content data is checked out.  This may not exist on jenkins.
+GITHUB_REPO_ROOT = ENV_ROOT / "data"
+
 
-# TODO (cpennington): We need to figure out how envs/test.py can inject things into common.py so that we don't have to repeat this sort of thing
+# TODO (cpennington): We need to figure out how envs/test.py can inject things
+# into common.py so that we don't have to repeat this sort of thing
 STATICFILES_DIRS = [
     COMMON_ROOT / "static",
     PROJECT_ROOT / "static",
@@ -67,7 +69,7 @@ DATABASES = {
 }
 
 CACHES = {
-    # This is the cache used for most things. Askbot will not work without a 
+    # This is the cache used for most things. Askbot will not work without a
     # functioning cache -- it relies on caching to load its settings in places.
     # In staging/prod envs, the sessions also live here.
     'default': {

lms/envs/with_cms.py

+"""
+Settings for the LMS that runs alongside the CMS on AWS
+"""
+
+from .aws import *
+
+with open(ENV_ROOT / "cms.auth.json") as auth_file:
+    CMS_AUTH_TOKENS = json.load(auth_file)
+
+MODULESTORE = CMS_AUTH_TOKENS['MODULESTORE']

lms/static/.gitignore

@@ -5,4 +5,5 @@
 *.orig
 *.DS_Store
 application.css
-ie.css
\ No newline at end of file
+ie.css
+Gemfile.lock

lms/urls.py

@@ -13,23 +13,23 @@ if settings.DEBUG:
 urlpatterns = ('',
     url(r'^$', 'student.views.index', name="root"), # Main marketing page, or redirect to courseware
     url(r'^dashboard$', 'student.views.dashboard', name="dashboard"),
-    
+
     url(r'^change_email$', 'student.views.change_email_request'),
     url(r'^email_confirm/(?P<key>[^/]*)$', 'student.views.confirm_email_change'),
     url(r'^change_name$', 'student.views.change_name_request'),
     url(r'^accept_name_change$', 'student.views.accept_name_change'),
     url(r'^reject_name_change$', 'student.views.reject_name_change'),
     url(r'^pending_name_changes$', 'student.views.pending_name_changes'),
-    
+
     url(r'^event$', 'track.views.user_track'),
     url(r'^t/(?P<template>[^/]*)$', 'static_template_view.views.index'), # TODO: Is this used anymore? What is STATIC_GRAB?
-    
-    url(r'^login$', 'student.views.login_user'),
+
+    url(r'^login$', 'student.views.login_user', name="login"),
     url(r'^login/(?P<error>[^/]*)$', 'student.views.login_user'),
     url(r'^logout$', 'student.views.logout_user', name='logout'),
     url(r'^create_account$', 'student.views.create_account'),
-    url(r'^activate/(?P<key>[^/]*)$', 'student.views.activate_account'),
-    
+    url(r'^activate/(?P<key>[^/]*)$', 'student.views.activate_account', name="activate"),
+
     url(r'^password_reset/$', 'student.views.password_reset', name='password_reset'),
     ## Obsolete Django views for password resets
     ## TODO: Replace with Mako-ized views
@@ -44,48 +44,48 @@ urlpatterns = ('',
         name='auth_password_reset_complete'),
     url(r'^password_reset_done/$', django.contrib.auth.views.password_reset_done,
         name='auth_password_reset_done'),
-    
+
     url(r'^heartbeat$', include('heartbeat.urls')),
-    
+
     url(r'^university_profile/(?P<org_id>[^/]+)$', 'courseware.views.university_profile', name="university_profile"),
-    
+
     #Semi-static views (these need to be rendered and have the login bar, but don't change)
-    url(r'^404$', 'static_template_view.views.render', 
+    url(r'^404$', 'static_template_view.views.render',
         {'template': '404.html'}, name="404"),
-    url(r'^about$', 'static_template_view.views.render', 
+    url(r'^about$', 'static_template_view.views.render',
         {'template': 'about.html'}, name="about_edx"),
-    url(r'^jobs$', 'static_template_view.views.render', 
+    url(r'^jobs$', 'static_template_view.views.render',
         {'template': 'jobs.html'}, name="jobs"),
-    url(r'^contact$', 'static_template_view.views.render', 
+    url(r'^contact$', 'static_template_view.views.render',
         {'template': 'contact.html'}, name="contact"),
     url(r'^press$', 'student.views.press', name="press"),
-    url(r'^faq$', 'static_template_view.views.render', 
+    url(r'^faq$', 'static_template_view.views.render',
         {'template': 'faq.html'}, name="faq_edx"),
-    url(r'^help$', 'static_template_view.views.render', 
+    url(r'^help$', 'static_template_view.views.render',
         {'template': 'help.html'}, name="help_edx"),
 
-    url(r'^tos$', 'static_template_view.views.render', 
+    url(r'^tos$', 'static_template_view.views.render',
         {'template': 'tos.html'}, name="tos"),
-    url(r'^privacy$', 'static_template_view.views.render', 
+    url(r'^privacy$', 'static_template_view.views.render',
         {'template': 'privacy.html'}, name="privacy_edx"),
     # TODO: (bridger) The copyright has been removed until it is updated for edX
-    # url(r'^copyright$', 'static_template_view.views.render', 
+    # url(r'^copyright$', 'static_template_view.views.render',
     #     {'template': 'copyright.html'}, name="copyright"),
-    url(r'^honor$', 'static_template_view.views.render', 
+    url(r'^honor$', 'static_template_view.views.render',
         {'template': 'honor.html'}, name="honor"),
-        
-    #Press releases    
-    url(r'^press/mit-and-harvard-announce-edx$', 'static_template_view.views.render', 
+
+    #Press releases
+    url(r'^press/mit-and-harvard-announce-edx$', 'static_template_view.views.render',
         {'template': 'press_releases/MIT_and_Harvard_announce_edX.html'}, name="press/mit-and-harvard-announce-edx"),
-    url(r'^press/uc-berkeley-joins-edx$', 'static_template_view.views.render', 
+    url(r'^press/uc-berkeley-joins-edx$', 'static_template_view.views.render',
         {'template': 'press_releases/UC_Berkeley_joins_edX.html'}, name="press/uc-berkeley-joins-edx"),
     # Should this always update to point to the latest press release?
-    (r'^pressrelease$', 'django.views.generic.simple.redirect_to', {'url': '/press/uc-berkeley-joins-edx'}), 
-    
-    
-    
+    (r'^pressrelease$', 'django.views.generic.simple.redirect_to', {'url': '/press/uc-berkeley-joins-edx'}),
+
+
+
     (r'^favicon\.ico$', 'django.views.generic.simple.redirect_to', {'url': '/static/images/favicon.ico'}),
-        
+
     # TODO: These urls no longer work. They need to be updated before they are re-enabled
     # url(r'^send_feedback$', 'util.views.send_feedback'),
     # url(r'^reactivate/(?P<key>[^/]*)$', 'student.views.reactivation_email'),
@@ -97,42 +97,43 @@ if settings.PERFSTATS:
 if settings.COURSEWARE_ENABLED:
     urlpatterns += (
         url(r'^masquerade/', include('masquerade.urls')),
-        url(r'^jumpto/(?P<probname>[^/]+)/$', 'courseware.views.jump_to'),
+        url(r'^jump_to/(?P<location>.*)$', 'courseware.views.jump_to', name="jump_to"),
+
         url(r'^modx/(?P<id>.*?)/(?P<dispatch>[^/]*)$', 'courseware.module_render.modx_dispatch'), #reset_problem'),
         url(r'^xqueue/(?P<userid>[^/]*)/(?P<id>.*?)/(?P<dispatch>[^/]*)$', 'courseware.module_render.xqueue_callback'),
         url(r'^change_setting$', 'student.views.change_setting'),
-        
+
         # TODO: These views need to be updated before they work
         # url(r'^calculate$', 'util.views.calculate'),
         # url(r'^gradebook$', 'courseware.views.gradebook'),
         # TODO: We should probably remove the circuit package. I believe it was only used in the old way of saving wiki circuits for the wiki
         # url(r'^edit_circuit/(?P<circuit>[^/]*)$', 'circuit.views.edit_circuit'),
         # url(r'^save_circuit/(?P<circuit>[^/]*)$', 'circuit.views.save_circuit'),
-                
-        url(r'^courses/?$', 'courseware.views.courses', name="courses"),     
-        url(r'^change_enrollment$', 
+
+        url(r'^courses/?$', 'courseware.views.courses', name="courses"),
+        url(r'^change_enrollment$',
             'student.views.change_enrollment_view', name="change_enrollment"),
-           
+
         #About the course
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/about$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/about$',
             'courseware.views.course_about', name="about_course"),
-        
+
         #Inside the course
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/info$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/info$',
             'courseware.views.course_info', name="info"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book$',
             'staticbook.views.index', name="book"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book/(?P<page>[^/]*)$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book/(?P<page>[^/]*)$',
             'staticbook.views.index'),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book-shifted/(?P<page>[^/]*)$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book-shifted/(?P<page>[^/]*)$',
             'staticbook.views.index_shifted'),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/?$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/?$',
             'courseware.views.index', name="courseware"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/$',
             'courseware.views.index', name="courseware_section"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile$',
             'courseware.views.profile', name="profile"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile/(?P<student_id>[^/]*)/$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile/(?P<student_id>[^/]*)/$',
             'courseware.views.profile'),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/news$', 
             'courseware.views.news', name="news"),
@@ -141,7 +142,7 @@ if settings.COURSEWARE_ENABLED:
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/discussion/',
             include('django_comment_client.urls')),
     )
-    
+
     # Multicourse wiki
 if settings.WIKI_ENABLED:
     urlpatterns += (
@@ -170,9 +171,9 @@ urlpatterns = patterns(*urlpatterns)
 
 if settings.DEBUG:
     urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
-    
-    
-#Custom error pages 
+
+
+#Custom error pages
 handler404 = 'static_template_view.views.render_404'
 handler500 = 'static_template_view.views.render_500'
 

rakefile

@@ -27,7 +27,7 @@ NORMALIZED_DEPLOY_NAME = DEPLOY_NAME.downcase().gsub(/[_\/]/, '-')
 INSTALL_DIR_PATH = File.join(DEPLOY_DIR, NORMALIZED_DEPLOY_NAME)
 PIP_REPO_REQUIREMENTS = "#{INSTALL_DIR_PATH}/repo-requirements.txt"
 # Set up the clean and clobber tasks
-CLOBBER.include(BUILD_DIR, REPORT_DIR, 'cover*', '.coverage', 'test_root/*_repo')
+CLOBBER.include(BUILD_DIR, REPORT_DIR, 'cover*', '.coverage', 'test_root/*_repo', 'test_root/staticfiles')
 CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util")
 
 def select_executable(*cmds)
@@ -51,6 +51,11 @@ default_options = {
 task :predjango do
     sh("find . -type f -name *.pyc -delete")
     sh('pip install -e common/lib/xmodule')
+    sh('git submodule update --init')
+end
+
+task :clean_test_files do
+    sh("git clean -fdx test_root")
 end
 
 [:lms, :cms, :common].each do |system|
@@ -92,7 +97,7 @@ end
 
     # Per System tasks
     desc "Run all django tests on our djangoapps for the #{system}"
-    task "test_#{system}" => ["#{system}:collectstatic:test", "fasttest_#{system}"]
+    task "test_#{system}" => ["clean_test_files", "#{system}:collectstatic:test", "fasttest_#{system}"]
 
     # Have a way to run the tests without running collectstatic -- useful when debugging without
     # messing with static files.
@@ -150,14 +155,14 @@ end
 
 task :package do
     FileUtils.mkdir_p(BUILD_DIR)
-    
+
     Dir.chdir(BUILD_DIR) do
         afterremove = Tempfile.new('afterremove')
         afterremove.write <<-AFTERREMOVE.gsub(/^\s*/, '')
         #! /bin/bash
         set -e
         set -x
-        
+
         # to be a little safer this rm is executed
         # as the makeitso user