Merge branch 'master' of github.com:MITx/mitx into jarv/add-local-logging

cms/djangoapps/github_sync/tests/__init__.py

@@ -61,7 +61,7 @@ class GithubSyncTestCase(TestCase):
         self.assertIn(
             Location('i4x://edX/toy/chapter/Overview'),
             [child.location for child in self.import_course.get_children()])
-        self.assertEquals(1, len(self.import_course.get_children()))
+        self.assertEquals(2, len(self.import_course.get_children()))
 
     @patch('github_sync.sync_with_github')
     def test_sync_all_with_github(self, sync_with_github):

common/lib/capa/capa/capa_problem.py

@@ -14,6 +14,8 @@ This is used by capa_module.
 
 from __future__ import division
 
+from datetime import datetime
+import json
 import logging
 import math
 import numpy
@@ -32,6 +34,7 @@ from correctmap import CorrectMap
 import eia
 import inputtypes
 from util import contextualize_text, convert_files_to_filenames
+import xqueue_interface
 
 # to be replaced with auto-registering
 import responsetypes
@@ -202,11 +205,24 @@ class LoncapaProblem(object):
         '''
         Returns True if any part of the problem has been submitted to an external queue
         '''
-        queued = False
-        for answer_id in self.correct_map:
-            if self.correct_map.is_queued(answer_id):
-                queued = True
-        return queued
+        return any(self.correct_map.is_queued(answer_id) for answer_id in self.correct_map)
+
+
+    def get_recentmost_queuetime(self):
+        '''
+        Returns a DateTime object that represents the timestamp of the most recent queueing request, or None if not queued
+        '''
+        if not self.is_queued():
+            return None
+
+        # Get a list of timestamps of all queueing requests, then convert it to a DateTime object
+        queuetime_strs = [self.correct_map.get_queuetime_str(answer_id)
+                          for answer_id in self.correct_map 
+                          if self.correct_map.is_queued(answer_id)]
+        queuetimes = [datetime.strptime(qt_str, xqueue_interface.dateformat) for qt_str in queuetime_strs]
+
+        return max(queuetimes)
+
 
     def grade_answers(self, answers):
         '''

common/lib/capa/capa/correctmap.py

@@ -15,7 +15,8 @@ class CorrectMap(object):
     - msg         : string (may have HTML) giving extra message response (displayed below textline or textbox)
     - hint        : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg)
     - hintmode    : one of (None,'on_request','always') criteria for displaying hint
-    - queuekey    : a random integer for xqueue_callback verification
+    - queuestate  : Dict {key:'', time:''} where key is a secret string, and time is a string dump
+                    of a DateTime object in the format '%Y%m%d%H%M%S'. Is None when not queued
 
     Behaves as a dict.
     '''
@@ -31,14 +32,15 @@ class CorrectMap(object):
     def __iter__(self):
         return self.cmap.__iter__()
 
-    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuekey=None):
+    # See the documentation for 'set_dict' for the use of kwargs
+    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None, **kwargs):
         if answer_id is not None:
             self.cmap[answer_id] = {'correctness': correctness,
                                     'npoints': npoints,
                                     'msg': msg,
                                     'hint': hint,
                                     'hintmode': hintmode,
-                                    'queuekey': queuekey,
+                                    'queuestate': queuestate,
                                     }
 
     def __repr__(self):
@@ -52,25 +54,39 @@ class CorrectMap(object):
 
     def set_dict(self, correct_map):
         '''
-        set internal dict to provided correct_map dict
-        for graceful migration, if correct_map is a one-level dict, then convert it to the new
-        dict of dicts format.
+        Set internal dict of CorrectMap to provided correct_map dict
+
+        correct_map is saved by LMS as a plaintext JSON dump of the correctmap dict. This means that
+        when the definition of CorrectMap (e.g. its properties) are altered, existing correct_map dict
+        not coincide with the newest CorrectMap format as defined by self.set.
+
+        For graceful migration, feed the contents of each correct map to self.set, rather than
+        making a direct copy of the given correct_map dict. This way, the common keys between 
+        the incoming correct_map dict and the new CorrectMap instance will be written, while
+        mismatched keys will be gracefully ignored.
+
+        Special migration case:
+            If correct_map is a one-level dict, then convert it to the new dict of dicts format.
         '''
         if correct_map and not (type(correct_map[correct_map.keys()[0]]) == dict):
-            self.__init__()							# empty current dict
-            for k in correct_map: self.set(k, correct_map[k])			# create new dict entries
+            self.__init__()	                                  # empty current dict
+            for k in correct_map: self.set(k, correct_map[k]) # create new dict entries
         else:
-            self.cmap = correct_map
+            self.__init__()
+            for k in correct_map: self.set(k, **correct_map[k])
 
     def is_correct(self, answer_id):
         if answer_id in self.cmap: return self.cmap[answer_id]['correctness'] == 'correct'
         return None
 
     def is_queued(self, answer_id):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] is not None
+        return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
 
     def is_right_queuekey(self, answer_id, test_key):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] == test_key
+        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
+
+    def get_queuetime_str(self, answer_id):
+        return self.cmap[answer_id]['queuestate']['time']
 
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')

common/lib/capa/capa/inputtypes.py

@@ -351,7 +351,7 @@ def filesubmission(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     context = { 'id': eid, 'state': status, 'msg': msg, 'value': value,
             'queue_len': queue_len, 'allowed_files': allowed_files,
@@ -384,7 +384,7 @@ def textbox(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     # For CodeMirror
     mode = element.get('mode','python')

common/lib/capa/capa/responsetypes.py

@@ -8,6 +8,7 @@ Used by capa_problem.py
 '''
 
 # standard library imports
+import cgi
 import inspect
 import json
 import logging
@@ -26,6 +27,7 @@ import xml.sax.saxutils as saxutils
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
+from datetime import datetime
 from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
@@ -317,30 +319,37 @@ class JavascriptResponse(LoncapaResponse):
     
     def compile_display_javascript(self):
 
-        latestTimestamp = 0
-        basepath = self.system.filestore.root_path + '/js/'
-        for filename in (self.display_dependencies + [self.display]):
-            filepath = basepath + filename
-            timestamp = os.stat(filepath).st_mtime
-            if timestamp > latestTimestamp:
-                latestTimestamp = timestamp
-        
-        h = hashlib.md5()
-        h.update(self.answer_id + str(self.display_dependencies))
-        compiled_filename = 'compiled/' + h.hexdigest() + '.js'
-        compiled_filepath = basepath + compiled_filename
-
-        if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
-            outfile = open(compiled_filepath, 'w')
-            for filename in (self.display_dependencies + [self.display]):
-                filepath = basepath + filename
-                infile = open(filepath, 'r')
-                outfile.write(infile.read())
-                outfile.write(';\n')
-                infile.close()
-            outfile.close()
-
-        self.display_filename = compiled_filename
+        # TODO FIXME
+        # arjun: removing this behavior for now (and likely forever). Keeping
+        # until we decide on exactly how to solve this issue. For now, files are
+        # manually being compiled to DATA_DIR/js/compiled.
+
+        #latestTimestamp = 0
+        #basepath = self.system.filestore.root_path + '/js/'
+        #for filename in (self.display_dependencies + [self.display]):
+        #    filepath = basepath + filename
+        #    timestamp = os.stat(filepath).st_mtime
+        #    if timestamp > latestTimestamp:
+        #        latestTimestamp = timestamp
+        #
+        #h = hashlib.md5()
+        #h.update(self.answer_id + str(self.display_dependencies))
+        #compiled_filename = 'compiled/' + h.hexdigest() + '.js'
+        #compiled_filepath = basepath + compiled_filename
+
+        #if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
+        #    outfile = open(compiled_filepath, 'w')
+        #    for filename in (self.display_dependencies + [self.display]):
+        #        filepath = basepath + filename
+        #        infile = open(filepath, 'r')
+        #        outfile.write(infile.read())
+        #        outfile.write(';\n')
+        #        infile.close()
+        #    outfile.close()
+
+        # TODO this should also be fixed when the above is fixed.
+        filename = self.system.ajax_url.split('/')[-1] + '.js'
+        self.display_filename = 'compiled/' + filename
 
     def parse_xml(self):
         self.generator_xml = self.xml.xpath('//*[@id=$id]//generator',
@@ -384,19 +393,23 @@ class JavascriptResponse(LoncapaResponse):
         node_path = self.system.node_path + ":" + os.path.normpath(js_dir)
         tmp_env["NODE_PATH"] = node_path
         return tmp_env
+    
+    def call_node(self, args):
+
+        subprocess_args = ["node"]
+        subprocess_args.extend(args)
+
+        return subprocess.check_output(subprocess_args, env=self.get_node_env())
 
 
     def generate_problem_state(self):
 
         generator_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_generator.js'
-        output = subprocess.check_output(["node",
-                                          generator_file, 
-                                          self.generator, 
-                                          json.dumps(self.generator_dependencies),
-                                          json.dumps(str(self.system.seed)), 
-                                          json.dumps(self.params)
-                                          ],
-                                          env=self.get_node_env()).strip()
+        output = self.call_node([generator_file,
+                                 self.generator, 
+                                 json.dumps(self.generator_dependencies),
+                                 json.dumps(str(self.system.seed)), 
+                                 json.dumps(self.params)]).strip()
 
         return json.loads(output)
 
@@ -407,7 +420,8 @@ class JavascriptResponse(LoncapaResponse):
         for param in self.xml.xpath('//*[@id=$id]//responseparam', 
                                         id=self.xml.get('id')):
 
-            params[param.get("name")] = json.loads(param.get("value"))
+            raw_param = param.get("value")
+            params[param.get("name")] = json.loads(contextualize_text(raw_param, self.context))
         
         return params
 
@@ -435,22 +449,23 @@ class JavascriptResponse(LoncapaResponse):
         (all_correct, evaluation, solution) = self.run_grader(json_submission)
         self.solution = solution
         correctness = 'correct' if all_correct else 'incorrect'
-        return CorrectMap(self.answer_id, correctness, msg=evaluation)
+        if all_correct:
+            points = self.get_max_score()
+        else:
+            points = 0
+        return CorrectMap(self.answer_id, correctness, npoints=points, msg=evaluation)
     
     def run_grader(self, submission):
         if submission is None or submission == '':
             submission = json.dumps(None)
 
         grader_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_grader.js'
-        outputs = subprocess.check_output(["node",
-                                           grader_file, 
-                                           self.grader, 
-                                           json.dumps(self.grader_dependencies),
-                                           submission, 
-                                           json.dumps(self.problem_state), 
-                                           json.dumps(self.params)
-                                          ],
-                                          env=self.get_node_env()).split('\n')
+        outputs = self.call_node([grader_file, 
+                                  self.grader, 
+                                  json.dumps(self.grader_dependencies),
+                                  submission, 
+                                  json.dumps(self.problem_state), 
+                                  json.dumps(self.params)]).split('\n')
 
         all_correct = json.loads(outputs[0].strip())
         evaluation  = outputs[1].strip()
@@ -711,7 +726,8 @@ class NumericalResponse(LoncapaResponse):
         # I think this is just pyparsing.ParseException, calc.UndefinedVariable:
         # But we'd need to confirm
         except:
-            raise StudentInputError('Invalid input -- please use a number only')
+            raise StudentInputError("Invalid input: could not parse '%s' as a number" %\
+                                    cgi.escape(student_answer))
 
         if correct:
             return CorrectMap(self.answer_id, 'correct')
@@ -1005,7 +1021,7 @@ class CodeResponse(LoncapaResponse):
     '''
     Grade student code using an external queueing server, called 'xqueue'
 
-    Expects 'xqueue' dict in ModuleSystem with the following keys:
+    Expects 'xqueue' dict in ModuleSystem with the following keys that are needed by CodeResponse:
         system.xqueue = { 'interface': XqueueInterface object,
                           'callback_url': Per-StudentModule callback URL where results are posted (string),
                           'default_queuename': Default queuename to submit request (string)
@@ -1026,7 +1042,7 @@ class CodeResponse(LoncapaResponse):
         TODO: Determines whether in synchronous or asynchronous (queued) mode
         '''
         xml = self.xml
-        self.url = xml.get('url', None) # XML can override external resource (grader/queue) URL
+        self.url = xml.get('url', None) # TODO: XML can override external resource (grader/queue) URL
         self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
 
         # VS[compat]:
@@ -1121,22 +1137,34 @@ class CodeResponse(LoncapaResponse):
 
         # Prepare xqueue request
         #------------------------------------------------------------ 
+
         qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
+
+        anonymous_student_id = self.system.anonymous_student_id
 
         # Generate header
-        queuekey = xqueue_interface.make_hashkey(str(self.system.seed)+self.answer_id)
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
+                                                 anonymous_student_id +  
+                                                 self.answer_id)
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
                                                 lms_key=queuekey,
                                                 queue_name=self.queue_name)
-
+        
         # Generate body
         if is_list_of_files(submission):
-            self.context.update({'submission': queuekey}) # For tracking. TODO: May want to record something else here
+            self.context.update({'submission': ''}) # TODO: Get S3 pointer from the Queue
         else:
             self.context.update({'submission': submission})
 
         contents = self.payload.copy() 
 
+        # Metadata related to the student submission revealed to the external grader
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                        'submission_time': qtime,
+                       }
+        contents.update({'student_info': json.dumps(student_info)})
+
         # Submit request. When successful, 'msg' is the prior length of the queue
         if is_list_of_files(submission):
             contents.update({'student_response': ''}) # TODO: Is there any information we want to send here?
@@ -1148,16 +1176,21 @@ class CodeResponse(LoncapaResponse):
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents))
 
+        # State associated with the queueing request
+        queuestate = {'key': queuekey,
+                      'time': qtime,
+                     }
+
         cmap = CorrectMap() 
         if error:
-            cmap.set(self.answer_id, queuekey=None,
+            cmap.set(self.answer_id, queuestate=None,
                      msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg)
         else:
             # Queueing mechanism flags:
-            #   1) Backend: Non-null CorrectMap['queuekey'] indicates that the problem has been queued
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that the problem has been queued
             #   2) Frontend: correctness='incomplete' eventually trickles down through inputtypes.textbox 
             #       and .filesubmission to inform the browser to poll the LMS
-            cmap.set(self.answer_id, queuekey=queuekey, correctness='incomplete', msg=msg)
+            cmap.set(self.answer_id, queuestate=queuestate, correctness='incomplete', msg=msg)
 
         return cmap
 
@@ -1165,7 +1198,7 @@ class CodeResponse(LoncapaResponse):
 
         (valid_score_msg, correct, points, msg) = self._parse_score_msg(score_msg) 
         if not valid_score_msg:
-            oldcmap.set(self.answer_id, msg='Error: Invalid grader reply.')
+            oldcmap.set(self.answer_id, msg='Invalid grader reply. Please contact the course staff.')
             return oldcmap
         
         correctness = 'correct' if correct else 'incorrect'
@@ -1180,7 +1213,7 @@ class CodeResponse(LoncapaResponse):
                 points = 0
             elif points > self.maxpoints[self.answer_id]:
                 points = self.maxpoints[self.answer_id]
-            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuekey=None)  # Queuekey is consumed
+            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)  # Queuestate is consumed
         else:
             log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id))
 
@@ -1197,26 +1230,41 @@ class CodeResponse(LoncapaResponse):
         '''
          Grader reply is a JSON-dump of the following dict
            { 'correct': True/False,
-             'score': # TODO -- Partial grading
+             'score': Numeric value (floating point is okay) to assign to answer
              'msg': grader_msg }
 
         Returns (valid_score_msg, correct, score, msg):
             valid_score_msg: Flag indicating valid score_msg format (Boolean)
             correct:         Correctness of submission (Boolean)
-            score:           # TODO: Implement partial grading
+            score:           Points to be assigned (numeric, can be float)
             msg:             Message from grader to display to student (string)
         '''
-        fail = (False, False, -1, '')
+        fail = (False, False, 0, '')
         try:
             score_result = json.loads(score_msg)
         except (TypeError, ValueError):
+            log.error("External grader message should be a JSON-serialized dict. Received score_msg = %s" % score_msg)
             return fail
         if not isinstance(score_result, dict):
+            log.error("External grader message should be a JSON-serialized dict. Received score_result = %s" % score_result)
             return fail
         for tag in ['correct', 'score', 'msg']:
-            if not score_result.has_key(tag):
+            if tag not in score_result:
+                log.error("External grader message is missing one or more required tags: 'correct', 'score', 'msg'")
                 return fail
-        return (True, score_result['correct'], score_result['score'], score_result['msg'])
+
+        # Next, we need to check that the contents of the external grader message 
+        #   is safe for the LMS.
+        # 1) Make sure that the message is valid XML (proper opening/closing tags)
+        # 2) TODO: Is the message actually HTML?
+        msg = score_result['msg']
+        try:
+            etree.fromstring(msg)
+        except etree.XMLSyntaxError as err:
+            log.error("Unable to parse external grader message as valid XML: score_msg['msg']=%s" % msg)
+            return fail
+            
+        return (True, score_result['correct'], score_result['score'], msg)
         
 
 #-----------------------------------------------------------------------------
@@ -1471,11 +1519,12 @@ class FormulaResponse(LoncapaResponse):
                                            cs=self.case_sensitive)
             except UndefinedVariable as uv:
                 log.debug('formularesponse: undefined variable in given=%s' % given)
-                raise StudentInputError(uv.message + " not permitted in answer")
+                raise StudentInputError("Invalid input: " + uv.message + " not permitted in answer")
             except Exception as err:
                 #traceback.print_exc()
                 log.debug('formularesponse: error %s in formula' % err)
-                raise StudentInputError("Error in formula")
+                raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %\
+                                        cgi.escape(given))
             if numpy.isnan(student_result) or numpy.isinf(student_result):
                 return "incorrect"
             if not compare_with_tolerance(student_result, instructor_result, self.tolerance):

common/lib/capa/capa/templates/filesubmission.html

@@ -10,7 +10,7 @@
     <span class="processing" id="status_${id}"></span>
     <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
-    <span class="debug">(${state})</span>
+    <span style="display:none;" class="debug">(${state})</span>
     <br/>
     <span class="message">${msg|n}</span>
     <br/>

common/lib/capa/capa/templates/textbox.html

@@ -21,7 +21,7 @@
   <div style="display:none;" name="${hidden}" inputid="input_${id}" />
   % endif
   <br/>
-  <span class="debug">(${state})</span>
+  <span style="display:none;" class="debug">(${state})</span>
   <br/>
   <span class="message">${msg|n}</span>
   <br/>

common/lib/capa/capa/xqueue_interface.py

@@ -5,20 +5,17 @@ import hashlib
 import json
 import logging
 import requests
-import time
 
 
 log = logging.getLogger('mitx.' + __name__)
+dateformat = '%Y%m%d%H%M%S'
 
-
-def make_hashkey(seed=None):
+def make_hashkey(seed):
     '''
     Generate a string key by hashing 
     '''
     h = hashlib.md5()
-    if seed is not None:
-        h.update(str(seed))
-    h.update(str(time.time()))
+    h.update(str(seed))
     return h.hexdigest()
 
 

common/lib/xmodule/xmodule/capa_module.py

@@ -462,6 +462,15 @@ class CapaModule(XModule):
             self.system.track_function('save_problem_check_fail', event_info)
             raise NotFoundError('Problem must be reset before it can be checked again')
 
+        # Problem queued. Students must wait a specified waittime before they are allowed to submit
+        if self.lcp.is_queued():
+            current_time = datetime.datetime.now()
+            prev_submit_time = self.lcp.get_recentmost_queuetime() 
+            waittime_between_requests = self.system.xqueue['waittime']
+            if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
+                msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
+                return {'success': msg, 'html': ''} # Prompts a modal dialog in ajax callback 
+
         try:
             old_state = self.lcp.get_state()
             lcp_id = self.lcp.problem_id

common/lib/xmodule/xmodule/discussion_module.py

 from lxml import etree
+from pkg_resources import resource_string, resource_listdir
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
@@ -6,6 +7,11 @@ from xmodule.raw_module import RawDescriptor
 import json
 
 class DiscussionModule(XModule):
+    js = {'coffee':
+            [resource_string(__name__, 'js/src/time.coffee'),
+            resource_string(__name__, 'js/src/discussion/display.coffee')]
+        }
+    js_module_name = "InlineDiscussion"
     def get_html(self):
         context = {
             'discussion_id': self.discussion_id,

common/lib/xmodule/xmodule/html_module.py

@@ -4,9 +4,10 @@ import logging
 import os
 import sys
 from lxml import etree
+from path import path
 
 from .x_module import XModule
-from .xml_module import XmlDescriptor
+from .xml_module import XmlDescriptor, name_to_pathname
 from .editing_module import EditingDescriptor
 from .stringify import stringify_children
 from .html_checker import check_html
@@ -75,9 +76,19 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             cls.clean_metadata_from_xml(definition_xml)
             return {'data': stringify_children(definition_xml)}
         else:
-            # html is special.  cls.filename_extension is 'xml', but if 'filename' is in the definition,
-            # that means to load from .html
-            filepath = "{category}/{name}.html".format(category='html', name=filename)
+            # html is special.  cls.filename_extension is 'xml', but
+            # if 'filename' is in the definition, that means to load
+            # from .html
+            # 'filename' in html pointers is a relative path
+            # (not same as 'html/blah.html' when the pointer is in a directory itself)
+            pointer_path = "{category}/{url_path}".format(category='html',
+                                                  url_path=name_to_pathname(location.name))
+            base = path(pointer_path).dirname()
+            #log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
+            filepath = "{base}/{name}.html".format(base=base, name=filename)
+            #log.debug("looking for html file for {0} at {1}".format(location, filepath))
+
+
 
             # VS[compat]
             # TODO (cpennington): If the file doesn't exist at the right path,
@@ -128,13 +139,18 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             pass
 
         # Not proper format.  Write html to file, return an empty tag
-        filepath = u'{category}/{name}.html'.format(category=self.category,
-                                                    name=self.url_name)
+        pathname = name_to_pathname(self.url_name)
+        pathdir = path(pathname).dirname()
+        filepath = u'{category}/{pathname}.html'.format(category=self.category,
+                                                    pathname=pathname)
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
             file.write(self.definition['data'])
 
+        # write out the relative name
+        relname = path(pathname).basename()
+
         elt = etree.Element('html')
-        elt.set("filename", self.url_name)
+        elt.set("filename", relname)
         return elt

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -192,8 +192,11 @@ class @Problem
     if file_not_selected
       errors.push 'You did not select any files to submit'
 
-    if errors.length > 0
-        alert errors.join("\n")
+    error_html = '<ul>\n'
+    for error in errors
+      error_html += '<li>' + error + '</li>\n'
+    error_html += '</ul>'
+    @gentle_alert error_html
 
     abort_submission = file_too_large or file_not_selected or unallowed_file_submitted or required_files_not_submitted
 
@@ -208,7 +211,7 @@ class @Problem
             @render(response.contents)
             @updateProgress response
           else
-            alert(response.success)
+            @gentle_alert response.success
     
     if not abort_submission
       $.ajaxWithPrefix("#{@url}/problem_check", settings)
@@ -220,8 +223,10 @@ class @Problem
         when 'incorrect', 'correct'
           @render(response.contents)
           @updateProgress response
+          if @el.hasClass 'showed'
+            @el.removeClass 'showed'
         else
-          alert(response.success)
+          @gentle_alert response.success
 
   reset: =>
     Logger.log 'problem_reset', @answers
@@ -253,11 +258,19 @@ class @Problem
       @el.removeClass 'showed'
       @$('.show').val 'Show Answer'
 
+  gentle_alert: (msg) =>
+    if @el.find('.capa_alert').length
+      @el.find('.capa_alert').remove()
+    alert_elem = "<div class='capa_alert'>" + msg + "</div>"
+    @el.find('.action').after(alert_elem)
+    @el.find('.capa_alert').animate(opacity: 0, 500).animate(opacity: 1, 500)
+    
   save: =>
     Logger.log 'problem_save', @answers
     $.postWithPrefix "#{@url}/problem_save", @answers, (response) =>
       if response.success
-        alert 'Saved'
+        saveMessage = "Your answers have been saved but not graded. Hit 'Check' to grade them."
+        @gentle_alert saveMessage
       @updateProgress response
 
   refreshMath: (event, element) =>
@@ -293,6 +306,9 @@ class @Problem
       problemState  = data.data("problem_state")
       displayClass  = window[data.data('display_class')]
 
+      if evaluation == ''
+          evaluation = null
+
       container = $(element).find(".javascriptinput_container")
       submissionField = $(element).find(".javascriptinput_input")
 

common/lib/xmodule/xmodule/js/src/discussion/display.coffee

+class @InlineDiscussion
+  constructor: (element) ->
+    @el = $(element).find('.discussion-module')
+    @view = new DiscussionModuleView(el: @el)

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -2,6 +2,7 @@ class @Sequence
   constructor: (element) ->
     @el = $(element).find('.sequence')
     @contents = @$('.seq_contents')
+    @num_contents = @contents.length
     @id = @el.data('id')
     @modx_url = @el.data('course_modx_root')
     @initProgress()
@@ -90,18 +91,28 @@ class @Sequence
       @toggleArrows()
       @hookUpProgressEvent()
 
+      sequence_links = @$('#seq_content a.seqnav')
+      sequence_links.click @goto
+
   goto: (event) =>
     event.preventDefault()
-    new_position = $(event.target).data('element')
-    Logger.log "seq_goto", old: @position, new: new_position, id: @id
+    if $(event.target).hasClass 'seqnav' # Links from courseware <a class='seqnav' href='n'>...</a>
+      new_position = $(event.target).attr('href')
+    else # Tab links generated by backend template
+      new_position = $(event.target).data('element')
+
+    if (1 <= new_position) and (new_position <= @num_contents)
+      Logger.log "seq_goto", old: @position, new: new_position, id: @id
     
-    # On Sequence chage, destroy any existing polling thread 
-    #   for queued submissions, see ../capa/display.coffee
-    if window.queuePollerID
-      window.clearTimeout(window.queuePollerID)
-      delete window.queuePollerID
+      # On Sequence chage, destroy any existing polling thread 
+      #   for queued submissions, see ../capa/display.coffee
+      if window.queuePollerID
+        window.clearTimeout(window.queuePollerID)
+        delete window.queuePollerID
 
-    @render new_position
+      @render new_position
+    else
+      alert 'Sequence error! Cannot navigate to tab ' + new_position + 'in the current SequenceModule. Please contact the course staff.'
 
   next: (event) =>
     event.preventDefault()

common/lib/xmodule/xmodule/js/src/video/display.coffee

@@ -3,6 +3,7 @@ class @Video
     @el = $(element).find('.video')
     @id = @el.attr('id').replace(/video_/, '')
     @caption_data_dir = @el.data('caption-data-dir')
+    @show_captions = @el.data('show-captions') == "true"
     window.player = null
     @el = $("#video_#{@id}")
     @parseVideos @el.data('streams')

common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee

 class @VideoCaption extends Subview
+  initialize: ->
+    @loaded = false  
+
   bind: ->
     $(window).bind('resize', @resize)
     @$('.hide-subtitles').click @toggle
@@ -10,8 +13,12 @@ class @VideoCaption extends Subview
     "/static/#{@captionDataDir}/subs/#{@youtubeId}.srt.sjson"
 
   render: ->
+    # TODO: make it so you can have a video with no captions.
+    #@$('.video-wrapper').after """
+    #  <ol class="subtitles"><li>Attempting to load captions...</li></ol>
+    #  """
     @$('.video-wrapper').after """
-      <ol class="subtitles"><li>Attempting to load captions...</li></ol>
+      <ol class="subtitles"></ol>
       """
     @$('.video-controls .secondary-controls').append """
       <a href="#" class="hide-subtitles" title="Turn off captions">Captions</a>
@@ -24,6 +31,8 @@ class @VideoCaption extends Subview
       @captions = captions.text
       @start = captions.start
 
+      @loaded = true
+
       if onTouchBasedDevice()
         $('.subtitles li').html "Caption will be displayed when you start playing the video."
       else
@@ -47,37 +56,40 @@ class @VideoCaption extends Subview
     @rendered = true
 
   search: (time) ->
-    min = 0
-    max = @start.length - 1
-
-    while min < max
-      index = Math.ceil((max + min) / 2)
-      if time < @start[index]
-        max = index - 1
-      if time >= @start[index]
-        min = index
-
-    return min
+    if @loaded
+      min = 0
+      max = @start.length - 1
+
+      while min < max
+        index = Math.ceil((max + min) / 2)
+        if time < @start[index]
+          max = index - 1
+        if time >= @start[index]
+          min = index
+      return min
 
   play: ->
-    @renderCaption() unless @rendered
-    @playing = true
+    if @loaded
+      @renderCaption() unless @rendered
+      @playing = true
 
   pause: ->
-    @playing = false
+    if @loaded
+      @playing = false
 
   updatePlayTime: (time) ->
-    # This 250ms offset is required to match the video speed
-    time = Math.round(Time.convert(time, @currentSpeed, '1.0') * 1000 + 250)
-    newIndex = @search time
-
-    if newIndex != undefined && @currentIndex != newIndex
-      if @currentIndex
-        @$(".subtitles li.current").removeClass('current')
-      @$(".subtitles li[data-index='#{newIndex}']").addClass('current')
-
-      @currentIndex = newIndex
-      @scrollCaption()
+    if @loaded
+      # This 250ms offset is required to match the video speed
+      time = Math.round(Time.convert(time, @currentSpeed, '1.0') * 1000 + 250)
+      newIndex = @search time
+
+      if newIndex != undefined && @currentIndex != newIndex
+        if @currentIndex
+          @$(".subtitles li.current").removeClass('current')
+        @$(".subtitles li[data-index='#{newIndex}']").addClass('current')
+
+        @currentIndex = newIndex
+        @scrollCaption()
 
   resize: =>
     @$('.subtitles').css maxHeight: @captionHeight()

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -13,18 +13,21 @@ from xmodule.errortracker import ErrorLog, make_error_tracker
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
+
 URL_RE = re.compile("""
     (?P<tag>[^:]+)://
     (?P<org>[^/]+)/
     (?P<course>[^/]+)/
     (?P<category>[^/]+)/
-    (?P<name>[^/]+)
-    (/(?P<revision>[^/]+))?
+    (?P<name>[^@]+)
+    (@(?P<revision>[^/]+))?
     """, re.VERBOSE)
 
 # TODO (cpennington): We should decide whether we want to expand the
 # list of valid characters in a location
 INVALID_CHARS = re.compile(r"[^\w.-]")
+# Names are allowed to have colons.
+INVALID_CHARS_NAME = re.compile(r"[^\w.:-]")
 
 _LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
 
@@ -34,7 +37,7 @@ class Location(_LocationBase):
     Encodes a location.
 
     Locations representations of URLs of the
-    form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+    form {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
     However, they can also be represented a dictionaries (specifying each component),
     tuples or list (specified in order), or as strings of the url
@@ -81,7 +84,7 @@ class Location(_LocationBase):
 
         location - Can be any of the following types:
             string: should be of the form
-                    {tag}://{org}/{course}/{category}/{name}[/{revision}]
+                    {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
             list: should be of the form [tag, org, course, category, name, revision]
 
@@ -99,10 +102,11 @@ class Location(_LocationBase):
         ommitted.
 
         Components must be composed of alphanumeric characters, or the
-        characters '_', '-', and '.'
+        characters '_', '-', and '.'.  The name component is additionally allowed to have ':',
+        which is interpreted specially for xml storage.
 
-        Components may be set to None, which may be interpreted by some contexts
-        to mean wildcard selection
+        Components may be set to None, which may be interpreted in some contexts
+        to mean wildcard selection.
         """
 
 
@@ -116,14 +120,23 @@ class Location(_LocationBase):
             return _LocationBase.__new__(_cls, *([None] * 6))
 
         def check_dict(dict_):
-            check_list(dict_.itervalues())
+            # Order matters, so flatten out into a list
+            keys = ['tag', 'org', 'course', 'category', 'name', 'revision']
+            list_ = [dict_[k] for k in keys]
+            check_list(list_)
 
         def check_list(list_):
-            for val in list_:
-                if val is not None and INVALID_CHARS.search(val) is not None:
+            def check(val, regexp):
+                if val is not None and regexp.search(val) is not None:
                     log.debug('invalid characters val="%s", list_="%s"' % (val, list_))
                     raise InvalidLocationError(location)
 
+            list_ = list(list_)
+            for val in list_[:4] + [list_[5]]:
+                check(val, INVALID_CHARS)
+            # names allow colons
+            check(list_[4], INVALID_CHARS_NAME)
+
         if isinstance(location, basestring):
             match = URL_RE.match(location)
             if match is None:
@@ -162,7 +175,7 @@ class Location(_LocationBase):
         """
         url = "{tag}://{org}/{course}/{category}/{name}".format(**self.dict())
         if self.revision:
-            url += "/" + self.revision
+            url += "@" + self.revision
         return url
 
     def html_id(self):
@@ -170,6 +183,7 @@ class Location(_LocationBase):
         Return a string with a version of the location that is safe for use in
         html id attributes
         """
+        # TODO: is ':' ok in html ids?
         return "-".join(str(v) for v in self.list()
                         if v is not None).replace('.', '_')
 

common/lib/xmodule/xmodule/modulestore/django.py

@@ -5,8 +5,8 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
 """
 
 from __future__ import absolute_import
-
 from importlib import import_module
+from os import environ
 
 from django.conf import settings
 
@@ -43,3 +43,8 @@ def modulestore(name='default'):
         )
 
     return _MODULESTORES[name]
+
+# if 'DJANGO_SETTINGS_MODULE' in environ:
+#     # Initialize the modulestores immediately
+#     for store_name in settings.MODULESTORE:
+#         modulestore(store_name)

common/lib/xmodule/xmodule/modulestore/tests/test_location.py

@@ -10,7 +10,7 @@ def check_string_roundtrip(url):
 
 def test_string_roundtrip():
     check_string_roundtrip("tag://org/course/category/name")
-    check_string_roundtrip("tag://org/course/category/name/revision")
+    check_string_roundtrip("tag://org/course/category/name@revision")
 
 
 input_dict = {
@@ -21,18 +21,28 @@ input_dict = {
     'org': 'org'
 }
 
+
+also_valid_dict = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name:more_name',
+    'org': 'org'
+}
+
+
 input_list = ['tag', 'org', 'course', 'category', 'name']
 
 input_str = "tag://org/course/category/name"
-input_str_rev = "tag://org/course/category/name/revision"
+input_str_rev = "tag://org/course/category/name@revision"
 
-valid = (input_list, input_dict, input_str, input_str_rev)
+valid = (input_list, input_dict, input_str, input_str_rev, also_valid_dict)
 
 invalid_dict = {
     'tag': 'tag',
     'course': 'course',
     'category': 'category',
-    'name': 'name/more_name',
+    'name': 'name@more_name',
     'org': 'org'
 }
 
@@ -45,8 +55,9 @@ invalid_dict2 = {
 }
 
 invalid = ("foo", ["foo"], ["foo", "bar"],
-           ["foo", "bar", "baz", "blat", "foo/bar"],
-           "tag://org/course/category/name with spaces/revision",
+           ["foo", "bar", "baz", "blat:blat", "foo:bar"],  # ':' ok in name, not in category
+           "tag://org/course/category/name with spaces@revision",
+           "tag://org/course/category/name/with/slashes@revision",
            invalid_dict,
            invalid_dict2)
 
@@ -62,16 +73,15 @@ def test_dict():
     assert_equals(dict(revision=None, **input_dict), Location(input_dict).dict())
 
     input_dict['revision'] = 'revision'
-    assert_equals("tag://org/course/category/name/revision", Location(input_dict).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_dict).url())
     assert_equals(input_dict, Location(input_dict).dict())
 
-
 def test_list():
     assert_equals("tag://org/course/category/name", Location(input_list).url())
     assert_equals(input_list + [None], Location(input_list).list())
 
     input_list.append('revision')
-    assert_equals("tag://org/course/category/name/revision", Location(input_list).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_list).url())
     assert_equals(input_list, Location(input_list).list())
 
 
@@ -87,8 +97,10 @@ def test_none():
 def test_invalid_locations():
     assert_raises(InvalidLocationError, Location, "foo")
     assert_raises(InvalidLocationError, Location, ["foo", "bar"])
+    assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat/blat", "foo"])
     assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat", "foo/bar"])
-    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces/revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces@revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name/revision")
 
 
 def test_equality():

common/lib/xmodule/xmodule/modulestore/xml.py

+import hashlib
 import json
 import logging
 import os
@@ -43,14 +44,76 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
 
         xmlstore: the XMLModuleStore to store the loaded modules in
         """
-        self.unnamed_modules = 0
-        self.used_slugs = set()
+        self.unnamed = defaultdict(int)     # category -> num of new url_names for that category
+        self.used_names = defaultdict(set)  # category -> set of used url_names
         self.org, self.course, self.url_name = course_id.split('/')
 
         def process_xml(xml):
             """Takes an xml string, and returns a XModuleDescriptor created from
             that xml.
             """
+
+            def make_name_unique(xml_data):
+                """
+                Make sure that the url_name of xml_data is unique.  If a previously loaded
+                unnamed descriptor stole this element's url_name, create a new one.
+
+                Removes 'slug' attribute if present, and adds or overwrites the 'url_name' attribute.
+                """
+                # VS[compat]. Take this out once course conversion is done (perhaps leave the uniqueness check)
+
+                attr = xml_data.attrib
+                tag = xml_data.tag
+                id = lambda x: x
+                # Things to try to get a name, in order  (key, cleaning function, remove key after reading?)
+                lookups = [('url_name', id, False),
+                           ('slug', id, True),
+                           ('name', Location.clean, False),
+                           ('display_name', Location.clean, False)]
+
+                url_name = None
+                for key, clean, remove in lookups:
+                    if key in attr:
+                        url_name = clean(attr[key])
+                        if remove:
+                            del attr[key]
+                        break
+
+                def fallback_name():
+                    """Return the fallback name for this module.  This is a function instead of a variable
+                    because we want it to be lazy."""
+                    # use the hash of the content--the first 12 bytes should be plenty.
+                    return tag + "_" + hashlib.sha1(xml).hexdigest()[:12]
+
+                # Fallback if there was nothing we could use:
+                if url_name is None or url_name == "":
+                    url_name = fallback_name()
+                    # Don't log a warning--we don't need this in the log.  Do
+                    # put it in the error tracker--content folks need to see it.
+                    need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
+
+                    if tag in need_uniq_names:
+                        error_tracker("ERROR: no name of any kind specified for {tag}.  Student "
+                                      "state won't work right.  Problem xml: '{xml}...'".format(tag=tag, xml=xml[:100]))
+                    else:
+                        # TODO (vshnayder): We may want to enable this once course repos are cleaned up.
+                        # (or we may want to give up on the requirement for non-state-relevant issues...)
+                        #error_tracker("WARNING: no name specified for module. xml='{0}...'".format(xml[:100]))
+                        pass
+
+                # Make sure everything is unique
+                if url_name in self.used_names[tag]:
+                    msg = ("Non-unique url_name in xml.  This may break content.  url_name={0}.  Content={1}"
+                                .format(url_name, xml[:100]))
+                    error_tracker("ERROR: " + msg)
+                    log.warning(msg)
+                    # Just set name to fallback_name--if there are multiple things with the same fallback name,
+                    # they are actually identical, so it's fragile, but not immediately broken.
+                    url_name = fallback_name()
+
+                self.used_names[tag].add(url_name)
+                xml_data.set('url_name', url_name)
+
             try:
                 # VS[compat]
                 # TODO (cpennington): Remove this once all fall 2012 courses
@@ -62,32 +125,11 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     err=str(err), xml=xml))
                 raise
 
-            # VS[compat]. Take this out once course conversion is done
-            if xml_data.get('slug') is None and xml_data.get('url_name') is None:
-                if xml_data.get('name'):
-                    slug = Location.clean(xml_data.get('name'))
-                elif xml_data.get('display_name'):
-                    slug = Location.clean(xml_data.get('display_name'))
-                else:
-                    self.unnamed_modules += 1
-                    slug = '{tag}_{count}'.format(tag=xml_data.tag,
-                                                  count=self.unnamed_modules)
-
-                while slug in self.used_slugs:
-                    self.unnamed_modules += 1
-                    slug = '{slug}_{count}'.format(slug=slug,
-                                                   count=self.unnamed_modules)
-
-                self.used_slugs.add(slug)
-                # log.debug('-> slug=%s' % slug)
-                xml_data.set('url_name', slug)
+            make_name_unique(xml_data)
 
             descriptor = XModuleDescriptor.load_from_xml(
                 etree.tostring(xml_data), self, self.org,
                 self.course, xmlstore.default_class)
-
-            #log.debug('==> importing descriptor location %s' %
-            #          repr(descriptor.location))
             descriptor.metadata['data_dir'] = course_dir
 
             xmlstore.modules[course_id][descriptor.location] = descriptor

common/lib/xmodule/xmodule/tests/__init__.py

@@ -19,6 +19,8 @@ import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
+from capa.xqueue_interface import dateformat
+from datetime import datetime
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
 from xmodule.graders import Score, aggregate_scores
@@ -35,8 +37,9 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
-    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'waittime': 10},
+    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
+    anonymous_student_id = 'student'
 )
 
 
@@ -282,71 +285,143 @@ class StringResponseWithHintTest(unittest.TestCase):
 class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
+    TODO: Add tests for external grader messages
     '''
-    def test_update_score(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
-
-        # CodeResponse requires internal CorrectMap state. Build it now in the 'queued' state
-        old_cmap = CorrectMap()
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        numAnswers = len(answer_ids)
-        for i in range(numAnswers):
-            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuekey=1000 + i))
-
-        # TODO: Message format inherited from ExternalResponse
-        #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
-        #incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
-
-        # New message format common to external graders
-        correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
-        incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
+    @staticmethod
+    def make_queuestate(key, time):
+        timestr = datetime.strftime(time, dateformat)
+        return {'key': key, 'time': timestr}
+
+    def test_is_queued(self):
+        ''' 
+        Simple test of whether LoncapaProblem knows when it's been queued
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+            
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+            test_lcp.correct_map.update(cmap)
+
+            self.assertEquals(test_lcp.is_queued(), False)
+
+            # Now we queue the LCP
+            cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids): 
+                queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
+
+            self.assertEquals(test_lcp.is_queued(), True)
 
-        xserver_msgs = {'correct': correct_score_msg,
-                        'incorrect': incorrect_score_msg,
-                        }
-
-        # Incorrect queuekey, state should not be updated
-        for correctness in ['correct', 'incorrect']:
-            test_lcp.correct_map = CorrectMap()
-            test_lcp.correct_map.update(old_cmap)  # Deep copy
-
-            test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
-            self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-            for i in range(numAnswers):
-                self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
-
-        # Correct queuekey, state should be updated
-        for correctness in ['correct', 'incorrect']:
-            for i in range(numAnswers):  # Target specific answer_id's
+    def test_update_score(self):
+        '''
+        Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            old_cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids):
+                queuekey = 1000 + i
+                queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
+                old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+
+            # Message format common to external graders
+            grader_msg = '<span>MESSAGE</span>' # Must be valid XML
+            correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg': grader_msg})
+            incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg': grader_msg})
+
+            xserver_msgs = {'correct': correct_score_msg,
+                            'incorrect': incorrect_score_msg,}
+
+            # Incorrect queuekey, state should not be updated
+            for correctness in ['correct', 'incorrect']:
                 test_lcp.correct_map = CorrectMap()
-                test_lcp.correct_map.update(old_cmap)
-
-                new_cmap = CorrectMap()
-                new_cmap.update(old_cmap)
-                npoints = 1 if correctness=='correct' else 0
-                new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuekey=None)
-
-                test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
-                self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
-
-                for j in range(numAnswers):
-                    if j == i:
-                        self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
-                    else:
-                        self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
-
-    def test_convert_files_to_filenames(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
-        fp = open(problem_file)
-        answers_with_file = {'1_2_1': 'String-based answer',
-                             '1_3_1': ['answer1', 'answer2', 'answer3'],
-                             '1_4_1': [fp, fp]}
-        answers_converted = convert_files_to_filenames(answers_with_file)
-        self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
-        self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
-        self.assertEquals(answers_converted['1_4_1'], [fp.name, fp.name])
+                test_lcp.correct_map.update(old_cmap)  # Deep copy
+
+                test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
+                self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
+
+                for answer_id in answer_ids:
+                    self.assertTrue(test_lcp.correct_map.is_queued(answer_id))  # Should be still queued, since message undelivered
+
+            # Correct queuekey, state should be updated
+            for correctness in ['correct', 'incorrect']:
+                for i, answer_id in enumerate(answer_ids):
+                    test_lcp.correct_map = CorrectMap()
+                    test_lcp.correct_map.update(old_cmap)
+
+                    new_cmap = CorrectMap()
+                    new_cmap.update(old_cmap)
+                    npoints = 1 if correctness=='correct' else 0
+                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg=grader_msg, queuestate=None)
+
+                    test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
+                    self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
+
+                    for j, test_id in enumerate(answer_ids):
+                        if j == i:
+                            self.assertFalse(test_lcp.correct_map.is_queued(test_id))  # Should be dequeued, message delivered
+                        else:
+                            self.assertTrue(test_lcp.correct_map.is_queued(test_id))  # Should be queued, message undelivered
+    
+
+    def test_recentmost_queuetime(self):
+        '''
+        Test whether the LoncapaProblem knows about the time of queue requests
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+            test_lcp.correct_map.update(cmap)
+            
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids):
+                queuekey = 1000 + i
+                latest_timestamp = datetime.now()
+                queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
+
+            # Queue state only tracks up to second
+            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp, dateformat), dateformat)
+
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
+
+        def test_convert_files_to_filenames(self):
+            '''
+            Test whether file objects are converted to filenames without altering other structures
+            '''
+            problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+            with open(problem_file) as fp:
+                answers_with_file = {'1_2_1': 'String-based answer',
+                                     '1_3_1': ['answer1', 'answer2', 'answer3'],
+                                     '1_4_1': [fp, fp]}
+                answers_converted = convert_files_to_filenames(answers_with_file)
+                self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
+                self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
+                self.assertEquals(answers_converted['1_4_1'], [fp.name, fp.name])
 
 
 class ChoiceResponseTest(unittest.TestCase):

common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml

@@ -9,91 +9,23 @@
 Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 
 <text>
-Write a program to compute the cube of a number
+Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 

common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml

+<problem>
+<text>
+<h2>Code response</h2>
+
+<p>
+</p>
+
+<text>
+Write a program to compute the square of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def square(n):
+"""
+
+answer = """
+def square(n):
+    return n**2    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testSquare(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: square(%d)'%n
+    return str(square(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testSquare(0))
+   elif test == 2: f.write(testSquare(1))
+   else:  f.write(testSquare())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+<text>
+Write a program to compute the cube of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def cube(n):
+"""
+
+answer = """
+def cube(n):
+    return n**3    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testCube(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: cube(%d)'%n
+    return str(cube(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testCube(0))
+   elif test == 2: f.write(testCube(1))
+   else:  f.write(testCube())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+</text>
+</problem>

common/lib/xmodule/xmodule/tests/test_files/js/compiled/javascriptresponse.js

+// Generated by CoffeeScript 1.3.3
+(function() {
+  var MinimaxProblemDisplay, root,
+    __hasProp = {}.hasOwnProperty,
+    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
+
+  MinimaxProblemDisplay = (function(_super) {
+
+    __extends(MinimaxProblemDisplay, _super);
+
+    function MinimaxProblemDisplay(state, submission, evaluation, container, submissionField, parameters) {
+      this.state = state;
+      this.submission = submission;
+      this.evaluation = evaluation;
+      this.container = container;
+      this.submissionField = submissionField;
+      this.parameters = parameters != null ? parameters : {};
+      MinimaxProblemDisplay.__super__.constructor.call(this, this.state, this.submission, this.evaluation, this.container, this.submissionField, this.parameters);
+    }
+
+    MinimaxProblemDisplay.prototype.render = function() {};
+
+    MinimaxProblemDisplay.prototype.createSubmission = function() {
+      var id, value, _ref, _results;
+      this.newSubmission = {};
+      if (this.submission != null) {
+        _ref = this.submission;
+        _results = [];
+        for (id in _ref) {
+          value = _ref[id];
+          _results.push(this.newSubmission[id] = value);
+        }
+        return _results;
+      }
+    };
+
+    MinimaxProblemDisplay.prototype.getCurrentSubmission = function() {
+      return this.newSubmission;
+    };
+
+    return MinimaxProblemDisplay;
+
+  })(XProblemDisplay);
+
+  root = typeof exports !== "undefined" && exports !== null ? exports : this;
+
+  root.TestProblemDisplay = TestProblemDisplay;
+
+}).call(this);
+;

common/lib/xmodule/xmodule/tests/test_import.py

@@ -255,3 +255,37 @@ class ImportTestCase(unittest.TestCase):
         two_toy_video =  modulestore.get_instance(two_toy_id, location)
         self.assertEqual(toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh8")
         self.assertEqual(two_toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh9")
+
+
+    def test_colon_in_url_name(self):
+        """Ensure that colons in url_names convert to file paths properly"""
+
+        print "Starting import"
+        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+        course_id = course.id
+
+        print "course errors:"
+        for (msg, err) in modulestore.get_item_errors(course.location):
+            print msg
+            print err
+
+        chapters = course.get_children()
+        self.assertEquals(len(chapters), 2)
+
+        ch2 = chapters[1]
+        self.assertEquals(ch2.url_name, "secret:magic")
+
+        print "Ch2 location: ", ch2.location
+
+        also_ch2 = modulestore.get_instance(course_id, ch2.location)
+        self.assertEquals(ch2, also_ch2)
+
+        print "making sure html loaded"
+        cloc = course.location
+        loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
+        html = modulestore.get_instance(course_id, loc)
+        self.assertEquals(html.display_name, "Toy lab")

common/lib/xmodule/xmodule/video_module.py

@@ -30,6 +30,7 @@ class VideoModule(XModule):
         xmltree = etree.fromstring(self.definition['data'])
         self.youtube = xmltree.get('youtube')
         self.position = 0
+        self.show_captions = xmltree.get('show_captions', 'true')
 
         if instance_state is not None:
             state = json.loads(instance_state)
@@ -75,6 +76,7 @@ class VideoModule(XModule):
             'display_name': self.display_name,
             # TODO (cpennington): This won't work when we move to data that isn't on the filesystem
             'data_dir': self.metadata['data_dir'],
+            'show_captions': self.show_captions
         })
 
 

common/lib/xmodule/xmodule/x_module.py

@@ -717,7 +717,8 @@ class ModuleSystem(object):
                  filestore=None,
                  debug=False,
                  xqueue=None,
-                 node_path=""):
+                 node_path="",
+                 anonymous_student_id=''):
         '''
         Create a closure around the system environment.
 
@@ -742,11 +743,16 @@ class ModuleSystem(object):
                          at settings.DATA_DIR.
 
         xqueue - Dict containing XqueueInterface object, as well as parameters
-                    for the specific StudentModule
+                    for the specific StudentModule:
+                    xqueue = {'interface': XQueueInterface object,
+                              'callback_url': Callback into the LMS,
+                              'queue_name': Target queuename in Xqueue}
 
         replace_urls - TEMPORARY - A function like static_replace.replace_urls
                          that capa_module can use to fix up the static urls in
                          ajax results.
+
+        anonymous_student_id - Used for tracking modules with student id
         '''
         self.ajax_url = ajax_url
         self.xqueue = xqueue
@@ -758,6 +764,7 @@ class ModuleSystem(object):
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
         self.node_path = node_path
+        self.anonymous_student_id = anonymous_student_id
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''

common/lib/xmodule/xmodule/xml_module.py

@@ -12,6 +12,12 @@ import sys
 
 log = logging.getLogger(__name__)
 
+def name_to_pathname(name):
+    """
+    Convert a location name for use in a path: replace ':' with '/'.
+    This allows users of the xml format to organize content into directories
+    """
+    return name.replace(':', '/')
 
 def is_pointer_tag(xml_obj):
     """
@@ -245,8 +251,8 @@ class XmlDescriptor(XModuleDescriptor):
         # VS[compat] -- detect new-style each-in-a-file mode
         if is_pointer_tag(xml_object):
             # new style:
-            # read the actual definition file--named using url_name
-            filepath = cls._format_filepath(xml_object.tag, url_name)
+            # read the actual definition file--named using url_name.replace(':','/')
+            filepath = cls._format_filepath(xml_object.tag, name_to_pathname(url_name))
             definition_xml = cls.load_file(filepath, system.resources_fs, location)
         else:
             definition_xml = xml_object	# this is just a pointer, not the real definition content
@@ -292,7 +298,8 @@ class XmlDescriptor(XModuleDescriptor):
         """If this returns True, write the definition of this descriptor to a separate
         file.
 
-        NOTE: Do not override this without a good reason.  It is here specifically for customtag...
+        NOTE: Do not override this without a good reason.  It is here
+        specifically for customtag...
         """
         return True
 
@@ -335,7 +342,8 @@ class XmlDescriptor(XModuleDescriptor):
 
         if self.export_to_file():
             # Write the definition to a file
-            filepath = self.__class__._format_filepath(self.category, self.url_name)
+            url_path = name_to_pathname(self.url_name)
+            filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
                 file.write(etree.tostring(xml_object, pretty_print=True))

common/test/data/toy/chapter/secret/magic.xml

+<chapter>
+    <video url_name="toyvideo" youtube="blahblah"/>
+</chapter>

common/test/data/toy/course/2012_Fall.xml

 <course>
   <chapter url_name="Overview">
     <videosequence url_name="Toy_Videos">
-      <html url_name="toylab"/>
+      <html url_name="secret:toylab"/>
       <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
     </videosequence>
     <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
   </chapter>
+  <chapter url_name="secret:magic"/>
 </course>

common/test/data/toy/policies/2012_Fall.json

@@ -11,7 +11,7 @@
         "display_name": "Toy Videos",
         "format": "Lecture Sequence"
     },
-    "html/toylab": {
+    "html/secret:toylab": {
         "display_name": "Toy lab"
     },
     "video/Video_Resources": {

create-dev-env.sh

@@ -105,7 +105,7 @@ NUMPY_VER="1.6.2"
 SCIPY_VER="0.10.1"
 BREW_FILE="$BASE/mitx/brew-formulas.txt"
 LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
-APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript graphviz"
+APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript graphviz libgraphviz-dev"
 
 if [[ $EUID -eq 0 ]]; then
     error "This script should not be run using sudo or as the root user"

doc/xml-format.md

-This doc is a rough spec of our xml format
+# edX xml format tutorial
 
-Every content element (within a course) should have a unique id.  This id is formed as {category}/{url_name}.  Categories are the different tag types ('chapter', 'problem', 'html', 'sequential', etc).  Url_name is a string containing a-z, A-Z, dot (.) and _.  This is what appears in urls that point to this object.
+## Goals of this document
 
-File layout:
+*	This was written assuming the reader has no prior programming/CS knowledge and has jumped cold turkey into the edX platform.
+*	To educate the reader on how to build and maintain the back end structure of the course content. This is important for debugging and standardization.
+*	After reading this, you should be able to add content to a course and make sure it shows up in the courseware and does not break the code.
+* __Prerequisites:__ it would be helpful to know a little bit about xml.  Here is a [simple example](http://www.ultraslavonic.info/intro-to-xml/) if you've never seen it before.
 
-- Xml files have content
-- "policy", which is also called metadata in various places, should live in a policy file.
+## Outline
 
-- each module (except customtag and course, which are special, see below) should live in a file, located at {category}/{url_name].xml
-To include this module in another one (e.g. to put a problem in a vertical), put in a "pointer tag":  <{category} url_name="{url_name}"/>.  When we read that, we'll load the actual contents.
+*	First, we will show a sample course structure as a case study/model of how xml and files in a course are organized to introductory understanding.
 
-Customtag is already a pointer, you can just use it in place: <customtag url_name="my_custom_tag" impl="blah" attr1="..."/>
+*	More technical details are below, including discussion of some special cases.
 
-Course tags:
-  - the top level course pointer tag lives in course.xml
-  - have 2 extra required attributes: "org" and "course" -- organization name, and course name.  Note that the course name is referring to the platonic ideal of this course, not to any particular run of this course.  The url_name should be particular run of this course.  E.g.
 
-If course.xml contains:
-<course org="HarvardX" course="cs50" url_name="2012"/>
+## Introduction
 
-we would load the actual course definition from course/2012.xml
+*	The course is organized hierarchically.  We start by describing course-wide parameters, then break the course into chapters, and then go deeper and deeper until we reach a specific pset, video, etc.
 
-To support multiple different runs of the course, you could have a different course.xml, containing
+*	You could make an analogy to finding a green shirt in your house - front door -> bedroom -> closet -> drawer -> shirts -> green shirt
 
-<course org="HarvardX" course="cs50" url_name="2012H"/>
 
-which would load the Harvard-internal version from course/2012H.xml
+## Case Study
 
-If there is only one run of the course for now, just have a single course.xml with the right url_name.
+Let's jump right in by looking at the directory structure of a very simple toy course:
 
-If there is more than one run of the course, the different course root pointer files should live in
-roots/url_name.xml, and course.xml should be a symbolic link to the one you want to run in your dev instance.
+    toy/
+        course
+        course.xml
+        problem
+        policies
+        roots
 
-If you want to run both versions, you need to checkout the repo twice, and have course.xml point to different root/{url_name}.xml files.
+The only top level file is `course.xml`, which should contain one line, looking something like this:
 
-Policies:
- - the policy for a course url_name lives in policies/{url_name}.json
+    <course org="edX" course="toy" url_name="2012_Fall"/>
 
-The format is called "json", and is best shown by example (though also feel free to google :)
+This gives all the information to uniquely identify a particular run of any course--which organization is producing the course, what the course name is, and what "run" this is, specified via the `url_name` attribute.
 
-the file is a dictionary (mapping from keys to values, syntax "{ key : value, key2 : value2, etc}"
+Obviously, this doesn't actually specify any of the course content, so we need to find that next.  To know where to look, you need to know the standard organizational structure of our system: _course elements are uniquely identified by the combination `(category, url_name)`_.  In this case, we are looking for a `course` element with the `url_name` "2012_Fall".  The definition of this element will be in `course/2012_Fall.xml`.  Let's look there next:
 
-Keys are in the form "{category}/{url_name}", which should uniquely id a content element.
-Values are dictionaries of the form {"metadata-key" : "metadata-value"}.
+`course/2012_Fall.xml`
+
+    <course>
+      <chapter url_name="Overview">
+        <videosequence url_name="Toy_Videos">
+          <problem url_name="warmup"/>
+          <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
+        </videosequence>
+        <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
+      </chapter>
+    </course>
+
+Aha.  Now we found some content.  We can see that the course is organized hierarchically, in this case with only one chapter, with `url_name` "Overview".   The chapter contains a `videosequence` and a `video`, with the sequence containing a problem and another video.  When viewed in the courseware, chapters are shown at the top level of the navigation accordion on the left, with any elements directly included in the chapter below.
+
+Looking at this file, we can see the course structure, and the youtube urls for the videos, but what about the "warmup" problem?  There is no problem content here!    Where should we look?  This is a good time to pause and try to answer that question based on our organizational structure above.
+
+As you hopefully guessed, the problem would be in `problem/warmup.xml`.  (Note: This tutorial doesn't discuss the xml format for problems--there are chapters of edx4edx that describe it.)  This is an instance of a _pointer tag:_ any xml tag with only the category and a url_name attribute will point to the file `{category}/{url_name}.xml`.  For example, this means that our toy `course.xml` could have also been written as
+
+`course/2012_Fall.xml`
+
+    <course>
+      <chapter url_name="Overview"/>
+    </course>
+
+with `chapter/Overview.xml` containing
+
+    <chapter>
+        <videosequence url_name="Toy_Videos">
+          <problem url_name="warmup"/>
+          <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
+        </videosequence>
+        <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
+    </chapter>
+
+In fact, this is the recommended structure for real courses--putting each chapter into its own file makes it easy to have different people work on each without conflicting or having to merge.  Similarly, as sequences get large, it can be handy to split them out as well (in `sequence/{url_name}.xml`, of course).
+
+Note that the `url_name` is only specified once per element--either the inline definition, or in the pointer tag.
+
+## Policy files
+
+We still haven't looked at two of the directoies in the top-level listing above: `policies` and `roots`.  Let's look at policies next.  The policy directory contains one file:
+
+    policies:
+        2012_Fall.json
+
+and that file is named {course-url_name}.json.  As you might expect, this file contains a policy for the course.  In our example, it looks like this:
+
+    2012_Fall.json:
+    {
+        "course/2012_Fall": {
+            "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
+            "start": "2015-07-17T12:00",
+            "display_name": "Toy Course"
+        },
+        "chapter/Overview": {
+            "display_name": "Overview"
+        },
+        "videosequence/Toy_Videos": {
+            "display_name": "Toy Videos",
+            "format": "Lecture Sequence"
+        },
+        "problem/warmup": {
+            "display_name": "Getting ready for the semester"
+        },
+        "video/Video_Resources": {
+            "display_name": "Video Resources"
+        },
+        "video/Welcome": {
+            "display_name": "Welcome"
+        }
+    }
+
+The policy specifies metadata about the content elements--things which are not inherent to the definition of the content, but which describe how the content is presented to the user and used in the course.  See below for a full list of metadata attributes; as the example shows, they include `display_name`, which is what is shown when this piece of content is referenced or shown in the courseware, and various dates and times, like `start`, which specifies when the content becomes visible to students, and various problem-specific parameters like the allowed number of attempts.  One important point is that some metadata is inherited: for example, specifying the start date on the course makes it the default for every element in the course.  See below for more details.
+
+It is possible to put metadata directly in the xml, as attributes of the appropriate tag, but using a policy file has two benefits: it puts all the policy in one place, making it easier to check that things like due dates are set properly, and it allows the content definitions to be easily used in another run of the same course, with the same or similar content, but different policy.
+
+## Roots
+
+The last directory in the top level listing is `roots`.  In our toy course, it contains a single file:
+
+    roots/
+        2012_Fall.xml
+
+This file is identical to the top-level `course.xml`, containing
+
+    <course org="edX" course="toy" url_name="2012_Fall"/>
+
+In fact, the top level `course.xml` is a symbolic link to this file.  When there is only one run of a course, the roots directory is not really necessary, and the top-level course.xml file can just specify the `url_name` of the course.  However, if we wanted to make a second run of our toy course, we could add another file called, e.g., `roots/2013_Spring.xml`, containing
+
+    <course org="edX" course="toy" url_name="2013_Spring"/>
+
+After creating `course/2013_Spring.xml` with the course structure (possibly as a symbolic link or copy of `course/2012_Fall.xml` if no content was changing), and `policies/2013_Spring.json`, we would have two different runs of the toy course in the course repository.  Our build system understands this roots structure, and will build a course package for each root.  (Dev note: if you're using a local development environment, make the top level `course.xml` point to the desired root, and check out the repo multiple times if you need multiple runs simultaneously).
+
+That's basically all there is to the organizational structure.  Read the next section for details on the tags we support, including some special case tags like `customtag` and `html` invariants, and look at the end for some tips that will make the editing process easier.
+
+----------
+
+# Tag types
 
-metadata can also live in the xml files, but anything defined in the policy file overrides anything in the xml.  This is primarily for backwards compatibility, and you should probably  not use both.  If you do leave some metadata tags in the xml, please be consistent (e.g. if display_names stay in xml, they should all stay in xml).
-   - note, some xml attributes are not metadata.  e.g. in <video youtube="xyz987293487293847"/>, the youtube attribute specifies what video this is, and is logically part of the content, not the policy, so it should stay in video/{url_name}.xml.
-
-Example policy file:
-{
-    "course/2012": {
-        "graceperiod": "1 day",
-        "start": "2012-10-15T12:00",
-        "display_name": "Introduction to Computer Science I",
-        "xqa_key": "z1y4vdYcy0izkoPeihtPClDxmbY1ogDK"
-    },
-    "chapter/Week_0": {
-        "display_name": "Week 0"
-    },
-    "sequential/Pre-Course_Survey": {
-        "display_name": "Pre-Course Survey",
-        "format": "Survey"
+* `abtest` -- Support for A/B testing.  TODO: add details..
+* `chapter` -- top level organization unit of a course.   The courseware display code currently expects the top level `course` element to contain only chapters, though there is no philosophical reason why this is required, so we may change it to properly display non-chapters at the top level.
+* `course` -- top level tag.  Contains everything else.
+* `customtag` -- render an html template, filling in some parameters, and return the resulting html.  See below for details.
+* `html` -- a reference to an html file.
+* `error`  -- don't put these in by hand :)   The internal representation of content that has an error, such as malformed xml or some broken invariant.  You may see this in the xml once the CMS is in use...
+* `problem` -- a problem.  See elsewhere in edx4edx for documentation on the format.
+* `problemset` -- logically, a series of related problems.  Currently displayed vertically.  May contain explanatory html, videos, etc.
+* `sequential` -- a sequence of content, currently displayed with a horizontal list of tabs.  If possible, use a more semantically meaningful tag (currently, we only have `videosequence`).
+* `vertical` -- a sequence of content, displayed vertically.  Content will be accessed all at once, on the right part of the page. No navigational bar. May have to use browser scroll bars. Content split with separators.   If possible, use a more semantically meaningful tag (currently, we only have `problemset`).
+* `video`  -- a link to a video, currently expected to be hosted on youtube.
+* `videosequence` -- a sequence of videos.  This can contain various non-video content; it just signals to the system that this is logically part of an explanatory sequence of content, as opposed to say an exam sequence.
+
+## Tag details
+
+### Container tags
+
+Container tags include `chapter`, `sequential`, `videosequence`, `vertical`, and `problemset`.  They are all specified in the same way in the xml, as shown in the tutorial above.
+
+### `course`
+
+`course` is also a container, and is similar, with one extra wrinkle: the top level pointer tag _must_ have  `org` and `course` attributes specified--the organization name, and course name.  Note that `course` is referring to the platonic ideal of this course (e.g. "6.002x"), not to any particular run of this course.  The `url_name` should be the particular run of this course.
+
+### `customtag`
+
+When we see `<customtag impl="special" animal="unicorn" hat="blue"/>`, we will:
+
+* look for a file called `custom_tags/special`  in your course dir.
+* render it as a mako template, passing parameters {'animal':'unicorn', 'hat':'blue'}, generating html.  (Google `mako` for template syntax, or look at existing examples).
+
+Since `customtag` is already a pointer, there is generally no need to put it into a separate file--just use it in place: <customtag url_name="my_custom_tag" impl="blah" attr1="..."/>
+
+
+### `html`
+
+Most of our content is in xml, but some html content may not be proper xml (all tags matched, single top-level tag, etc), since browsers are fairly lenient in what they'll display.  So, there are two ways to include html content:
+
+* If your html content is in a proper xml format, just put it in `html/{url_name}.xml`.
+* If your html content is not in proper xml format, you can put it in `html/{filename}.html`, and put `<html filename={filename} />` in `html/{filename}.xml`.  This allows another level of indirection, and makes sure that we can read the xml file and then just return the actual html content without trying to parse it.
+
+### `video`
+
+Videos have an attribute youtube, which specifies a series of speeds + youtube videos id:
+
+    <video youtube="0.75:1yk1A8-FPbw,1.0:vNMrbPvwhU4,1.25:gBW_wqe7rDc,1.50:7AE_TKgaBwA" url_name="S15V14_Response_to_impulse_limit_case"/>
+
+This video has been encoded at 4 different speeds: 0.75x, 1x, 1.25x, and 1.5x.
+
+## More on `url_name`s
+
+Every content element (within a course) should have a unique id.  This id is formed as `{category}/{url_name}`, or automatically generated from the content if `url_name` is not specified.  Categories are the different tag types ('chapter', 'problem', 'html', 'sequential', etc).  Url_name is a string containing a-z, A-Z, dot (.), underscore (_), and ':'.  This is what appears in urls that point to this object.
+
+Colon (':') is special--when looking for the content definition in an xml, ':' will be replaced with '/'.  This allows organizing content into folders.  For example, given the pointer tag
+
+    <problem url_name="conceptual:add_apples_and_oranges"/>
+
+we would look for the problem definition in `problem/conceptual/add_apples_and_oranges.xml`.   (There is a technical reason why we can't just allow '/' in the url_name directly.)
+
+__IMPORTANT__: A student's state for a particular content element is tied to the element id, so the automatic id generation if only ok for elements that do not need to store any student state (e.g. verticals or customtags).  For problems, sequentials, and videos, and any other element where we keep track of what the student has done and where they are at, you should specify a unique `url_name`.  Of course, any content element that is split out into a file will need a `url_name` to specify where to find the definition.  When the CMS comes online, it will use these ids to enable content reuse, so if there is a logical name for something, please do specify it.
+
+-----
+
+## Policy files
+
+*	A policy file is useful when running different versions of a course e.g. internal, external, fall, spring, etc. as you can change due dates, etc, by creating multiple policy files.
+*	A policy file provides information on the metadata of the course--things that are not inherent to the definitions of the contents, but that may vary from run to run.
+* Note: We will be expanding our understanding and format for metadata in the not-too-distant future, but for now it is simply a set of key-value pairs.
+
+### Policy file location
+* The policy for a course run `some_url_name` lives in `policies/some_url_name.json`
+
+### Policy file contents
+* The file format is "json", and is best shown by example, as in the tutorial above (though also feel free to google :)
+* The expected contents are a dictionary mapping from keys to values (syntax "{ key : value, key2 : value2, etc}")
+* Keys are in the form "{category}/{url_name}", which should uniquely identify a content element.
+Values are dictionaries of the form {"metadata-key" : "metadata-value"}.
+* The order in which things appear does not matter, though it may be helpful to organize the file in the same order as things appear in the content.
+* NOTE: json is picky about commas.  If you have trailing commas before closing braces, it will complain and refuse to parse the file.  This can be irritating at first.
+
+### Available metadata
+
+__Not inherited:__
+
+* `display_name` - name that will appear when this content is displayed in the courseware.  Useful for all tag types.
+*	`format` - subheading under display name -- currently only displayed for chapter sub-sections.  Also used by the the grader to know how to process students assessments that the
+    section contains. New formats can be defined as a 'type' in the GRADER variable in course_settings.json. Optional.  (TODO: double check this--what's the current behavior?)
+* `hide_from_toc` -- If set to true for a chapter or chapter subsection, will hide that element from the courseware navigation accordion.  This is useful if you'd like to link to the content directly instead (e.g. for tutorials)
+* `ispublic` -- specify whether the course is public.  You should be able to use start dates instead (?)
+
+__Inherited:__
+
+* `start` -- when this content should be shown to students.  Note that anyone with staff access to the course will always see everything.
+*	`showanswer` - When to show answer. For 'attempted', will show answer after first attempt. Values: never, attempted, answered, closed. Default: closed. Optional.
+*	`graded` - Whether this section will count towards the students grade. "true" or "false". Defaults to "false".
+*	`rerandomise` - Randomize question on each attempt. Values: 'always' (students see a different version of the problem after each attempt to solve it)
+                                                            'never' (all students see the same version of the problem)
+                                                            'per_student' (individual students see the same version of the problem each time the look at it, but that version is different from what other students see)
+                                                            Default: 'always'. Optional.
+*	`due` - Due date for assignment. Assignment will be closed after that.  Values: valid date. Default: none. Optional.
+* attempts: Number of allowed attempts. Values: integer. Default: infinite. Optional.
+* `graceperiod` - A default length of time that the problem is still accessible after the due date in the format "2 days 3 hours" or "1 day 15 minutes".  Note, graceperiods are currently the easiest way to handle time zones. Due dates are all expressed in UCT.
+* `xqa_key` -- for integration with Ike's content QA server. -- should typically be specified at the course level.
+
+__Inheritance example:__
+
+This is a sketch ("tue" is not a valid start date), that should help illustrate how metadata inheritance works.
+
+    <course start="tue">
+      <chap1> -- start tue
+        <problem>   --- start tue
+      </chap1>
+      <chap2 start="wed">  -- start wed
+       <problem2 start="thu">  -- start thu
+       <problem3>      -- start wed
+      </chap2>
+    </course>
+
+
+## Specifying metadata in the xml file
+
+Metadata can also live in the xml files, but anything defined in the policy file overrides anything in the xml.  This is primarily for backwards compatibility, and you should probably  not use both.  If you do leave some metadata tags in the xml, you should be consistent (e.g. if `display_name`s stay in xml, they should all stay in xml).
+   - note, some xml attributes are not metadata.  e.g. in `<video youtube="xyz987293487293847"/>`, the `youtube` attribute specifies what video this is, and is logically part of the content, not the policy, so it should stay in the xml.
+
+Another example policy file:
+
+    {
+        "course/2012": {
+            "graceperiod": "1 day",
+            "start": "2012-10-15T12:00",
+            "display_name": "Introduction to Computer Science I",
+            "xqa_key": "z1y4vdYcy0izkoPeihtPClDxmbY1ogDK"
+        },
+        "chapter/Week_0": {
+            "display_name": "Week 0"
+        },
+        "sequential/Pre-Course_Survey": {
+            "display_name": "Pre-Course Survey",
+            "format": "Survey"
+        }
     }
-}
 
-NOTE: json is picky about commas.  If you have trailing commas before closing braces, it will complain and refuse to parse the file.  This is irritating.
 
 
-Valid tag categories:
+## Deprecated formats
 
-abtest
-chapter
-course
-customtag
-html
-error  -- don't put these in by hand :)
-problem
-problemset
-sequential
-vertical
-video
-videosequence
+If you look at some older xml, you may see some tags or metadata attributes that aren't listed above.  They are deprecated, and should not be used in new content.  We include them here so that you can understand how old-format content works.
 
-Obsolete tags:
-Use customtag instead:
-  videodev
-  book
-  slides
-  image
-  discuss
+### Obsolete tags:
 
-Ex: instead of <book page="12"/>, use <customtag impl="book" page="12"/>
+* `section` : this used to be necessary within chapters.  Now, you can just use any standard tag inside a chapter, so use the container tag that makes the most sense for grouping content--e.g. `problemset`, `videosequence`, and just include content directly if it belongs inside a chapter (e.g. `html`, `video`, `problem`)
 
-Use something semantic instead, as makes sense: sequential, vertical, videosequence if it's actually a sequence.  If the section would only contain a single element, just include that element directly.
-  section
+* There used to be special purpose tags that all basically did the same thing, and have been subsumed by `customtag`.  The list is `videodev, book, slides, image, discuss`.  Use `customtag` in new content.  (e.g. instead of `<book page="12"/>`, use `<customtag impl="book" page="12"/>`)
 
-In general, prefer the most "semantic" name for containers: e.g. use problemset rather than vertical for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
+### Obsolete attributes
 
-How customtags work:
- When we see <customtag impl="special" animal="unicorn" hat="blue"/>, we will:
+* `slug` -- old term for `url_name`.  Use `url_name`
+* `name` -- we didn't originally have a distinction between `url_name` and `display_name` -- this made content element ids fragile, so please use `url_name` as a stable unique identifier for the content, and `display_name` as the particular string you'd like to display for it.
 
- - look for a file called custom_tags/special  in your course dir.
- - render it as a mako template, passing parameters {'animal':'unicorn', 'hat':'blue'}, generating html.
 
+# Static links
 
-METADATA
+if your content links (e.g. in an html file)  to `"static/blah/ponies.jpg"`, we will look for this in `YOUR_COURSE_DIR/blah/ponies.jpg`.  Note that this is not looking in a `static/` subfolder in your course dir.  This may (should?) change at some point.   Links that include `/course` will be rewritten to the root of your course in the courseware (e.g. `courses/{org}/{course}/{url_name}/` in the current url structure).  This is useful for linking to the course wiki, for example.
 
-Metadata that we generally understand:
-Only on course tag in courses/url_name.xml
-  ispublic
-  xqa_key  -- set only on course, inherited to everything else
+# Tips for content developers
 
-Everything:
-  display_name
-  format   (maybe only content containers, e.g. "Lecture sequence", "problem set", "lab", etc. )
-  start  -- modules will not show up to non-course-staff users before the start date (in production)
-  hide_from_toc  -- if this is true, don't show in table of contents for the course.  Useful on chapters, and chapter subsections that are linked to from somewhere else.
+* We will be making better tools for managing policy files soon.  In the meantime, you can add dummy definitions to make it easier to search and separate the file visually.  For example, you could add:
 
-Used for problems
-graceperiod
-showanswer
-rerandomize
-graded
-due
+    "WEEK 1" : "##################################################",
 
+before the week 1 material to make it easy to find in the file.
 
-These are _inherited_ : if specified on the course, will apply to everything in the course, except for things that explicitly specify them, and their children.
-        'graded', 'start', 'due', 'graceperiod', 'showanswer', 'rerandomize',
-        # TODO (ichuang): used for Fall 2012 xqa server access
-        'xqa_key',
+* Come up with a consistent pattern for url_names, so that it's easy to know where to look for any piece of content.  It will also help to come up with a standard way of splitting your content files.  As a point of departure, we suggest splitting chapters, sequences, html, and problems into separate files.
 
-Example sketch:
-<course start="tue">
-  <chap1> -- start tue
-    <problem>   --- start tue
-  </chap1>
-  <chap2 start="wed">  -- start wed
-   <problem2 start="thu">  -- start thu
-   <problem3>      -- start wed
-  </chap2>
-</course>
+* A heads up: our content management system will allow you to develop content through a web browser, but will be backed by this same xml at first.  Once that happens, every element will be in its own file to make access and updates faster.
 
+* Prefer the most "semantic" name for containers: e.g., use problemset rather than vertical for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
 
-STATIC LINKS:
+----
 
-if your content links (e.g. in an html file)  to "static/blah/ponies.jpg", we will look for this in YOUR_COURSE_DIR/blah/ponies.jpg.  Note that this is not looking in a static/ subfolder in your course dir.  This may (should?) change at some point.
+(Dev note: This file is generated from the mitx repo, in `doc/xml-format.md`.  Please make edits there.)

lms/djangoapps/course_wiki/editors.py

@@ -4,9 +4,12 @@ from django.utils.encoding import force_unicode
 from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
+from django.template.loader import render_to_string
+
 from wiki.editors.base import BaseEditor
 from wiki.editors.markitup import MarkItUpAdminWidget
 
+
 class CodeMirrorWidget(forms.Widget):
     def __init__(self, attrs=None):
         # The 'rows' and 'cols' attributes are required for HTML correctness.
@@ -18,9 +21,15 @@ class CodeMirrorWidget(forms.Widget):
     
     def render(self, name, value, attrs=None):
         if value is None: value = ''
+
         final_attrs = self.build_attrs(attrs, name=name)
-        return mark_safe(u'<div><textarea%s>%s</textarea></div>' % (flatatt(final_attrs),
-                conditional_escape(force_unicode(value))))
+
+        # TODO use the help_text field of edit form instead of rendering a template
+
+        return render_to_string('wiki/includes/editor_widget.html',
+                                {'attrs': mark_safe(flatatt(final_attrs)),
+                                 'content': conditional_escape(force_unicode(value)),
+                                 })
 
 
 class CodeMirror(BaseEditor):
@@ -50,5 +59,6 @@ class CodeMirror(BaseEditor):
               "js/vendor/CodeMirror/xml.js",
               "js/vendor/CodeMirror/mitx_markdown.js",
               "js/wiki/CodeMirror.init.js",
+              "js/wiki/cheatsheet.js",
               )
 

lms/djangoapps/courseware/grades.py

@@ -17,6 +17,7 @@ log = logging.getLogger("mitx.courseware")
 
 def yield_module_descendents(module):
     stack = module.get_display_items()
+    stack.reverse()
 
     while len(stack) > 0:
         next_module = stack.pop()

lms/djangoapps/courseware/module_render.py

+import hashlib
 import json
 import logging
 import sys
@@ -143,8 +144,9 @@ def get_module(user, request, location, student_module_cache, course_id, positio
     exists.
 
     Arguments:
-      - user                  : current django User
-      - request               : current django HTTPrequest
+      - user                  : User for whom we're getting the module
+      - request               : current django HTTPrequest.  Note: request.user isn't used for anything--all auth
+                                and such works based on user.
       - location              : A Location-like object identifying the module to load
       - student_module_cache  : a StudentModuleCache
       - course_id             : the course_id in the context of which to load module
@@ -173,7 +175,13 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     if not has_access(user, descriptor, 'load'):
         return None
 
-    #TODO Only check the cache if this module can possibly have state
+    # Anonymized student identifier
+    h = hashlib.md5()
+    h.update(settings.SECRET_KEY)
+    h.update(str(user.id))
+    anonymous_student_id = h.hexdigest()
+
+    # Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
     if user.is_authenticated():
@@ -197,6 +205,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                                    location=descriptor.location.url(),
                                    dispatch=''),
                        )
+    # Intended use is as {ajax_url}/{dispatch_command}, so get rid of the trailing slash.
+    ajax_url = ajax_url.rstrip('/')
 
     # Fully qualified callback URL for external queueing system
     xqueue_callback_url = '{proto}://{host}'.format(
@@ -217,7 +227,9 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
 
     xqueue = {'interface': xqueue_interface,
               'callback_url': xqueue_callback_url,
-              'default_queuename': xqueue_default_queuename.replace(' ', '_')}
+              'default_queuename': xqueue_default_queuename.replace(' ', '_'),
+              'waittime': settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+             }
 
     def inner_get_module(location):
         """
@@ -241,7 +253,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
-                          node_path=settings.NODE_PATH
+                          node_path=settings.NODE_PATH,
+                          anonymous_student_id=anonymous_student_id
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
@@ -409,6 +422,10 @@ def modx_dispatch(request, dispatch, location, course_id):
     '''
     # ''' (fix emacs broken parsing)
 
+    # Check parameters and fail fast if there's a problem
+    if not Location.is_valid(location):
+        raise Http404("Invalid location")
+
     # Check for submitted files and basic file size checks
     p = request.POST.copy()
     if request.FILES:

lms/djangoapps/courseware/tests/tests.py

@@ -411,8 +411,6 @@ class TestViewAuth(PageLoader):
             """list of urls that only instructors/staff should be able to see"""
             urls = reverse_urls(['instructor_dashboard','gradebook','grade_summary'],
                                 course)
-            urls.append(reverse('student_progress', kwargs={'course_id': course.id,
-                                                     'student_id': user(self.student).id}))
             return urls
 
         def check_non_staff(course):
@@ -435,6 +433,17 @@ class TestViewAuth(PageLoader):
                 print 'checking for 200 on {0}'.format(url)
                 self.check_for_get_code(200, url)
 
+            # The student progress tab is not accessible to a student
+            # before launch, so the instructor view-as-student feature should return a 404 as well.
+            # TODO (vshnayder): If this is not the behavior we want, will need
+            # to make access checking smarter and understand both the effective
+            # user (the student), and the requesting user (the prof)
+            url = reverse('student_progress', kwargs={'course_id': course.id,
+                                                     'student_id': user(self.student).id})
+            print 'checking for 404 on view-as-student: {0}'.format(url)
+            self.check_for_get_code(404, url)
+
+
         # First, try with an enrolled student
         print '=== Testing student access....'
         self.login(self.student, self.password)

lms/djangoapps/courseware/views.py

@@ -325,14 +325,21 @@ def progress(request, course_id, student_id=None):
             raise Http404
         student = User.objects.get(id=int(student_id))
 
+    # NOTE: To make sure impersonation by instructor works, use
+    # student instead of request.user in the rest of the function.
+
     student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
-        course_id, request.user, course)
-    course_module = get_module(request.user, request, course.location,
+        course_id, student, course)
+    course_module = get_module(student, request, course.location,
                                student_module_cache, course_id)
 
+    # The course_module should be accessible, but check anyway just in case something went wrong:
+    if course_module is None:
+        raise Http404("Course does not exist")
+
     courseware_summary = grades.progress_summary(student, course_module,
                                                  course.grader, student_module_cache)
-    grade_summary = grades.grade(request.user, request, course, student_module_cache)
+    grade_summary = grades.grade(student, request, course, student_module_cache)
 
     context = {'course': course,
                'courseware_summary': courseware_summary,

lms/djangoapps/dashboard/views.py

@@ -21,11 +21,17 @@ def dashboard(request):
     if not request.user.is_staff:
         raise Http404
 
-    query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc"
-
+    queries=[]
+    queries.append("select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc;")
+    queries.append("select count(distinct user_id) as unique_students from student_courseenrollment;")
+    queries.append("select registrations, count(registrations) from (select count(user_id) as registrations from student_courseenrollment group by user_id) as registrations_per_user group by registrations;")
+    
     from django.db import connection
     cursor = connection.cursor()
-    cursor.execute(query)
-    results = dictfetchall(cursor)
+    results =[]
+
+    for query in queries:
+        cursor.execute(query)
+        results.append(dictfetchall(cursor))
 
     return HttpResponse(json.dumps(results, indent=4))

lms/djangoapps/lms_migration/management/commands/manage_course_groups.py

+#!/usr/bin/python
+#
+# File:   manage_course_groups
+#
+# interactively list and edit membership in course staff and instructor groups
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+
+#-----------------------------------------------------------------------------
+# get all staff groups
+
+class Command(BaseCommand):
+    help = "Manage course group membership, interactively."
+
+    def handle(self, *args, **options):
+
+        gset = Group.objects.all()
+        
+        print "Groups:"
+        for cnt,g in zip(range(len(gset)), gset):
+            print "%d. %s" % (cnt,g)
+        
+        gnum = int(raw_input('Choose group to manage (enter #): '))
+        
+        group = gset[gnum]
+        
+        #-----------------------------------------------------------------------------
+        # users in group
+        
+        uall = User.objects.all()
+        if uall.count()<50:
+            print "----"
+            print "List of All Users: %s" % [str(x.username) for x in uall]
+            print "----"
+        else:
+            print "----"
+            print "There are %d users, which is too many to list" % uall.count()
+            print "----"
+        
+        while True:
+        
+            print "Users in the group:"
+            
+            uset = group.user_set.all()
+            for cnt, u in zip(range(len(uset)), uset):
+                print "%d. %s" % (cnt, u)
+        
+            action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ')
+        
+            m = re.match('^[0-9]+$',action)
+            if m:
+                unum = int(action)
+                u = uset[unum]
+                print "Deleting user %s" % u
+                u.groups.remove(group)
+            
+            else:
+                for uname in action.split(','):
+                    try:
+                        user = User.objects.get(username=action)
+                    except Exception as err:
+                        print "Error %s" % err
+                        continue
+                    print "adding %s to group %s" % (user, group)
+                    user.groups.add(group)
+                
+            
+            

lms/djangoapps/lms_migration/migrate.py

@@ -139,7 +139,7 @@ def gitreload(request, reload_dir=None):
 
     ALLOWED_IPS = []	# allow none by default
     if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
-        ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
+        ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS
 
     if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
         if request.user and request.user.is_staff:

lms/envs/common.py

@@ -86,6 +86,9 @@ DEFAULT_GROUPS = []
 # If this is true, random scores will be generated for the purpose of debugging the profile graphs
 GENERATE_PROFILE_SCORES = False
 
+# Used with XQueue
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
+
 ############################# SET PATH INFORMATION #############################
 PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /mitx/lms
 REPO_ROOT = PROJECT_ROOT.dirname()

lms/envs/dev.py

@@ -15,7 +15,7 @@ TEMPLATE_DEBUG = True
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
-MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = True
+MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
 
 WIKI_ENABLED = True
@@ -78,10 +78,10 @@ COURSE_LISTINGS = {
                 'MITx/3.091x/2012_Fall',
                 'MITx/6.002x/2012_Fall',
                 'MITx/6.00x/2012_Fall'],
-    'berkeley': ['BerkeleyX/CS169.1x/Cal_2012_Fall',
-                 'BerkeleyX/CS188.1x/Cal_2012_Fall'],
+    'berkeley': ['BerkeleyX/CS169/fa12',
+                 'BerkeleyX/CS188/fa12'],
     'harvard': ['HarvardX/CS50x/2012H'],
-    'mit': [],
+    'mit': ['MITx/3.091/MIT_2012_Fall'],
     'sjsu': ['MITx/6.002x-EE98/2012_Fall_SJSU'],
 }
 
@@ -107,7 +107,7 @@ LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 MITX_FEATURES['AUTH_USE_OPENID'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
 
-INSTALLED_APPS += ('external_auth',) 
+INSTALLED_APPS += ('external_auth',)
 INSTALLED_APPS += ('django_openid_auth',)
 
 OPENID_CREATE_USERS = False

lms/envs/test.py

@@ -58,7 +58,7 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
-
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # TODO (cpennington): We need to figure out how envs/test.py can inject things
 # into common.py so that we don't have to repeat this sort of thing

lms/static/coffee/src/discussion/content.coffee

@@ -124,7 +124,7 @@ if Backbone?
         url = @model.urlFor('retrieve')
         DiscussionUtil.safeAjax
           $elem: $elem
-          $loading: $(event.target) if event
+          $loading: @$(".discussion-show-comments")
           type: "GET"
           url: url
           success: (response, textStatus) =>

lms/static/coffee/src/discussion/discussion.coffee

@@ -169,7 +169,7 @@ if Backbone?
       url = URI($elem.attr("action")).addSearch({text: @$(".search-input").val()})
       @reload($elem, url)
 
-    sort: ->
+    sort: (event) ->
       $elem = $(event.target)
       url = $elem.attr("sort-url")
       @reload($elem, url)

lms/static/coffee/src/discussion/main.coffee

@@ -3,9 +3,6 @@ $ ->
   window.$$contents = {}
   window.$$discussions = {}
 
-  $(".discussion-module").each (index, elem) ->
-    view = new DiscussionModuleView(el: elem)
-
   $("section.discussion").each (index, elem) ->
     discussionData = DiscussionUtil.getDiscussionData($(elem).attr("_id"))
     discussion = new Discussion()

lms/static/coffee/src/discussion/utils.coffee

 $ ->
   $.fn.extend
     loading: ->
-      $(this).after("<span class='discussion-loading'></span>")
+      @$_loading = $("<span class='discussion-loading'></span>")
+      $(this).after(@$_loading)
     loaded: ->
-      $(this).parent().children(".discussion-loading").remove()
+      @$_loading.remove()
 
 class @DiscussionUtil
 

lms/static/js/wiki/cheatsheet.js

+$(document).ready(function () {
+    $('#cheatsheetLink').click(function() {
+        $('#cheatsheetModal').modal('show');
+    });
+    
+    $('#cheatsheetModal .close-btn').click(function(e) {
+        $('#cheatsheetModal').modal('hide');
+    });
+});
\ No newline at end of file

lms/static/sass/course/courseware/courses/_cs188.scss

@@ -27,6 +27,12 @@ body.cs188 {
             margin-bottom: 1.416em;
         }
 
+        .choicegroup {
+            input[type=checkbox], input[type=radio] {
+                margin-right: 5px;
+            }
+        }
+
     }
 
 }

lms/static/sass/course/wiki/_wiki.scss

@@ -391,6 +391,18 @@ section.wiki {
     line-height: 1.4em;
   }
 
+  #div_id_content {
+    position: relative;
+  }
+
+  #hint_id_content {
+    position: absolute;
+    top: 10px;
+    right: 0%;
+    font-size: 12px;
+    text-align:right;
+  }
+
   .CodeMirror {
     background: #fafafa;
     border: 1px solid #c8c8c8;
@@ -567,11 +579,73 @@ section.wiki {
     background: #f00 !important;
   }
 
-  
+  #cheatsheetLink {
+    text-align: right;
+    display: float;
+  }
+
+  #cheatsheetModal {
+    width: 950px;
+    margin-left: -450px;
+    margin-top: -100px;
+
+    .left-column {
+      margin-right: 10px;
+    }
+
+    .left-column,
+    .right-column {
+      float: left;
+      width: 450px;
+    }
+
+    .close-btn {
+      display: block;
+      position: absolute;
+      top: -8px;
+      right: -8px;
+      width: 30px;
+      height: 30px;
+      border-radius: 30px;
+      border: 1px solid #ccc;
+      @include linear-gradient(top, #eee, #d2d2d2);
+      font-size: 22px;
+      line-height: 28px;    
+      color: #333;
+      text-align: center;
+      @include box-shadow(0 1px 0 #fff inset, 0 1px 2px rgba(0, 0, 0, .2));
+    }
+  }
+
+  #cheatsheet-body {
+    background: #fff;
+    text-align: left;
+    padding: 20px;
+    font-size: 14px;
+    @include clearfix;
 
+    h3 {
+      font-weight: bold;
+    }
 
+    ul {
+      list-style: circle;
+      line-height: 1.4;
+      color: #333;
+    }
+  }
 
+  #cheatsheet-body section + section {
+    margin-top: 40px;
+  }
 
+  #cheatsheet-body pre{
+    color: #000;
+    text-align: left;
+    background: #eee;
+    padding: 10px;
+    font-size: 12px;
+  }
 
   /*-----------------
 

lms/templates/discussion/mustache/_content.mustache

@@ -7,11 +7,11 @@
     </div>
     <div class="discussion-right-wrapper">
       <ul class="admin-actions">
-        <li><a href="javascript:void(0)" class="admin-endorse">Endorse</a></li>
-        <li><a href="javascript:void(0)" class="admin-edit">Edit</a></li>
-        <li><a href="javascript:void(0)" class="admin-delete">Delete</a></li>
+        <li style="display: none;"><a href="javascript:void(0)" class="admin-endorse">Endorse</a></li>
+        <li style="display: none;"><a href="javascript:void(0)" class="admin-edit">Edit</a></li>
+        <li style="display: none;"><a href="javascript:void(0)" class="admin-delete">Delete</a></li>
         {{#thread}}
-          <li><a href="javascript:void(0)" class="admin-openclose">{{close_thread_text}}</a></li>
+          <li style="display: none;"><a href="javascript:void(0)" class="admin-openclose">{{close_thread_text}}</a></li>
         {{/thread}}
       </ul>
       {{#thread}}

lms/templates/video.html

-% if name is not UNDEFINED and name is not None:
-  <h1> ${display_name} </h1>
+% if display_name is not UNDEFINED and display_name is not None:
+  <h2> ${display_name} </h2>
 % endif
 
-<div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}">
+<div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}" data-show-captions="${show_captions}">
   <div class="tc-wrapper">
     <article class="video-wrapper">
       <section class="video-player">

lms/templates/wiki/base.html

@@ -27,21 +27,20 @@
       });
     }
   </script>
-  
-  
+
+
   {% addtoblock 'js' %}
     {% comment %} These scripts load at the bottom of the body {% endcomment %}
-  
+
     <script src="{% static 'js/bootstrap-alert.js' %}"></script>
     <script src="{% static 'js/bootstrap-collapse.js' %}"></script>
     <script src="{% static 'js/bootstrap-modal.js' %}"></script>
-    
+
     {% with mathjax_mode='wiki' %}
       {% include "mathjax_include.html" %}
     {% endwith %}
-    
   {% endaddtoblock %}
-  
+
 {% endblock %}
 
 
@@ -64,11 +63,12 @@
             </div>
           {% endfor %}
         {% endif %}
-        
+
         {% block wiki_contents %}{% endblock %}
-        
+
       {% endblock %}
     </div>
+
   </section>
 
 {% endblock %}

lms/templates/wiki/create.html

@@ -42,6 +42,7 @@
             {% trans "Go back" %}
           </a>
       </div>  
+      {% include "wiki/includes/cheatsheet.html" %}
     </form>
 
   </article>

lms/templates/wiki/edit.html

@@ -40,7 +40,10 @@
       </a>
     </div>
   </div>
+  {% include "wiki/includes/cheatsheet.html" %}
 </form>
 
 {% endblock %}
 
+
+

lms/templates/wiki/includes/cheatsheet.html

+<div class="modal hide fade" id="cheatsheetModal">
+	<a href="#" class="close-btn">×</a>
+	<div id="cheatsheet-body" class="modal-body">
+		<div class="left-column">
+			<section>
+				<h2>Wiki Syntax Help</h2>
+				<p>This wiki uses <strong>Markdown</strong> for styling. There are several useful guides online. See any of the links below for in-depth details:</p>
+				<ul>
+					<li><a href="http://daringfireball.net/projects/markdown/basics" target="_blank">Markdown: Basics</a></li>
+					<li><a href="http://greg.vario.us/doc/markdown.txt" target="_blank">Quick Markdown Syntax Guide</a></li>
+					<li><a href="http://www.lowendtalk.com/discussion/6/miniature-markdown-guide" target="_blank">Miniature Markdown Guide</a></li>
+				</ul>
+				<p>To create a new wiki article, create a link to it. Clicking the link gives you the creation page.</p>
+				<pre>[Article Name](wiki:ArticleName)</pre>
+			</section>
+			<section>
+				<h3>edX Additions:</h3>
+				<pre>
+circuit-schematic:</pre>
+				<pre>
+$LaTeX Math Expression$</pre>
+			</section>
+		</div>
+
+		<div class="right-column">
+			<section>
+				<h3>Useful examples:</h3>
+				<pre>
+http://wikipedia.org
+[Wikipedia](http://wikipedia.org)
+[edX Wiki](wiki:/edx/)</pre>
+				<pre>
+Huge Header
+===========</pre>
+				<pre>
+Smaller Header
+--------------</pre>
+				<pre>
+*emphasis* or _emphasis_</pre>
+				<pre>
+**strong** or __strong__</pre>
+				<pre>
+- Unordered List
+	- Sub Item 1
+	- Sub Item 2</pre>
+				<pre>
+1. Ordered
+2. List</pre>
+				<pre>
+> Quotes</pre>
+			</section>
+		</div>
+
+		</div>
+	</div>
+</div>

lms/templates/wiki/includes/editor_widget.html

+<textarea {{ attrs }}>{{ content }}</textarea>
+<p id="hint_id_content" class="help-block">
+  Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.
+</p>

lms/wsgi.py

+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws")
+
+# This application object is used by the development server
+# as well as any WSGI server configured to use this file.
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+from django.conf import settings
+from xmodule.modulestore.django import modulestore
+
+for store_name in settings.MODULESTORE:
+    modulestore(store_name)

repo-requirements.txt

 -e git://github.com/MITx/django-staticfiles.git@6d2504e5c8#egg=django-staticfiles
 -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline
--e git://github.com/benjaoming/django-wiki.git@cd1c23e1#egg=django-wiki
+-e git://github.com/MITx/django-wiki.git@e2e84558#egg=django-wiki
 -e git://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
 -e common/lib/capa
 -e common/lib/xmodule

requirements.txt

@@ -46,4 +46,5 @@ django-sekizai<0.7
 django-mptt>=0.5.3
 sorl-thumbnail
 networkx
+pygraphviz
 -r repo-requirements.txt