Merge branch 'master' into kimth/check-grader-html

cms/djangoapps/github_sync/tests/__init__.py

@@ -61,7 +61,7 @@ class GithubSyncTestCase(TestCase):
         self.assertIn(
             Location('i4x://edX/toy/chapter/Overview'),
             [child.location for child in self.import_course.get_children()])
-        self.assertEquals(1, len(self.import_course.get_children()))
+        self.assertEquals(2, len(self.import_course.get_children()))
 
     @patch('github_sync.sync_with_github')
     def test_sync_all_with_github(self, sync_with_github):

common/lib/capa/capa/capa_problem.py

@@ -14,6 +14,8 @@ This is used by capa_module.
 
 from __future__ import division
 
+from datetime import datetime
+import json
 import logging
 import math
 import numpy
@@ -32,6 +34,7 @@ from correctmap import CorrectMap
 import eia
 import inputtypes
 from util import contextualize_text, convert_files_to_filenames
+import xqueue_interface
 
 # to be replaced with auto-registering
 import responsetypes
@@ -202,11 +205,24 @@ class LoncapaProblem(object):
         '''
         Returns True if any part of the problem has been submitted to an external queue
         '''
-        queued = False
-        for answer_id in self.correct_map:
-            if self.correct_map.is_queued(answer_id):
-                queued = True
-        return queued
+        return any(self.correct_map.is_queued(answer_id) for answer_id in self.correct_map)
+
+
+    def get_recentmost_queuetime(self):
+        '''
+        Returns a DateTime object that represents the timestamp of the most recent queueing request, or None if not queued
+        '''
+        if not self.is_queued():
+            return None
+
+        # Get a list of timestamps of all queueing requests, then convert it to a DateTime object
+        queuetime_strs = [self.correct_map.get_queuetime_str(answer_id)
+                          for answer_id in self.correct_map 
+                          if self.correct_map.is_queued(answer_id)]
+        queuetimes = [datetime.strptime(qt_str, xqueue_interface.dateformat) for qt_str in queuetime_strs]
+
+        return max(queuetimes)
+
 
     def grade_answers(self, answers):
         '''

common/lib/capa/capa/correctmap.py

@@ -15,7 +15,8 @@ class CorrectMap(object):
     - msg         : string (may have HTML) giving extra message response (displayed below textline or textbox)
     - hint        : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg)
     - hintmode    : one of (None,'on_request','always') criteria for displaying hint
-    - queuekey    : a random integer for xqueue_callback verification
+    - queuestate  : Dict {key:'', time:''} where key is a secret string, and time is a string dump
+                    of a DateTime object in the format '%Y%m%d%H%M%S'. Is None when not queued
 
     Behaves as a dict.
     '''
@@ -31,14 +32,15 @@ class CorrectMap(object):
     def __iter__(self):
         return self.cmap.__iter__()
 
-    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuekey=None):
+    # See the documentation for 'set_dict' for the use of kwargs
+    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None, **kwargs):
         if answer_id is not None:
             self.cmap[answer_id] = {'correctness': correctness,
                                     'npoints': npoints,
                                     'msg': msg,
                                     'hint': hint,
                                     'hintmode': hintmode,
-                                    'queuekey': queuekey,
+                                    'queuestate': queuestate,
                                     }
 
     def __repr__(self):
@@ -52,25 +54,39 @@ class CorrectMap(object):
 
     def set_dict(self, correct_map):
         '''
-        set internal dict to provided correct_map dict
-        for graceful migration, if correct_map is a one-level dict, then convert it to the new
-        dict of dicts format.
+        Set internal dict of CorrectMap to provided correct_map dict
+
+        correct_map is saved by LMS as a plaintext JSON dump of the correctmap dict. This means that
+        when the definition of CorrectMap (e.g. its properties) are altered, existing correct_map dict
+        not coincide with the newest CorrectMap format as defined by self.set.
+
+        For graceful migration, feed the contents of each correct map to self.set, rather than
+        making a direct copy of the given correct_map dict. This way, the common keys between 
+        the incoming correct_map dict and the new CorrectMap instance will be written, while
+        mismatched keys will be gracefully ignored.
+
+        Special migration case:
+            If correct_map is a one-level dict, then convert it to the new dict of dicts format.
         '''
         if correct_map and not (type(correct_map[correct_map.keys()[0]]) == dict):
             self.__init__()	                                  # empty current dict
             for k in correct_map: self.set(k, correct_map[k]) # create new dict entries
         else:
-            self.cmap = correct_map
+            self.__init__()
+            for k in correct_map: self.set(k, **correct_map[k])
 
     def is_correct(self, answer_id):
         if answer_id in self.cmap: return self.cmap[answer_id]['correctness'] == 'correct'
         return None
 
     def is_queued(self, answer_id):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] is not None
+        return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
 
     def is_right_queuekey(self, answer_id, test_key):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] == test_key
+        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
+
+    def get_queuetime_str(self, answer_id):
+        return self.cmap[answer_id]['queuestate']['time']
 
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')

common/lib/capa/capa/inputtypes.py

@@ -351,7 +351,7 @@ def filesubmission(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     context = { 'id': eid, 'state': status, 'msg': msg, 'value': value,
             'queue_len': queue_len, 'allowed_files': allowed_files,
@@ -384,7 +384,7 @@ def textbox(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     # For CodeMirror
     mode = element.get('mode','python')

common/lib/capa/capa/responsetypes.py

@@ -26,6 +26,7 @@ import xml.sax.saxutils as saxutils
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
+from datetime import datetime
 from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
@@ -435,7 +436,11 @@ class JavascriptResponse(LoncapaResponse):
         (all_correct, evaluation, solution) = self.run_grader(json_submission)
         self.solution = solution
         correctness = 'correct' if all_correct else 'incorrect'
-        return CorrectMap(self.answer_id, correctness, msg=evaluation)
+        if all_correct:
+            points = self.get_max_score()
+        else:
+            points = 0
+        return CorrectMap(self.answer_id, correctness, npoints=points, msg=evaluation)
     
     def run_grader(self, submission):
         if submission is None or submission == '':
@@ -1005,7 +1010,7 @@ class CodeResponse(LoncapaResponse):
     '''
     Grade student code using an external queueing server, called 'xqueue'
 
-    Expects 'xqueue' dict in ModuleSystem with the following keys:
+    Expects 'xqueue' dict in ModuleSystem with the following keys that are needed by CodeResponse:
         system.xqueue = { 'interface': XqueueInterface object,
                           'callback_url': Per-StudentModule callback URL where results are posted (string),
                           'default_queuename': Default queuename to submit request (string)
@@ -1026,7 +1031,7 @@ class CodeResponse(LoncapaResponse):
         TODO: Determines whether in synchronous or asynchronous (queued) mode
         '''
         xml = self.xml
-        self.url = xml.get('url', None) # XML can override external resource (grader/queue) URL
+        self.url = xml.get('url', None) # TODO: XML can override external resource (grader/queue) URL
         self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
 
         # VS[compat]:
@@ -1121,22 +1126,34 @@ class CodeResponse(LoncapaResponse):
 
         # Prepare xqueue request
         #------------------------------------------------------------ 
+
         qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
+
+        anonymous_student_id = self.system.anonymous_student_id
 
         # Generate header
-        queuekey = xqueue_interface.make_hashkey(str(self.system.seed)+self.answer_id)
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
+                                                 anonymous_student_id +  
+                                                 self.answer_id)
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
                                                 lms_key=queuekey,
                                                 queue_name=self.queue_name)
         
         # Generate body
         if is_list_of_files(submission):
-            self.context.update({'submission': queuekey}) # For tracking. TODO: May want to record something else here
+            self.context.update({'submission': ''}) # TODO: Get S3 pointer from the Queue
         else:
             self.context.update({'submission': submission})
 
         contents = self.payload.copy() 
 
+        # Metadata related to the student submission revealed to the external grader
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                        'submission_time': qtime,
+                       }
+        contents.update({'student_info': json.dumps(student_info)})
+
         # Submit request. When successful, 'msg' is the prior length of the queue
         if is_list_of_files(submission):
             contents.update({'student_response': ''}) # TODO: Is there any information we want to send here?
@@ -1148,16 +1165,21 @@ class CodeResponse(LoncapaResponse):
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents))
 
+        # State associated with the queueing request
+        queuestate = {'key': queuekey,
+                      'time': qtime,
+                     }
+
         cmap = CorrectMap() 
         if error:
-            cmap.set(self.answer_id, queuekey=None,
+            cmap.set(self.answer_id, queuestate=None,
                      msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg)
         else:
             # Queueing mechanism flags:
-            #   1) Backend: Non-null CorrectMap['queuekey'] indicates that the problem has been queued
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that the problem has been queued
             #   2) Frontend: correctness='incomplete' eventually trickles down through inputtypes.textbox 
             #       and .filesubmission to inform the browser to poll the LMS
-            cmap.set(self.answer_id, queuekey=queuekey, correctness='incomplete', msg=msg)
+            cmap.set(self.answer_id, queuestate=queuestate, correctness='incomplete', msg=msg)
 
         return cmap
 
@@ -1180,7 +1202,7 @@ class CodeResponse(LoncapaResponse):
                 points = 0
             elif points > self.maxpoints[self.answer_id]:
                 points = self.maxpoints[self.answer_id]
-            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace(' ', ' '), queuekey=None)  # Queuekey is consumed
+            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace(' ', ' '), queuestate=None)  # Queuestate is consumed
         else:
             log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id))
 
@@ -1197,7 +1219,7 @@ class CodeResponse(LoncapaResponse):
         '''
          Grader reply is a JSON-dump of the following dict
            { 'correct': True/False,
-             'score': # TODO -- Partial grading
+             'score': Numeric value (floating point is okay) to assign to answer
              'msg': grader_msg }
 
         Returns (valid_score_msg, correct, score, msg):

common/lib/capa/capa/templates/filesubmission.html

@@ -10,7 +10,7 @@
     <span class="processing" id="status_${id}"></span>
     <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
-    <span class="debug">(${state})</span>
+    <span style="display:none;" class="debug">(${state})</span>
     <br/>
     <span class="message">${msg|n}</span>
     <br/>

common/lib/capa/capa/templates/textbox.html

@@ -21,7 +21,7 @@
   <div style="display:none;" name="${hidden}" inputid="input_${id}" />
   % endif
   <br/>
-  <span class="debug">(${state})</span>
+  <span style="display:none;" class="debug">(${state})</span>
   <br/>
   <span class="message">${msg|n}</span>
   <br/>

common/lib/capa/capa/xqueue_interface.py

@@ -5,20 +5,17 @@ import hashlib
 import json
 import logging
 import requests
-import time
 
 
 log = logging.getLogger('mitx.' + __name__)
+dateformat = '%Y%m%d%H%M%S'
 
-
-def make_hashkey(seed=None):
+def make_hashkey(seed):
     '''
     Generate a string key by hashing 
     '''
     h = hashlib.md5()
-    if seed is not None:
     h.update(str(seed))
-    h.update(str(time.time()))
     return h.hexdigest()
 
 

common/lib/xmodule/xmodule/capa_module.py

@@ -462,6 +462,15 @@ class CapaModule(XModule):
             self.system.track_function('save_problem_check_fail', event_info)
             raise NotFoundError('Problem must be reset before it can be checked again')
 
+        # Problem queued. Students must wait a specified waittime before they are allowed to submit
+        if self.lcp.is_queued():
+            current_time = datetime.datetime.now()
+            prev_submit_time = self.lcp.get_recentmost_queuetime() 
+            waittime_between_requests = self.system.xqueue['waittime']
+            if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
+                msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
+                return {'success': msg, 'html': ''} # Prompts a modal dialog in ajax callback 
+
         try:
             old_state = self.lcp.get_state()
             lcp_id = self.lcp.problem_id

common/lib/xmodule/xmodule/html_module.py

@@ -4,9 +4,10 @@ import logging
 import os
 import sys
 from lxml import etree
+from path import path
 
 from .x_module import XModule
-from .xml_module import XmlDescriptor
+from .xml_module import XmlDescriptor, name_to_pathname
 from .editing_module import EditingDescriptor
 from .stringify import stringify_children
 from .html_checker import check_html
@@ -75,9 +76,19 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             cls.clean_metadata_from_xml(definition_xml)
             return {'data': stringify_children(definition_xml)}
         else:
-            # html is special.  cls.filename_extension is 'xml', but if 'filename' is in the definition,
-            # that means to load from .html
-            filepath = "{category}/{name}.html".format(category='html', name=filename)
+            # html is special.  cls.filename_extension is 'xml', but
+            # if 'filename' is in the definition, that means to load
+            # from .html
+            # 'filename' in html pointers is a relative path
+            # (not same as 'html/blah.html' when the pointer is in a directory itself)
+            pointer_path = "{category}/{url_path}".format(category='html',
+                                                  url_path=name_to_pathname(location.name))
+            base = path(pointer_path).dirname()
+            #log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
+            filepath = "{base}/{name}.html".format(base=base, name=filename)
+            #log.debug("looking for html file for {0} at {1}".format(location, filepath))
+
+
 
             # VS[compat]
             # TODO (cpennington): If the file doesn't exist at the right path,
@@ -128,13 +139,18 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             pass
 
         # Not proper format.  Write html to file, return an empty tag
-        filepath = u'{category}/{name}.html'.format(category=self.category,
-                                                    name=self.url_name)
+        pathname = name_to_pathname(self.url_name)
+        pathdir = path(pathname).dirname()
+        filepath = u'{category}/{pathname}.html'.format(category=self.category,
+                                                    pathname=pathname)
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
             file.write(self.definition['data'])
 
+        # write out the relative name
+        relname = path(pathname).basename()
+
         elt = etree.Element('html')
-        elt.set("filename", self.url_name)
+        elt.set("filename", relname)
         return elt

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -13,18 +13,21 @@ from xmodule.errortracker import ErrorLog, make_error_tracker
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
+
 URL_RE = re.compile("""
     (?P<tag>[^:]+)://
     (?P<org>[^/]+)/
     (?P<course>[^/]+)/
     (?P<category>[^/]+)/
-    (?P<name>[^/]+)
-    (/(?P<revision>[^/]+))?
+    (?P<name>[^@]+)
+    (@(?P<revision>[^/]+))?
     """, re.VERBOSE)
 
 # TODO (cpennington): We should decide whether we want to expand the
 # list of valid characters in a location
 INVALID_CHARS = re.compile(r"[^\w.-]")
+# Names are allowed to have colons.
+INVALID_CHARS_NAME = re.compile(r"[^\w.:-]")
 
 _LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
 
@@ -34,7 +37,7 @@ class Location(_LocationBase):
     Encodes a location.
 
     Locations representations of URLs of the
-    form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+    form {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
     However, they can also be represented a dictionaries (specifying each component),
     tuples or list (specified in order), or as strings of the url
@@ -81,7 +84,7 @@ class Location(_LocationBase):
 
         location - Can be any of the following types:
             string: should be of the form
-                    {tag}://{org}/{course}/{category}/{name}[/{revision}]
+                    {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
             list: should be of the form [tag, org, course, category, name, revision]
 
@@ -99,10 +102,11 @@ class Location(_LocationBase):
         ommitted.
 
         Components must be composed of alphanumeric characters, or the
-        characters '_', '-', and '.'
+        characters '_', '-', and '.'.  The name component is additionally allowed to have ':',
+        which is interpreted specially for xml storage.
 
-        Components may be set to None, which may be interpreted by some contexts
-        to mean wildcard selection
+        Components may be set to None, which may be interpreted in some contexts
+        to mean wildcard selection.
         """
 
 
@@ -116,14 +120,23 @@ class Location(_LocationBase):
             return _LocationBase.__new__(_cls, *([None] * 6))
 
         def check_dict(dict_):
-            check_list(dict_.itervalues())
+            # Order matters, so flatten out into a list
+            keys = ['tag', 'org', 'course', 'category', 'name', 'revision']
+            list_ = [dict_[k] for k in keys]
+            check_list(list_)
 
         def check_list(list_):
-            for val in list_:
-                if val is not None and INVALID_CHARS.search(val) is not None:
+            def check(val, regexp):
+                if val is not None and regexp.search(val) is not None:
                     log.debug('invalid characters val="%s", list_="%s"' % (val, list_))
                     raise InvalidLocationError(location)
 
+            list_ = list(list_)
+            for val in list_[:4] + [list_[5]]:
+                check(val, INVALID_CHARS)
+            # names allow colons
+            check(list_[4], INVALID_CHARS_NAME)
+
         if isinstance(location, basestring):
             match = URL_RE.match(location)
             if match is None:
@@ -162,7 +175,7 @@ class Location(_LocationBase):
         """
         url = "{tag}://{org}/{course}/{category}/{name}".format(**self.dict())
         if self.revision:
-            url += "/" + self.revision
+            url += "@" + self.revision
         return url
 
     def html_id(self):
@@ -170,6 +183,7 @@ class Location(_LocationBase):
         Return a string with a version of the location that is safe for use in
         html id attributes
         """
+        # TODO: is ':' ok in html ids?
         return "-".join(str(v) for v in self.list()
                         if v is not None).replace('.', '_')
 

common/lib/xmodule/xmodule/modulestore/django.py

@@ -5,8 +5,8 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
 """
 
 from __future__ import absolute_import
-
 from importlib import import_module
+from os import environ
 
 from django.conf import settings
 
@@ -43,3 +43,8 @@ def modulestore(name='default'):
         )
 
     return _MODULESTORES[name]
+
+# if 'DJANGO_SETTINGS_MODULE' in environ:
+#     # Initialize the modulestores immediately
+#     for store_name in settings.MODULESTORE:
+#         modulestore(store_name)

common/lib/xmodule/xmodule/modulestore/tests/test_location.py

@@ -10,7 +10,7 @@ def check_string_roundtrip(url):
 
 def test_string_roundtrip():
     check_string_roundtrip("tag://org/course/category/name")
-    check_string_roundtrip("tag://org/course/category/name/revision")
+    check_string_roundtrip("tag://org/course/category/name@revision")
 
 
 input_dict = {
@@ -21,18 +21,28 @@ input_dict = {
     'org': 'org'
 }
 
+
+also_valid_dict = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name:more_name',
+    'org': 'org'
+}
+
+
 input_list = ['tag', 'org', 'course', 'category', 'name']
 
 input_str = "tag://org/course/category/name"
-input_str_rev = "tag://org/course/category/name/revision"
+input_str_rev = "tag://org/course/category/name@revision"
 
-valid = (input_list, input_dict, input_str, input_str_rev)
+valid = (input_list, input_dict, input_str, input_str_rev, also_valid_dict)
 
 invalid_dict = {
     'tag': 'tag',
     'course': 'course',
     'category': 'category',
-    'name': 'name/more_name',
+    'name': 'name@more_name',
     'org': 'org'
 }
 
@@ -45,8 +55,9 @@ invalid_dict2 = {
 }
 
 invalid = ("foo", ["foo"], ["foo", "bar"],
-           ["foo", "bar", "baz", "blat", "foo/bar"],
-           "tag://org/course/category/name with spaces/revision",
+           ["foo", "bar", "baz", "blat:blat", "foo:bar"],  # ':' ok in name, not in category
+           "tag://org/course/category/name with spaces@revision",
+           "tag://org/course/category/name/with/slashes@revision",
            invalid_dict,
            invalid_dict2)
 
@@ -62,16 +73,15 @@ def test_dict():
     assert_equals(dict(revision=None, **input_dict), Location(input_dict).dict())
 
     input_dict['revision'] = 'revision'
-    assert_equals("tag://org/course/category/name/revision", Location(input_dict).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_dict).url())
     assert_equals(input_dict, Location(input_dict).dict())
 
-
 def test_list():
     assert_equals("tag://org/course/category/name", Location(input_list).url())
     assert_equals(input_list + [None], Location(input_list).list())
 
     input_list.append('revision')
-    assert_equals("tag://org/course/category/name/revision", Location(input_list).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_list).url())
     assert_equals(input_list, Location(input_list).list())
 
 
@@ -87,8 +97,10 @@ def test_none():
 def test_invalid_locations():
     assert_raises(InvalidLocationError, Location, "foo")
     assert_raises(InvalidLocationError, Location, ["foo", "bar"])
+    assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat/blat", "foo"])
     assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat", "foo/bar"])
-    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces/revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces@revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name/revision")
 
 
 def test_equality():

common/lib/xmodule/xmodule/modulestore/xml.py

+import hashlib
 import json
 import logging
 import os
@@ -43,14 +44,76 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
 
         xmlstore: the XMLModuleStore to store the loaded modules in
         """
-        self.unnamed_modules = 0
-        self.used_slugs = set()
+        self.unnamed = defaultdict(int)     # category -> num of new url_names for that category
+        self.used_names = defaultdict(set)  # category -> set of used url_names
         self.org, self.course, self.url_name = course_id.split('/')
 
         def process_xml(xml):
             """Takes an xml string, and returns a XModuleDescriptor created from
             that xml.
             """
+
+            def make_name_unique(xml_data):
+                """
+                Make sure that the url_name of xml_data is unique.  If a previously loaded
+                unnamed descriptor stole this element's url_name, create a new one.
+
+                Removes 'slug' attribute if present, and adds or overwrites the 'url_name' attribute.
+                """
+                # VS[compat]. Take this out once course conversion is done (perhaps leave the uniqueness check)
+
+                attr = xml_data.attrib
+                tag = xml_data.tag
+                id = lambda x: x
+                # Things to try to get a name, in order  (key, cleaning function, remove key after reading?)
+                lookups = [('url_name', id, False),
+                           ('slug', id, True),
+                           ('name', Location.clean, False),
+                           ('display_name', Location.clean, False)]
+
+                url_name = None
+                for key, clean, remove in lookups:
+                    if key in attr:
+                        url_name = clean(attr[key])
+                        if remove:
+                            del attr[key]
+                        break
+
+                def fallback_name():
+                    """Return the fallback name for this module.  This is a function instead of a variable
+                    because we want it to be lazy."""
+                    # use the hash of the content--the first 12 bytes should be plenty.
+                    return tag + "_" + hashlib.sha1(xml).hexdigest()[:12]
+
+                # Fallback if there was nothing we could use:
+                if url_name is None or url_name == "":
+                    url_name = fallback_name()
+                    # Don't log a warning--we don't need this in the log.  Do
+                    # put it in the error tracker--content folks need to see it.
+                    need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
+
+                    if tag in need_uniq_names:
+                        error_tracker("ERROR: no name of any kind specified for {tag}.  Student "
+                                      "state won't work right.  Problem xml: '{xml}...'".format(tag=tag, xml=xml[:100]))
+                    else:
+                        # TODO (vshnayder): We may want to enable this once course repos are cleaned up.
+                        # (or we may want to give up on the requirement for non-state-relevant issues...)
+                        #error_tracker("WARNING: no name specified for module. xml='{0}...'".format(xml[:100]))
+                        pass
+
+                # Make sure everything is unique
+                if url_name in self.used_names[tag]:
+                    msg = ("Non-unique url_name in xml.  This may break content.  url_name={0}.  Content={1}"
+                                .format(url_name, xml[:100]))
+                    error_tracker("ERROR: " + msg)
+                    log.warning(msg)
+                    # Just set name to fallback_name--if there are multiple things with the same fallback name,
+                    # they are actually identical, so it's fragile, but not immediately broken.
+                    url_name = fallback_name()
+
+                self.used_names[tag].add(url_name)
+                xml_data.set('url_name', url_name)
+
             try:
                 # VS[compat]
                 # TODO (cpennington): Remove this once all fall 2012 courses
@@ -62,32 +125,11 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     err=str(err), xml=xml))
                 raise
 
-            # VS[compat]. Take this out once course conversion is done
-            if xml_data.get('slug') is None and xml_data.get('url_name') is None:
-                if xml_data.get('name'):
-                    slug = Location.clean(xml_data.get('name'))
-                elif xml_data.get('display_name'):
-                    slug = Location.clean(xml_data.get('display_name'))
-                else:
-                    self.unnamed_modules += 1
-                    slug = '{tag}_{count}'.format(tag=xml_data.tag,
-                                                  count=self.unnamed_modules)
-
-                while slug in self.used_slugs:
-                    self.unnamed_modules += 1
-                    slug = '{slug}_{count}'.format(slug=slug,
-                                                   count=self.unnamed_modules)
-
-                self.used_slugs.add(slug)
-                # log.debug('-> slug=%s' % slug)
-                xml_data.set('url_name', slug)
+            make_name_unique(xml_data)
 
             descriptor = XModuleDescriptor.load_from_xml(
                 etree.tostring(xml_data), self, self.org,
                 self.course, xmlstore.default_class)
-
-            #log.debug('==> importing descriptor location %s' %
-            #          repr(descriptor.location))
             descriptor.metadata['data_dir'] = course_dir
 
             xmlstore.modules[course_id][descriptor.location] = descriptor

common/lib/xmodule/xmodule/tests/__init__.py

@@ -19,6 +19,8 @@ import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
+from capa.xqueue_interface import dateformat
+from datetime import datetime
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
 from xmodule.graders import Score, aggregate_scores
@@ -35,8 +37,9 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
-    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'waittime': 10},
+    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
+    anonymous_student_id = 'student'
 )
 
 
@@ -283,28 +286,61 @@ class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
     '''
+    @staticmethod
+    def make_queuestate(key, time):
+        timestr = datetime.strftime(time, dateformat)
+        return {'key': key, 'time': timestr}
+
+    def test_is_queued(self):
+        ''' 
+        Simple test of whether LoncapaProblem knows when it's been queued
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+            
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+            test_lcp.correct_map.update(cmap)
+
+            self.assertEquals(test_lcp.is_queued(), False)
+
+            # Now we queue the LCP
+            cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids): 
+                queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
+
+            self.assertEquals(test_lcp.is_queued(), True)
+
     def test_update_score(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+        '''
+        Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the 'queued' state
-        old_cmap = CorrectMap()
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        numAnswers = len(answer_ids)
-        for i in range(numAnswers):
-            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuekey=1000 + i))
+            answer_ids = sorted(test_lcp.get_question_answers())
 
-        # TODO: Message format inherited from ExternalResponse
-        #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
-        #incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            old_cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids):
+                queuekey = 1000 + i
+                queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
+                old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
-        # New message format common to external graders
+            # Message format common to external graders
             correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
             incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
 
             xserver_msgs = {'correct': correct_score_msg,
-                        'incorrect': incorrect_score_msg,
-                        }
+                            'incorrect': incorrect_score_msg,}
 
             # Incorrect queuekey, state should not be updated
             for correctness in ['correct', 'incorrect']:
@@ -314,32 +350,68 @@ class CodeResponseTest(unittest.TestCase):
                 test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
                 self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-            for i in range(numAnswers):
-                self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
+                for answer_id in answer_ids:
+                    self.assertTrue(test_lcp.correct_map.is_queued(answer_id))  # Should be still queued, since message undelivered
 
             # Correct queuekey, state should be updated
             for correctness in ['correct', 'incorrect']:
-            for i in range(numAnswers):  # Target specific answer_id's
+                for i, answer_id in enumerate(answer_ids):
                     test_lcp.correct_map = CorrectMap()
                     test_lcp.correct_map.update(old_cmap)
 
                     new_cmap = CorrectMap()
                     new_cmap.update(old_cmap)
                     npoints = 1 if correctness=='correct' else 0
-                new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuekey=None)
+                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
 
                     test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                     self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
 
-                for j in range(numAnswers):
+                    for j, test_id in enumerate(answer_ids):
                         if j == i:
-                        self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
+                            self.assertFalse(test_lcp.correct_map.is_queued(test_id))  # Should be dequeued, message delivered
                         else:
-                        self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
+                            self.assertTrue(test_lcp.correct_map.is_queued(test_id))  # Should be queued, message undelivered
+    
+
+    def test_recentmost_queuetime(self):
+        '''
+        Test whether the LoncapaProblem knows about the time of queue requests
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+
+            answer_ids = sorted(test_lcp.get_question_answers())
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+            test_lcp.correct_map.update(cmap)
+            
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            cmap = CorrectMap()
+            for i, answer_id in enumerate(answer_ids):
+                queuekey = 1000 + i
+                latest_timestamp = datetime.now()
+                queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
+
+            # Queue state only tracks up to second
+            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp, dateformat), dateformat)
+
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
 
         def test_convert_files_to_filenames(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
-        fp = open(problem_file)
+            '''
+            Test whether file objects are converted to filenames without altering other structures
+            '''
+            problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+            with open(problem_file) as fp:
                 answers_with_file = {'1_2_1': 'String-based answer',
                                      '1_3_1': ['answer1', 'answer2', 'answer3'],
                                      '1_4_1': [fp, fp]}

common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml

@@ -9,91 +9,23 @@
 Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 
 <text>
-Write a program to compute the cube of a number
+Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 

common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml

+<problem>
+<text>
+<h2>Code response</h2>
+
+<p>
+</p>
+
+<text>
+Write a program to compute the square of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def square(n):
+"""
+
+answer = """
+def square(n):
+    return n**2    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testSquare(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: square(%d)'%n
+    return str(square(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testSquare(0))
+   elif test == 2: f.write(testSquare(1))
+   else:  f.write(testSquare())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+<text>
+Write a program to compute the cube of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def cube(n):
+"""
+
+answer = """
+def cube(n):
+    return n**3    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testCube(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: cube(%d)'%n
+    return str(cube(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testCube(0))
+   elif test == 2: f.write(testCube(1))
+   else:  f.write(testCube())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+</text>
+</problem>

common/lib/xmodule/xmodule/tests/test_import.py

@@ -255,3 +255,37 @@ class ImportTestCase(unittest.TestCase):
         two_toy_video =  modulestore.get_instance(two_toy_id, location)
         self.assertEqual(toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh8")
         self.assertEqual(two_toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh9")
+
+
+    def test_colon_in_url_name(self):
+        """Ensure that colons in url_names convert to file paths properly"""
+
+        print "Starting import"
+        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+        course_id = course.id
+
+        print "course errors:"
+        for (msg, err) in modulestore.get_item_errors(course.location):
+            print msg
+            print err
+
+        chapters = course.get_children()
+        self.assertEquals(len(chapters), 2)
+
+        ch2 = chapters[1]
+        self.assertEquals(ch2.url_name, "secret:magic")
+
+        print "Ch2 location: ", ch2.location
+
+        also_ch2 = modulestore.get_instance(course_id, ch2.location)
+        self.assertEquals(ch2, also_ch2)
+
+        print "making sure html loaded"
+        cloc = course.location
+        loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
+        html = modulestore.get_instance(course_id, loc)
+        self.assertEquals(html.display_name, "Toy lab")

common/lib/xmodule/xmodule/x_module.py

@@ -717,7 +717,8 @@ class ModuleSystem(object):
                  filestore=None,
                  debug=False,
                  xqueue=None,
-                 node_path=""):
+                 node_path="",
+                 anonymous_student_id=''):
         '''
         Create a closure around the system environment.
 
@@ -742,11 +743,16 @@ class ModuleSystem(object):
                          at settings.DATA_DIR.
 
         xqueue - Dict containing XqueueInterface object, as well as parameters
-                    for the specific StudentModule
+                    for the specific StudentModule:
+                    xqueue = {'interface': XQueueInterface object,
+                              'callback_url': Callback into the LMS,
+                              'queue_name': Target queuename in Xqueue}
 
         replace_urls - TEMPORARY - A function like static_replace.replace_urls
                          that capa_module can use to fix up the static urls in
                          ajax results.
+
+        anonymous_student_id - Used for tracking modules with student id
         '''
         self.ajax_url = ajax_url
         self.xqueue = xqueue
@@ -758,6 +764,7 @@ class ModuleSystem(object):
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
         self.node_path = node_path
+        self.anonymous_student_id = anonymous_student_id
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''

common/lib/xmodule/xmodule/xml_module.py

@@ -12,6 +12,12 @@ import sys
 
 log = logging.getLogger(__name__)
 
+def name_to_pathname(name):
+    """
+    Convert a location name for use in a path: replace ':' with '/'.
+    This allows users of the xml format to organize content into directories
+    """
+    return name.replace(':', '/')
 
 def is_pointer_tag(xml_obj):
     """
@@ -245,8 +251,8 @@ class XmlDescriptor(XModuleDescriptor):
         # VS[compat] -- detect new-style each-in-a-file mode
         if is_pointer_tag(xml_object):
             # new style:
-            # read the actual definition file--named using url_name
-            filepath = cls._format_filepath(xml_object.tag, url_name)
+            # read the actual definition file--named using url_name.replace(':','/')
+            filepath = cls._format_filepath(xml_object.tag, name_to_pathname(url_name))
             definition_xml = cls.load_file(filepath, system.resources_fs, location)
         else:
             definition_xml = xml_object	# this is just a pointer, not the real definition content
@@ -292,7 +298,8 @@ class XmlDescriptor(XModuleDescriptor):
         """If this returns True, write the definition of this descriptor to a separate
         file.
 
-        NOTE: Do not override this without a good reason.  It is here specifically for customtag...
+        NOTE: Do not override this without a good reason.  It is here
+        specifically for customtag...
         """
         return True
 
@@ -335,7 +342,8 @@ class XmlDescriptor(XModuleDescriptor):
 
         if self.export_to_file():
             # Write the definition to a file
-            filepath = self.__class__._format_filepath(self.category, self.url_name)
+            url_path = name_to_pathname(self.url_name)
+            filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
                 file.write(etree.tostring(xml_object, pretty_print=True))

common/test/data/toy/chapter/secret/magic.xml

+<chapter>
+    <video url_name="toyvideo" youtube="blahblah"/>
+</chapter>

common/test/data/toy/course/2012_Fall.xml

 <course>
   <chapter url_name="Overview">
     <videosequence url_name="Toy_Videos">
-      <html url_name="toylab"/>
+      <html url_name="secret:toylab"/>
       <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
     </videosequence>
     <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
   </chapter>
+  <chapter url_name="secret:magic"/>
 </course>

common/test/data/toy/policies/2012_Fall.json

@@ -11,7 +11,7 @@
         "display_name": "Toy Videos",
         "format": "Lecture Sequence"
     },
-    "html/toylab": {
+    "html/secret:toylab": {
         "display_name": "Toy lab"
     },
     "video/Video_Resources": {

lms/djangoapps/course_wiki/editors.py

@@ -4,9 +4,12 @@ from django.utils.encoding import force_unicode
 from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
+from django.template.loader import render_to_string
+
 from wiki.editors.base import BaseEditor
 from wiki.editors.markitup import MarkItUpAdminWidget
 
+
 class CodeMirrorWidget(forms.Widget):
     def __init__(self, attrs=None):
         # The 'rows' and 'cols' attributes are required for HTML correctness.
@@ -18,9 +21,15 @@ class CodeMirrorWidget(forms.Widget):
     
     def render(self, name, value, attrs=None):
         if value is None: value = ''
+
         final_attrs = self.build_attrs(attrs, name=name)
-        return mark_safe(u'<div><textarea%s>%s</textarea></div>' % (flatatt(final_attrs),
-                conditional_escape(force_unicode(value))))
+
+        # TODO use the help_text field of edit form instead of rendering a template
+
+        return render_to_string('wiki/includes/editor_widget.html',
+                                {'attrs': mark_safe(flatatt(final_attrs)),
+                                 'content': conditional_escape(force_unicode(value)),
+                                 })
 
 
 class CodeMirror(BaseEditor):
@@ -50,5 +59,6 @@ class CodeMirror(BaseEditor):
               "js/vendor/CodeMirror/xml.js",
               "js/vendor/CodeMirror/mitx_markdown.js",
               "js/wiki/CodeMirror.init.js",
+              "js/wiki/cheatsheet.js",
               )
 

lms/djangoapps/courseware/grades.py

@@ -17,6 +17,7 @@ log = logging.getLogger("mitx.courseware")
 
 def yield_module_descendents(module):
     stack = module.get_display_items()
+    stack.reverse()
 
     while len(stack) > 0:
         next_module = stack.pop()

lms/djangoapps/courseware/module_render.py

+import hashlib
 import json
 import logging
 import sys
@@ -144,8 +145,8 @@ def get_module(user, request, location, student_module_cache, course_id, positio
 
     Arguments:
       - user                  : User for whom we're getting the module
-      - request               : current django HTTPrequest -- used in particular for auth
-                                (This is important e.g. for prof impersonation of students in progress view)
+      - request               : current django HTTPrequest.  Note: request.user isn't used for anything--all auth
+                                and such works based on user.
       - location              : A Location-like object identifying the module to load
       - student_module_cache  : a StudentModuleCache
       - course_id             : the course_id in the context of which to load module
@@ -171,12 +172,16 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     descriptor = modulestore().get_instance(course_id, location)
 
     # Short circuit--if the user shouldn't have access, bail without doing any work
-    # NOTE: Do access check on request.user -- that's who actually needs access (e.g. could be prof
-    # impersonating a user)
-    if not has_access(request.user, descriptor, 'load'):
+    if not has_access(user, descriptor, 'load'):
         return None
 
-    #TODO Only check the cache if this module can possibly have state
+    # Anonymized student identifier
+    h = hashlib.md5()
+    h.update(settings.SECRET_KEY)
+    h.update(str(user.id))
+    anonymous_student_id = h.hexdigest()
+
+    # Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
     if user.is_authenticated():
@@ -200,6 +205,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                                    location=descriptor.location.url(),
                                    dispatch=''),
                        )
+    # Intended use is as {ajax_url}/{dispatch_command}, so get rid of the trailing slash.
+    ajax_url = ajax_url.rstrip('/')
 
     # Fully qualified callback URL for external queueing system
     xqueue_callback_url = '{proto}://{host}'.format(
@@ -220,7 +227,9 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
 
     xqueue = {'interface': xqueue_interface,
               'callback_url': xqueue_callback_url,
-              'default_queuename': xqueue_default_queuename.replace(' ', '_')}
+              'default_queuename': xqueue_default_queuename.replace(' ', '_'),
+              'waittime': settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+             }
 
     def inner_get_module(location):
         """
@@ -244,7 +253,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
-                          node_path=settings.NODE_PATH
+                          node_path=settings.NODE_PATH,
+                          anonymous_student_id=anonymous_student_id
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
@@ -412,6 +422,10 @@ def modx_dispatch(request, dispatch, location, course_id):
     '''
     # ''' (fix emacs broken parsing)
 
+    # Check parameters and fail fast if there's a problem
+    if not Location.is_valid(location):
+        raise Http404("Invalid location")
+
     # Check for submitted files and basic file size checks
     p = request.POST.copy()
     if request.FILES:

lms/djangoapps/courseware/tests/tests.py

@@ -411,8 +411,6 @@ class TestViewAuth(PageLoader):
             """list of urls that only instructors/staff should be able to see"""
             urls = reverse_urls(['instructor_dashboard','gradebook','grade_summary'],
                                 course)
-            urls.append(reverse('student_progress', kwargs={'course_id': course.id,
-                                                     'student_id': user(self.student).id}))
             return urls
 
         def check_non_staff(course):
@@ -435,6 +433,17 @@ class TestViewAuth(PageLoader):
                 print 'checking for 200 on {0}'.format(url)
                 self.check_for_get_code(200, url)
 
+            # The student progress tab is not accessible to a student
+            # before launch, so the instructor view-as-student feature should return a 404 as well.
+            # TODO (vshnayder): If this is not the behavior we want, will need
+            # to make access checking smarter and understand both the effective
+            # user (the student), and the requesting user (the prof)
+            url = reverse('student_progress', kwargs={'course_id': course.id,
+                                                     'student_id': user(self.student).id})
+            print 'checking for 404 on view-as-student: {0}'.format(url)
+            self.check_for_get_code(404, url)
+
+
         # First, try with an enrolled student
         print '=== Testing student access....'
         self.login(self.student, self.password)

lms/djangoapps/courseware/views.py

@@ -333,6 +333,10 @@ def progress(request, course_id, student_id=None):
     course_module = get_module(student, request, course.location,
                                student_module_cache, course_id)
 
+    # The course_module should be accessible, but check anyway just in case something went wrong:
+    if course_module is None:
+        raise Http404("Course does not exist")
+
     courseware_summary = grades.progress_summary(student, course_module,
                                                  course.grader, student_module_cache)
     grade_summary = grades.grade(student, request, course, student_module_cache)

lms/djangoapps/dashboard/views.py

@@ -21,11 +21,17 @@ def dashboard(request):
     if not request.user.is_staff:
         raise Http404
 
-    query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc"
+    queries=[]
+    queries.append("select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc;")
+    queries.append("select count(distinct user_id) as unique_students from student_courseenrollment;")
+    queries.append("select registrations, count(registrations) from (select count(user_id) as registrations from student_courseenrollment group by user_id) as registrations_per_user group by registrations;")
     
     from django.db import connection
     cursor = connection.cursor()
+    results =[]
+
+    for query in queries:
         cursor.execute(query)
-    results = dictfetchall(cursor)
+        results.append(dictfetchall(cursor))
 
     return HttpResponse(json.dumps(results, indent=4))

lms/djangoapps/lms_migration/management/commands/manage_course_groups.py

+#!/usr/bin/python
+#
+# File:   manage_course_groups
+#
+# interactively list and edit membership in course staff and instructor groups
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+
+#-----------------------------------------------------------------------------
+# get all staff groups
+
+class Command(BaseCommand):
+    help = "Manage course group membership, interactively."
+
+    def handle(self, *args, **options):
+
+        gset = Group.objects.all()
+        
+        print "Groups:"
+        for cnt,g in zip(range(len(gset)), gset):
+            print "%d. %s" % (cnt,g)
+        
+        gnum = int(raw_input('Choose group to manage (enter #): '))
+        
+        group = gset[gnum]
+        
+        #-----------------------------------------------------------------------------
+        # users in group
+        
+        uall = User.objects.all()
+        if uall.count()<50:
+            print "----"
+            print "List of All Users: %s" % [str(x.username) for x in uall]
+            print "----"
+        else:
+            print "----"
+            print "There are %d users, which is too many to list" % uall.count()
+            print "----"
+        
+        while True:
+        
+            print "Users in the group:"
+            
+            uset = group.user_set.all()
+            for cnt, u in zip(range(len(uset)), uset):
+                print "%d. %s" % (cnt, u)
+        
+            action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ')
+        
+            m = re.match('^[0-9]+$',action)
+            if m:
+                unum = int(action)
+                u = uset[unum]
+                print "Deleting user %s" % u
+                u.groups.remove(group)
+            
+            else:
+                for uname in action.split(','):
+                    try:
+                        user = User.objects.get(username=action)
+                    except Exception as err:
+                        print "Error %s" % err
+                        continue
+                    print "adding %s to group %s" % (user, group)
+                    user.groups.add(group)
+                
+            
+            

lms/envs/common.py

@@ -86,6 +86,9 @@ DEFAULT_GROUPS = []
 # If this is true, random scores will be generated for the purpose of debugging the profile graphs
 GENERATE_PROFILE_SCORES = False
 
+# Used with XQueue
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
+
 ############################# SET PATH INFORMATION #############################
 PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /mitx/lms
 REPO_ROOT = PROJECT_ROOT.dirname()

lms/envs/dev.py

@@ -15,7 +15,7 @@ TEMPLATE_DEBUG = True
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
-MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = True
+MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
 
 WIKI_ENABLED = True

lms/envs/test.py

@@ -58,7 +58,7 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
-
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # TODO (cpennington): We need to figure out how envs/test.py can inject things
 # into common.py so that we don't have to repeat this sort of thing

lms/static/coffee/src/discussion/content.coffee

@@ -124,7 +124,7 @@ if Backbone?
         url = @model.urlFor('retrieve')
         DiscussionUtil.safeAjax
           $elem: $elem
-          $loading: $(event.target) if event
+          $loading: @$(".discussion-show-comments")
           type: "GET"
           url: url
           success: (response, textStatus) =>

lms/static/coffee/src/discussion/discussion.coffee

@@ -169,7 +169,7 @@ if Backbone?
       url = URI($elem.attr("action")).addSearch({text: @$(".search-input").val()})
       @reload($elem, url)
 
-    sort: ->
+    sort: (event) ->
       $elem = $(event.target)
       url = $elem.attr("sort-url")
       @reload($elem, url)

lms/static/coffee/src/discussion/utils.coffee

 $ ->
   $.fn.extend
     loading: ->
-      $(this).after("<span class='discussion-loading'></span>")
+      @$_loading = $("<span class='discussion-loading'></span>")
+      $(this).after(@$_loading)
     loaded: ->
-      $(this).parent().children(".discussion-loading").remove()
+      @$_loading.remove()
 
 class @DiscussionUtil
 

lms/static/js/wiki/cheatsheet.js

+$(document).ready(function () {
+    $('#cheatsheetLink').click(function() {
+        $('#cheatsheetModal').modal('show');
+    });
+    
+    $('#cheatsheetModal .close-btn').click(function(e) {
+        $('#cheatsheetModal').modal('hide');
+    });
+});
\ No newline at end of file

lms/static/sass/course/wiki/_wiki.scss

@@ -391,6 +391,18 @@ section.wiki {
     line-height: 1.4em;
   }
 
+  #div_id_content {
+    position: relative;
+  }
+
+  #hint_id_content {
+    position: absolute;
+    top: 10px;
+    right: 0%;
+    font-size: 12px;
+    text-align:right;
+  }
+
   .CodeMirror {
     background: #fafafa;
     border: 1px solid #c8c8c8;
@@ -567,11 +579,73 @@ section.wiki {
     background: #f00 !important;
   }
 
+  #cheatsheetLink {
+    text-align: right;
+    display: float;
+  }
+
+  #cheatsheetModal {
+    width: 950px;
+    margin-left: -450px;
+    margin-top: -100px;
+
+    .left-column {
+      margin-right: 10px;
+    }
+
+    .left-column,
+    .right-column {
+      float: left;
+      width: 450px;
+    }
+
+    .close-btn {
+      display: block;
+      position: absolute;
+      top: -8px;
+      right: -8px;
+      width: 30px;
+      height: 30px;
+      border-radius: 30px;
+      border: 1px solid #ccc;
+      @include linear-gradient(top, #eee, #d2d2d2);
+      font-size: 22px;
+      line-height: 28px;    
+      color: #333;
+      text-align: center;
+      @include box-shadow(0 1px 0 #fff inset, 0 1px 2px rgba(0, 0, 0, .2));
+    }
+  }
 
+  #cheatsheet-body {
+    background: #fff;
+    text-align: left;
+    padding: 20px;
+    font-size: 14px;
+    @include clearfix;
 
+    h3 {
+      font-weight: bold;
+    }
 
+    ul {
+      list-style: circle;
+      line-height: 1.4;
+      color: #333;
+    }
+  }
 
+  #cheatsheet-body section + section {
+    margin-top: 40px;
+  }
 
+  #cheatsheet-body pre{
+    color: #000;
+    text-align: left;
+    background: #eee;
+    padding: 10px;
+    font-size: 12px;
+  }
 
   /*-----------------
 

lms/templates/discussion/mustache/_content.mustache

@@ -7,11 +7,11 @@
     </div>
     <div class="discussion-right-wrapper">
       <ul class="admin-actions">
-        <li><a href="javascript:void(0)" class="admin-endorse">Endorse</a></li>
-        <li><a href="javascript:void(0)" class="admin-edit">Edit</a></li>
-        <li><a href="javascript:void(0)" class="admin-delete">Delete</a></li>
+        <li style="display: none;"><a href="javascript:void(0)" class="admin-endorse">Endorse</a></li>
+        <li style="display: none;"><a href="javascript:void(0)" class="admin-edit">Edit</a></li>
+        <li style="display: none;"><a href="javascript:void(0)" class="admin-delete">Delete</a></li>
         {{#thread}}
-          <li><a href="javascript:void(0)" class="admin-openclose">{{close_thread_text}}</a></li>
+          <li style="display: none;"><a href="javascript:void(0)" class="admin-openclose">{{close_thread_text}}</a></li>
         {{/thread}}
       </ul>
       {{#thread}}

lms/templates/wiki/base.html

@@ -39,7 +39,6 @@
     {% with mathjax_mode='wiki' %}
       {% include "mathjax_include.html" %}
     {% endwith %}
-    
   {% endaddtoblock %}
 
 {% endblock %}
@@ -69,6 +68,7 @@
 
       {% endblock %}
     </div>
+
   </section>
 
 {% endblock %}

lms/templates/wiki/create.html

@@ -42,6 +42,7 @@
             {% trans "Go back" %}
           </a>
       </div>  
+      {% include "wiki/includes/cheatsheet.html" %}
     </form>
 
   </article>

lms/templates/wiki/edit.html

@@ -40,7 +40,10 @@
       </a>
     </div>
   </div>
+  {% include "wiki/includes/cheatsheet.html" %}
 </form>
 
 {% endblock %}
 
+
+

lms/templates/wiki/includes/cheatsheet.html

+<div class="modal hide fade" id="cheatsheetModal">
+	<a href="#" class="close-btn">×</a>
+	<div id="cheatsheet-body" class="modal-body">
+		<div class="left-column">
+			<section>
+				<h2>Wiki Syntax Help</h2>
+				<p>This wiki uses <strong>Markdown</strong> for styling. There are several useful guides online. See any of the links below for in-depth details:</p>
+				<ul>
+					<li><a href="http://daringfireball.net/projects/markdown/basics" target="_blank">Markdown: Basics</a></li>
+					<li><a href="http://greg.vario.us/doc/markdown.txt" target="_blank">Quick Markdown Syntax Guide</a></li>
+					<li><a href="http://www.lowendtalk.com/discussion/6/miniature-markdown-guide" target="_blank">Miniature Markdown Guide</a></li>
+				</ul>
+				<p>To create a new wiki article, create a link to it. Clicking the link gives you the creation page.</p>
+				<pre>[Article Name](wiki:ArticleName)</pre>
+			</section>
+			<section>
+				<h3>edX Additions:</h3>
+				<pre>
+circuit-schematic:</pre>
+				<pre>
+$LaTeX Math Expression$</pre>
+			</section>
+		</div>
+
+		<div class="right-column">
+			<section>
+				<h3>Useful examples:</h3>
+				<pre>
+http://wikipedia.org
+[Wikipedia](http://wikipedia.org)
+[edX Wiki](wiki:/edx/)</pre>
+				<pre>
+Huge Header
+===========</pre>
+				<pre>
+Smaller Header
+--------------</pre>
+				<pre>
+*emphasis* or _emphasis_</pre>
+				<pre>
+**strong** or __strong__</pre>
+				<pre>
+- Unordered List
+	- Sub Item 1
+	- Sub Item 2</pre>
+				<pre>
+1. Ordered
+2. List</pre>
+				<pre>
+> Quotes</pre>
+			</section>
+		</div>
+
+		</div>
+	</div>
+</div>

lms/templates/wiki/includes/editor_widget.html

+<textarea {{ attrs }}>{{ content }}</textarea>
+<p id="hint_id_content" class="help-block">
+  Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.
+</p>

lms/wsgi.py

+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws")
+
+# This application object is used by the development server
+# as well as any WSGI server configured to use this file.
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+from django.conf import settings
+from xmodule.modulestore.django import modulestore
+
+for store_name in settings.MODULESTORE:
+    modulestore(store_name)