Merge remote-tracking branch 'origin/master' into cpennington/cms-xml-processing

README

-This branch (re-)adds dynamic math and symbolicresponse.
-Test cases included.
+see doc/ for documentation.

common/lib/mitxmako/shortcuts.py

@@ -12,13 +12,16 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+import logging
+
+log = logging.getLogger("mitx.common.lib.mitxmako")
+
 from django.template import Context
 from django.http import HttpResponse
 
 from . import middleware
 from django.conf import settings
 
-
 def render_to_string(template_name, dictionary, context=None, namespace='main'):
     context_instance = Context(dictionary)
     # add dictionary to context_instance
@@ -27,8 +30,11 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
     context_dictionary = {}
     context_instance['settings'] = settings
     context_instance['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
-    for d in middleware.requestcontext:
-        context_dictionary.update(d)
+
+    # In various testing contexts, there might not be a current request context.
+    if middleware.requestcontext is not None:
+        for d in middleware.requestcontext:
+            context_dictionary.update(d)
     for d in context_instance:
         context_dictionary.update(d)
     if context:

common/lib/xmodule/capa_module.py

@@ -108,7 +108,9 @@ class Module(XModule):
         d = self.get_score()
         score = d['score']
         total = d['total']
-        return Progress(score, total)
+        if total > 0:
+            return Progress(score, total)
+        return None
 
 
     def get_html(self):
@@ -305,7 +307,7 @@ class Module(XModule):
         after = self.get_progress()
         d.update({
             'progress_changed' : after != before,
-            'progress' : after.ternary_str(),
+            'progress_status' : Progress.to_js_status_str(after),
             })
         return json.dumps(d, cls=ComplexEncoder)
 

common/lib/xmodule/progress.py

 '''
 Progress class for modules.  Represents where a student is in a module.
-'''
+
+Useful things to know:
+ - Use Progress.to_js_status_str() to convert a progress into a simple
+   status string to pass to js.  
+ - Use Progress.to_js_detail_str() to convert a progress into a more detailed
+   string to pass to js.
+
+In particular, these functions have a canonical handing of None.
+
+For most subclassing needs, you should only need to reimplement
+frac() and __str__().
+''' 
 
 from collections import namedtuple
 import numbers
@@ -124,3 +135,26 @@ class Progress(object):
         (n, d) = a.frac()
         (n2, d2) = b.frac()
         return Progress(n + n2, d + d2)
+
+    @staticmethod
+    def to_js_status_str(progress):
+        '''
+        Return the "status string" version of the passed Progress
+        object that should be passed to js.  Use this function when
+        sending Progress objects to js to limit dependencies.
+        '''
+        if progress is None:
+            return "NA"
+        return progress.ternary_str()
+
+
+    @staticmethod
+    def to_js_detail_str(progress):
+        '''
+        Return the "detail string" version of the passed Progress
+        object that should be passed to js.  Use this function when
+        passing Progress objects to js to limit dependencies.
+        '''
+        if progress is None:
+            return "NA"
+        return str(progress)

common/lib/xmodule/seq_module.py

@@ -72,8 +72,8 @@ class Module(XModule):
 
         for contents, title, progress in zip(self.contents, titles, progresses):
             contents['title'] = title
-            contents['progress_str'] = str(progress) if progress is not None else ""
-            contents['progress_stat'] = progress.ternary_str() if progress is not None else ""
+            contents['progress_status'] = Progress.to_js_status_str(progress)
+            contents['progress_detail'] = Progress.to_js_detail_str(progress)
 
         for (content, element_class) in zip(self.contents, child_classes):
             new_class = 'other'

common/lib/xmodule/tests.py

@@ -568,6 +568,22 @@ class ProgressTest(unittest.TestCase):
         self.assertEqual(self.half_done.ternary_str(), "in_progress")
         self.assertEqual(self.done.ternary_str(), "done")
 
+    def test_to_js_status(self):
+        '''Test the Progress.to_js_status_str() method'''
+        
+        self.assertEqual(Progress.to_js_status_str(self.not_started), "none")
+        self.assertEqual(Progress.to_js_status_str(self.half_done), "in_progress")
+        self.assertEqual(Progress.to_js_status_str(self.done), "done")
+        self.assertEqual(Progress.to_js_status_str(None), "NA")
+
+    def test_to_js_detail_str(self):
+        '''Test the Progress.to_js_detail_str() method'''
+        f = Progress.to_js_detail_str
+        for p in (self.not_started, self.half_done, self.done):
+            self.assertEqual(f(p), str(p))
+        # But None should be encoded as NA
+        self.assertEqual(f(None), "NA")
+
     def test_add(self):
         '''Test the Progress.add_counts() method'''
         p = Progress(0, 2)

common/lib/xmodule/video_module.py

@@ -4,6 +4,7 @@ import logging
 from lxml import etree
 
 from x_module import XModule, XModuleDescriptor
+from progress import Progress
 
 log = logging.getLogger("mitx.courseware.modules")
 
@@ -15,17 +16,32 @@ class Module(XModule):
     video_time = 0
 
     def handle_ajax(self, dispatch, get):
+        '''
+        Handle ajax calls to this video.
+        TODO (vshnayder): This is not being called right now, so the position
+        is not being saved.
+        '''
         log.debug(u"GET {0}".format(get))
         log.debug(u"DISPATCH {0}".format(dispatch))
         if dispatch == 'goto_position':
             self.position = int(float(get['position']))
-            log.debug(u"NEW POSITION {0}".format(self.position))
+            log.info(u"NEW POSITION {0}".format(self.position))
             return json.dumps({'success':True})
         raise Http404()
 
+    def get_progress(self):
+        ''' TODO (vshnayder): Get and save duration of youtube video, then return
+        fraction watched.
+        (Be careful to notice when video link changes and update)
+
+        For now, we have no way of knowing if the video has even been watched, so
+        just return None.
+        '''
+        return None
+
     def get_state(self):
         log.debug(u"STATE POSITION {0}".format(self.position))
-        return json.dumps({ 'position':self.position })
+        return json.dumps({ 'position': self.position })
 
     @classmethod
     def get_xml_tags(c):
@@ -41,15 +57,16 @@ class Module(XModule):
             'id': self.item_id,
             'position': self.position,
             'name': self.name,
-            'annotations': self.annotations
+            'annotations': self.annotations,
         })
 
     def __init__(self, system, xml, item_id, state=None):
         XModule.__init__(self, system, xml, item_id, state)
-        xmltree=etree.fromstring(xml)
+        xmltree = etree.fromstring(xml)
         self.youtube = xmltree.get('youtube')
         self.name = xmltree.get('name')
         self.position = 0
+
         if state is not None:
             state = json.loads(state)
             if 'position' in state:

common/test/README.md

+Common test infrastructure for LMS + CMS
+===========================
+
+data/ has some test course data.  
+
+Once the course validation is separated from django, we should have scripts here that checks that a course consists only of xml that we understand.

common/test/data/full/README.md

+This is a realistic course, with many different module types and a lot of structure.  It is based on 6.002x.

common/test/data/simple/README.md

+This is a simple, but non-trivial, course using multiple module types and some nested structure.
+ 

common/test/data/toy/README.md

+This is a very very simple course, useful for initial debugging of processing code.

common/test/data/toy/toy_course.xml

+<course name="Toy Course" graceperiod="1 day 5 hours 59 minutes 59 seconds" showanswer="always" rerandomize="never">
+  <chapter name="Overview">
+    <section format="Video" name="Welcome">
+      <video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8"/>
+    </section>
+    <section format="Lecture Sequence" name="System Usage Sequence">
+      <html id="Lab2A" filename="Lab2A.html"/>
+      <video name="S0V1: Video Resources" youtube="0.75:EuzkdzfR0i8,1.0:1bK-WdDi6Qw,1.25:0v1VzoDVUTM,1.50:Bxk_-ZJb240"/>
+    </section>
+  </chapter>
+</course>

lms/djangoapps/courseware/content_parser.py

@@ -26,10 +26,13 @@ import xmodule
 
 ''' This file will eventually form an abstraction layer between the
 course XML file and the rest of the system. 
-
-TODO: Shift everything from xml.dom.minidom to XPath (or XQuery)
 '''
 
+# ==== This section has no direct dependencies on django ====================================
+# NOTE: it does still have some indirect dependencies:
+# util.memcache.fasthash (which does not depend on memcache at all)
+# 
+
 class ContentException(Exception):
     pass
 
@@ -38,29 +41,6 @@ log = logging.getLogger("mitx.courseware")
 def format_url_params(params):
     return [ urllib.quote(string.replace(' ','_')) for string in params ]
 
-def xpath(xml, query_string, **args):
-    ''' Safe xpath query into an xml tree:
-        * xml is the tree.
-        * query_string is the query
-        * args are the parameters. Substitute for {params}. 
-        We should remove this with the move to lxml. 
-        We should also use lxml argument passing. '''
-    doc = etree.fromstring(xml)
-    #print type(doc)
-    def escape(x):
-        # TODO: This should escape the string. For now, we just assume it's made of valid characters. 
-        # Couldn't figure out how to escape for lxml in a few quick Googles
-        valid_chars="".join(map(chr, range(ord('a'),ord('z')+1)+range(ord('A'),ord('Z')+1)+range(ord('0'), ord('9')+1)))+"_ "
-        for e in x:
-            if e not in valid_chars:
-                raise Exception("Invalid char in xpath expression. TODO: Escape")
-        return x
-
-    args=dict( ((k, escape(args[k])) for k in args) )
-    #print args
-    results = doc.xpath(query_string.format(**args))
-    return results
-
 def xpath_remove(tree, path):
     ''' Remove all items matching path from lxml tree.  Works in
         place.'''
@@ -69,35 +49,34 @@ def xpath_remove(tree, path):
         item.getparent().remove(item)
     return tree
 
-if __name__=='__main__':
-    print xpath('<html><problem name="Bob"></problem></html>', '/{search}/problem[@name="{name}"]', 
-                search='html', name="Bob")
-
 def id_tag(course):
     ''' Tag all course elements with unique IDs '''
     default_ids = xmodule.get_default_ids()
 
     # Tag elements with unique IDs
-    elements = course.xpath("|".join(['//'+c for c in default_ids]))
+    elements = course.xpath("|".join('//' + c for c in default_ids))
     for elem in elements:
         if elem.get('id'):
             pass
         elif elem.get(default_ids[elem.tag]):
-            new_id = elem.get(default_ids[elem.tag]) 
-            new_id = "".join([a for a in new_id if a.isalnum()]) # Convert to alphanumeric
-            # Without this, a conflict may occur between an hmtl or youtube id
+            new_id = elem.get(default_ids[elem.tag])
+            # Convert to alphanumeric
+            new_id = "".join(a for a in new_id if a.isalnum()) 
+
+            # Without this, a conflict may occur between an html or youtube id
             new_id = default_ids[elem.tag] + new_id
             elem.set('id', new_id)
         else:
-            elem.set('id', "id"+fasthash(etree.tostring(elem)))
+            elem.set('id', "id" + fasthash(etree.tostring(elem)))
             
 def propogate_downward_tag(element, attribute_name, parent_attribute = None):
     ''' This call is to pass down an attribute to all children. If an element
     has this attribute, it will be "inherited" by all of its children. If a
     child (A) already has that attribute, A will keep the same attribute and
     all of A's children will inherit A's attribute. This is a recursive call.'''
-    
-    if (parent_attribute is None): #This is the entry call. Select all elements with this attribute
+
+    if (parent_attribute is None):
+        #This is the entry call. Select all elements with this attribute
         all_attributed_elements = element.xpath("//*[@" + attribute_name +"]")
         for attributed_element in all_attributed_elements:
             attribute_value = attributed_element.get(attribute_name)
@@ -118,32 +97,6 @@ def propogate_downward_tag(element, attribute_name, parent_attribute = None):
             #to its children later.
             return
 
-def user_groups(user):
-    if not user.is_authenticated():
-        return []
-
-    # TODO: Rewrite in Django
-    key = 'user_group_names_{user.id}'.format(user=user)
-    cache_expiration = 60 * 60 # one hour
-    
-    # Kill caching on dev machines -- we switch groups a lot
-    group_names = cache.get(key)
- 
-    if group_names is None:
-        group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
-        cache.set(key, group_names, cache_expiration)
-
-    return group_names
-
-    # return [u.name for u in UserTestGroup.objects.raw("select * from auth_user, student_usertestgroup, student_usertestgroup_users where auth_user.id = student_usertestgroup_users.user_id and student_usertestgroup_users.usertestgroup_id = student_usertestgroup.id and auth_user.id = %s", [user.id])]
-
-def replace_custom_tags(tree):
-    tags = os.listdir(settings.DATA_DIR+'/custom_tags')
-    for tag in tags:
-        for element in tree.iter(tag):
-            element.tag = 'customtag'
-            impl = etree.SubElement(element, 'impl')
-            impl.text = tag
 
 def course_xml_process(tree):
     ''' Do basic pre-processing of an XML tree. Assign IDs to all
@@ -159,130 +112,254 @@ def course_xml_process(tree):
     propogate_downward_tag(tree, "rerandomize")
     return tree
 
-def course_file(user,coursename=None):
-    ''' Given a user, return course.xml'''
 
-    if user.is_authenticated():
-        filename = UserProfile.objects.get(user=user).courseware # user.profile_cache.courseware 
-    else:
-        filename = 'guest_course.xml'
+def toc_from_xml(dom, active_chapter, active_section):
+    '''
+    Create a table of contents from the course xml.
 
-    # if a specific course is specified, then use multicourse to get the right path to the course XML directory
-    if coursename and settings.ENABLE_MULTICOURSE:
-        xp = multicourse_settings.get_course_xmlpath(coursename)
-        filename = xp + filename	# prefix the filename with the path
+    Return format:
+    [ {'name': name, 'sections': SECTIONS, 'active': bool}, ... ]
 
-    groups = user_groups(user)
-    options = {'dev_content':settings.DEV_CONTENT, 
-               'groups' : groups}
+    where SECTIONS is a list
+    [ {'name': name, 'format': format, 'due': due, 'active' : bool}, ...]
 
-    
-    cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups))
-    if "dev" not in settings.DEFAULT_GROUPS:
-        tree_string = cache.get(cache_key)
-    else: 
-        tree_string = None
+    active is set for the section and chapter corresponding to the passed
+    parameters.  Everything else comes from the xml, or defaults to "".
 
-    if settings.DEBUG:
-        log.info('[courseware.content_parser.course_file] filename=%s, cache_key=%s' % (filename,cache_key))
-        # print '[courseware.content_parser.course_file] tree_string = ',tree_string
+    chapters with name 'hidden' are skipped.
+    '''
+    name = dom.xpath('//course/@name')[0]
 
-    if not tree_string:
-        tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course')))
-        tree_string = etree.tostring(tree)
-        
-        cache.set(cache_key, tree_string, 60)
-    else:
-        tree = etree.XML(tree_string)
+    chapters = dom.xpath('//course[@name=$name]/chapter', name=name)
+    ch = list()
+    for c in chapters:
+        if c.get('name') == 'hidden':
+            continue
+        sections = list()
+        for s in dom.xpath('//course[@name=$name]/chapter[@name=$chname]/section',
+                           name=name, chname=c.get('name')):
+            
+            format = s.get("subtitle") if s.get("subtitle") else s.get("format") or ""
+            active = (c.get("name") == active_chapter and
+                      s.get("name") == active_section)
+
+            sections.append({'name': s.get("name") or "", 
+                             'format': format, 
+                             'due': s.get("due") or "",
+                             'active': active})
+            
+        ch.append({'name': c.get("name"), 
+                   'sections': sections,
+                   'active': c.get("name") == active_chapter})
+    return ch
 
-    return tree
 
-def section_file(user, section, coursename=None, dironly=False):
+def replace_custom_tags_dir(tree, dir):
     '''
-    Given a user and the name of a section, return that section.
-    This is done specific to each course.
-    If dironly=True then return the sections directory.
-    TODO: This is a bit weird; dironly should be scrapped. 
+    Process tree to replace all custom tags defined in dir.
     '''
-    filename = section+".xml"
+    tags = os.listdir(dir)
+    for tag in tags:
+        for element in tree.iter(tag):
+            element.tag = 'customtag'
+            impl = etree.SubElement(element, 'impl')
+            impl.text = tag
 
-    # if a specific course is specified, then use multicourse to get the right path to the course XML directory
-    xp = ''
-    if coursename and settings.ENABLE_MULTICOURSE: xp = multicourse_settings.get_course_xmlpath(coursename)
+def parse_course_file(filename, options, namespace):
+    '''
+    Parse a course file with the given options, and return the resulting
+    xml tree object.
 
-    dirname = settings.DATA_DIR + xp + '/sections/'
+    Options should be a dictionary including keys
+        'dev_content': bool,
+        'groups' : [list, of, user, groups]
+
+    namespace is used to in searching for the file.  Could be e.g. 'course',
+    'sections'.
+    '''
+    xml = etree.XML(render_to_string(filename, options, namespace=namespace))
+    return course_xml_process(xml)
 
-    if dironly: return dirname
+
+def get_section(section, options, dirname):
+    '''
+    Given the name of a section, an options dict containing keys
+    'dev_content' and 'groups', and a directory to look in,
+    returns the xml tree for the section, or None if there's no
+    such section.
+    ''' 
+    filename = section + ".xml"
 
     if filename not in os.listdir(dirname):
-        log.error(filename+" not in "+str(os.listdir(dirname)))
+        log.error(filename + " not in " + str(os.listdir(dirname)))
         return None
 
-    options = {'dev_content':settings.DEV_CONTENT, 
-               'groups' : user_groups(user)}
-
-    tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'sections')))
+    tree = parse_course_file(filename, options, namespace='sections')
     return tree
 
 
-def module_xml(user, module, id_tag, module_id, coursename=None):
-    ''' Get XML for a module based on module and module_id. Assumes
-        module occurs once in courseware XML file or hidden section. '''
-    # Sanitize input
+def get_module(tree, module, id_tag, module_id, sections_dirname, options):
+    '''
+    Given the xml tree of the course, get the xml string for a module
+    with the specified module type, id_tag, module_id.  Looks in
+    sections_dirname for sections.
+
+    id_tag -- use id_tag if the place the module stores its id is not 'id'
+    '''
+        # Sanitize input
     if not module.isalnum():
         raise Exception("Module is not alphanumeric")
+        
     if not module_id.isalnum():
         raise Exception("Module ID is not alphanumeric")
+
     # Generate search
-    xpath_search='//{module}[(@{id_tag} = "{id}") or (@id = "{id}")]'.format(module=module, 
-                                                                             id_tag=id_tag,
-                                                                             id=module_id)
-    #result_set=doc.xpathEval(xpath_search)
-    doc = course_file(user,coursename)
-    sdirname = section_file(user,'',coursename,True)	# get directory where sections information is stored
-    section_list = (s[:-4] for s in os.listdir(sdirname) if s[-4:]=='.xml')
-
-    result_set=doc.xpath(xpath_search)
-    if len(result_set)<1:
+    xpath_search='//{module}[(@{id_tag} = "{id}") or (@id = "{id}")]'.format(
+        module=module, 
+        id_tag=id_tag,
+        id=module_id)
+    
+
+    result_set = tree.xpath(xpath_search)
+    if len(result_set) < 1:
+        # Not found in main tree.  Let's look in the section files.
+        section_list = (s[:-4] for s in os.listdir(sections_dirname) if s[-4:]=='.xml')
         for section in section_list:
             try: 
-                s = section_file(user, section, coursename)
+                s = get_section(section, options, sections_dirname)
             except etree.XMLSyntaxError: 
-                ex= sys.exc_info()
-                raise ContentException("Malformed XML in " + section+ "("+str(ex[1].msg)+")")
+                ex = sys.exc_info()
+                raise ContentException("Malformed XML in " + section +
+                                       "(" + str(ex[1].msg) + ")")
             result_set = s.xpath(xpath_search)
-            if len(result_set) != 0: 
+            if len(result_set) != 0:
                 break
 
-    if len(result_set)>1:
+    if len(result_set) > 1:
         log.error("WARNING: Potentially malformed course file", module, module_id)
+        
     if len(result_set)==0:
-        if settings.DEBUG:
-            log.error('[courseware.content_parser.module_xml] cannot find %s in course.xml tree' % xpath_search)
-            log.error('tree = %s' % etree.tostring(doc,pretty_print=True))
+        log.error('[content_parser.get_module] cannot find %s in course.xml tree',
+                      xpath_search)
+        log.error('tree = %s' % etree.tostring(tree, pretty_print=True))
         return None
-    if settings.DEBUG:
-        log.info('[courseware.content_parser.module_xml] found %s' % result_set)
+
+    # log.debug('[courseware.content_parser.module_xml] found %s' % result_set)
+
     return etree.tostring(result_set[0])
-    #return result_set[0].serialize()
 
-def toc_from_xml(dom, active_chapter, active_section):
-    name = dom.xpath('//course/@name')[0]
 
-    chapters = dom.xpath('//course[@name=$name]/chapter', name=name)
-    ch=list()
-    for c in chapters:
-        if c.get('name') == 'hidden':
-            continue
-        sections=list()
-        for s in dom.xpath('//course[@name=$name]/chapter[@name=$chname]/section', name=name, chname=c.get('name')): 
-            sections.append({'name':s.get("name") or "", 
-                             'format':s.get("subtitle") if s.get("subtitle") else s.get("format") or "", 
-                             'due':s.get("due") or "",
-                             'active':(c.get("name")==active_chapter and \
-                                           s.get("name")==active_section)})
-        ch.append({'name':c.get("name"), 
-                   'sections':sections,
-                   'active':(c.get("name")==active_chapter)})
-    return ch
+
+
+
+
+# ==== All Django-specific code below =============================================
+
+def user_groups(user):
+    if not user.is_authenticated():
+        return []
+
+    # TODO: Rewrite in Django
+    key = 'user_group_names_{user.id}'.format(user=user)
+    cache_expiration = 60 * 60 # one hour
+    
+    # Kill caching on dev machines -- we switch groups a lot
+    group_names = cache.get(key)
+ 
+    if group_names is None:
+        group_names = [u.name for u in UserTestGroup.objects.filter(users=user)]
+        cache.set(key, group_names, cache_expiration)
+
+    return group_names
+
+
+def get_options(user):
+    return {'dev_content': settings.DEV_CONTENT, 
+            'groups': user_groups(user)}
+
+
+def replace_custom_tags(tree):
+    '''Replace custom tags defined in our custom_tags dir'''
+    replace_custom_tags_dir(tree, settings.DATA_DIR+'/custom_tags')
+
+
+def course_file(user, coursename=None):
+    ''' Given a user, return an xml tree object for the course file.
+
+    Handles getting the right file, and processing it depending on the
+    groups the user is in.  Does caching of the xml strings.
+    '''
+
+    if user.is_authenticated():
+        # use user.profile_cache.courseware?
+        filename = UserProfile.objects.get(user=user).courseware 
+    else:
+        filename = 'guest_course.xml'
+
+    # if a specific course is specified, then use multicourse to get
+    # the right path to the course XML directory
+    if coursename and settings.ENABLE_MULTICOURSE:
+        xp = multicourse_settings.get_course_xmlpath(coursename)
+        filename = xp + filename	# prefix the filename with the path
+
+    groups = user_groups(user)
+    options = get_options(user)
+
+    # Try the cache...
+    cache_key = "{0}_processed?dev_content:{1}&groups:{2}".format(
+        filename,
+        options['dev_content'],
+        sorted(groups))
+    
+    if "dev" in settings.DEFAULT_GROUPS:
+        tree_string = None
+    else: 
+        tree_string = cache.get(cache_key)
+
+    if tree_string:
+        tree = etree.XML(tree_string)
+    else:
+        tree = parse_course_file(filename, options, namespace='course')
+        # Cache it
+        tree_string = etree.tostring(tree)
+        cache.set(cache_key, tree_string, 60)
+
+    return tree
+
+
+def sections_dir(coursename=None):
+    ''' Get directory where sections information is stored.
+    '''
+    # if a specific course is specified, then use multicourse to get the
+    # right path to the course XML directory
+    xp = ''
+    if coursename and settings.ENABLE_MULTICOURSE:
+        xp = multicourse_settings.get_course_xmlpath(coursename)
+
+    return settings.DATA_DIR + xp + '/sections/'
+    
+
+
+def section_file(user, section, coursename=None):
+    '''
+    Given a user and the name of a section, return that section.
+    This is done specific to each course.
+
+    Returns the xml tree for the section, or None if there's no such section.
+    '''
+    dirname = sections_dir(coursename)
+
+
+    return get_section(section, options, dirname)
+
+
+def module_xml(user, module, id_tag, module_id, coursename=None):
+    ''' Get XML for a module based on module and module_id. Assumes
+        module occurs once in courseware XML file or hidden section.
+    ''' 
+    tree = course_file(user, coursename)
+    sdirname = sections_dir(coursename)
+    options = get_options(user)
+
+    return get_module(tree, module, id_tag, module_id, sdirname, options)
 

lms/djangoapps/courseware/management/commands/check_course.py

@@ -10,46 +10,98 @@ from courseware.content_parser import course_file
 import courseware.module_render
 import xmodule
 
+import mitxmako.middleware as middleware
+middleware.MakoMiddleware()
+
+def check_names(user, course):
+    '''
+    Complain if any problems have alphanumeric names.
+    TODO (vshnayder): there are some in 6.002x that don't.  Is that actually a problem?
+    '''
+    all_ok = True
+    print "Confirming all problems have alphanumeric names"
+    for problem in course.xpath('//problem'):
+        filename = problem.get('filename')
+        if not filename.isalnum():
+            print "==============> Invalid (non-alphanumeric) filename", filename
+            all_ok = False
+    return all_ok
+
+def check_rendering(user, course):
+    '''Check that all modules render'''
+    all_ok = True
+    print "Confirming all modules render. Nothing should print during this step. "
+    for module in course.xpath('//problem|//html|//video|//vertical|//sequential|/tab'):
+        module_class = xmodule.modx_modules[module.tag]
+        # TODO: Abstract this out in render_module.py
+        try: 
+            module_class(etree.tostring(module), 
+                         module.get('id'), 
+                         ajax_url='',
+                         state=None, 
+                         track_function = lambda x,y,z:None, 
+                         render_function = lambda x: {'content':'','type':'video'})
+        except Exception as ex:
+            print "==============> Error in ", etree.tostring(module)
+            print ""
+            print ex
+            all_ok = False
+    print "Module render check finished"
+    return all_ok
+
+def check_sections(user, course):
+    all_ok = True
+    sections_dir = settings.DATA_DIR + "/sections"
+    print "Checking that all sections exist and parse properly"
+    if os.path.exists(sections_dir):
+        print "Checking all section includes are valid XML"
+        for f in os.listdir(sections_dir):
+            sectionfile = sections_dir + '/' + f
+            #print sectionfile
+            # skip non-xml files:
+            if not sectionfile.endswith('xml'):
+                continue
+            try:
+                etree.parse(sectionfile)
+            except Exception as ex:
+                print "================> Error parsing ", sectionfile
+                print ex
+                all_ok = False
+        print "checked all sections"
+    else:
+        print "Skipping check of include files -- no section includes dir ("+sections_dir+")"
+    return all_ok
+
 class Command(BaseCommand):
     help = "Does basic validity tests on course.xml."
     def handle(self, *args, **options):
-        check = True
+        all_ok = True
+
+        # TODO (vshnayder): create dummy user objects.  Anon, authenticated, staff.
+        # Check that everything works for each.
+        # The objects probably shouldn't be actual django users to avoid unneeded
+        # dependency on django.
+
+        # TODO: use args as list of files to check.  Fix loading to work for other files.
+
         sample_user = User.objects.all()[0]
+
+        
         print "Attempting to load courseware"
         course = course_file(sample_user)
-        print "Confirming all problems have alphanumeric names"
-        for problem in course.xpath('//problem'):
-            filename = problem.get('filename')
-            if not filename.isalnum():
-                print "==============> Invalid (non-alphanumeric) filename", filename
-                check = False
-        print "Confirming all modules render. Nothing should print during this step. "
-        for module in course.xpath('//problem|//html|//video|//vertical|//sequential|/tab'):
-            module_class = xmodule.modx_modules[module.tag]
-            # TODO: Abstract this out in render_module.py
-            try: 
-                module_class(etree.tostring(module), 
-                             module.get('id'), 
-                             ajax_url='',
-                             state=None, 
-                             track_function = lambda x,y,z:None, 
-                             render_function = lambda x: {'content':'','type':'video'})
-            except:
-                print "==============> Error in ", etree.tostring(module)
-                check = False
-        print "Module render check finished"
-        sections_dir = settings.DATA_DIR+"sections"
-        if os.path.exists(sections_dir):
-            print "Checking all section includes are valid XML"
-            for f in os.listdir(sections_dir):
-                print f
-                etree.parse(sections_dir+'/'+f)
-        else:
-            print "Skipping check of include files -- no section includes dir ("+sections_dir+")"
+
+        to_run = [check_names,
+                  # TODO (vshnayder) : make check_rendering work (use module_render.py),
+                  # turn it on
+                  #                  check_rendering,
+                  check_sections,
+                  ]
+        for check in to_run:
+            all_ok = check(sample_user, course) and all_ok
+
         # TODO: print "Checking course properly annotated with preprocess.py"
         
-            
-        if check:
+        if all_ok:
             print 'Courseware passes all checks!'
         else: 
             print "Courseware fails some checks"

lms/djangoapps/courseware/module_render.py

@@ -199,7 +199,7 @@ def get_module(user, request, module_xml, student_module_cache, position=None):
                             etree.tostring(module_xml), 
                             module_id, 
                             state=state)
-    
+
     # If StudentModule for this instance wasn't already in the database,
     # and this isn't a guest user, create it.
     if not smod and user.is_authenticated():

lms/static/coffee/src/modules/problem.coffee

@@ -19,7 +19,7 @@ class @Problem
 
   update_progress: (response) =>
     if response.progress_changed
-        @element.attr progress: response.progress
+        @element.attr progress: response.progress_status
         @element.trigger('progressChanged')
 
   render: (content) ->

lms/static/coffee/src/modules/sequence.coffee

@@ -20,8 +20,16 @@ class @Sequence
     $('.problems-wrapper').bind 'progressChanged', @updateProgress
 
   mergeProgress: (p1, p2) ->
+    # if either is "NA", return the other one
+    if p1 == "NA"
+      return p2
+    if p2 == "NA"
+      return p1
+
+    # Both real progresses
     if p1 == "done" and p2 == "done"
       return "done"
+
     # not done, so if any progress on either, in_progress
     w1 = p1 == "done" or p1 == "in_progress"
     w2 = p2 == "done" or p2 == "in_progress"
@@ -31,7 +39,7 @@ class @Sequence
     return "none"
 
   updateProgress: =>
-    new_progress = "none"
+    new_progress = "NA"
     _this = this
     $('.problems-wrapper').each (index) ->
       progress = $(this).attr 'progress'
@@ -41,6 +49,7 @@ class @Sequence
     @setProgress(new_progress, @link_for(@position))
 
   setProgress: (progress, element) ->
+      # If progress is "NA", don't add any css class
       element.removeClass('progress-none')
              .removeClass('progress-some')
              .removeClass('progress-done')
@@ -53,10 +62,12 @@ class @Sequence
     $.each @elements, (index, item) =>
       link = $('<a>').attr class: "seq_#{item.type}_inactive", 'data-element': index + 1
       title = $('<p>').html(item.title)
-      # TODO: add item.progress_str either to the title or somewhere else.
-      # Make sure it gets updated after ajax calls
+      # TODO (vshnayder): add item.progress_detail either to the title or somewhere else.
+      # Make sure it gets updated after ajax calls.
+      # implementation note: will need to figure out how to handle combining detail
+      # statuses of multiple modules in js. 
       list_item = $('<li>').append(link.append(title))
-      @setProgress item.progress_stat, link
+      @setProgress item.progress_status, link
         
       @$('#sequence-list').append list_item
 

lms/static/sass/courseware/_sequence-nav.scss

@@ -66,6 +66,26 @@ nav.sequence-nav {
         @include transition(all, .4s, $ease-in-out-quad);
         width: 100%;
 
+        &.progress {
+          border-bottom-style: solid;
+          border-bottom-width: 4px;
+        }
+
+        &.progress-none {
+          @extend .progress;
+          border-bottom-color: red;
+        }
+
+        &.progress-some {
+          @extend .progress;
+          border-bottom-color: yellow;
+        }
+
+        &.progress-done {
+          @extend .progress;
+          border-bottom-color: green;
+        }
+
         //video
         &.seq_video_inactive {
           @extend .inactive;