Merge branch 'master' into cpennington/cms

Conflicts:
	rakefile
	requirements.txt

.pylintrc

+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once).
+#disable=
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html
+output-format=text
+
+# Include message's id in output
+include-ids=no
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=no
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input
+
+# Regular expression which should only match correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression which should only match correct module level names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression which should only match correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression which should only match correct function names
+function-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct method names
+method-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct variable names
+variable-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Regular expression which should only match functions or classes name which do
+# not require a docstring
+no-docstring-rgx=__.*__
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=120
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the beginning of the name of dummy variables
+# (i.e. not used).
+dummy-variables-rgx=_|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,string,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branchs=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defines in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

common/lib/capa/capa_problem.py

 #
 # File:   capa/capa_problem.py
 #
+# Nomenclature:
+#
+# A capa Problem is a collection of text and capa Response questions.  Each Response may have one or more
+# Input entry fields.  The capa Problem may include a solution.
+#
 '''
 Main module which shows problems (of "capa" type).
 
 This is used by capa_module.
 '''
 
-import copy
+from __future__ import division
+
 import logging
 import math
 import numpy
@@ -18,44 +24,26 @@ import scipy
 import struct
 
 from lxml import etree
-from lxml.etree import Element
 from xml.sax.saxutils import unescape
 
-from util import contextualize_text
+import calc
+from correctmap import CorrectMap
+import eia
 import inputtypes
+from util import contextualize_text
 
-from responsetypes import NumericalResponse, FormulaResponse, CustomResponse, SchematicResponse, MultipleChoiceResponse, TrueFalseResponse, ExternalResponse, ImageResponse, OptionResponse, SymbolicResponse
+# to be replaced with auto-registering
+import responsetypes
 
-import calc
-import eia
+# dict of tagname, Response Class -- this should come from auto-registering
+response_tag_dict = dict([(x.response_tag,x) for x in responsetypes.__all__])
 
-log = logging.getLogger(__name__)
-
-response_types = {'numericalresponse': NumericalResponse,
-                  'formularesponse': FormulaResponse,
-                  'customresponse': CustomResponse,
-                  'schematicresponse': SchematicResponse,
-                  'externalresponse': ExternalResponse,
-                  'multiplechoiceresponse': MultipleChoiceResponse,
-                  'truefalseresponse': TrueFalseResponse,
-                  'imageresponse': ImageResponse,
-                  'optionresponse': OptionResponse,
-                  'symbolicresponse': SymbolicResponse,
-                  }
 entry_types = ['textline', 'schematic', 'choicegroup', 'textbox', 'imageinput', 'optioninput']
 solution_types = ['solution']    			# extra things displayed after "show answers" is pressed
 response_properties = ["responseparam", "answer"]    	# these get captured as student responses
 
-# How to convert from original XML to HTML
-# We should do this with xlst later
+# special problem tags which should be turned into innocuous HTML
 html_transforms = {'problem': {'tag': 'div'},
-                   "numericalresponse": {'tag': 'span'},
-                   "customresponse": {'tag': 'span'},
-                   "externalresponse": {'tag': 'span'},
-                   "schematicresponse": {'tag': 'span'},
-                   "formularesponse": {'tag': 'span'},
-                   "symbolicresponse": {'tag': 'span'},
-                   "multiplechoiceresponse": {'tag': 'span'},
                    "text": {'tag': 'span'},
                    "math": {'tag': 'span'},
                    }
@@ -68,31 +56,37 @@ global_context = {'random': random,
                   'eia': eia}
 
 # These should be removed from HTML output, including all subelements
-html_problem_semantics = ["responseparam", "answer", "script"]
-# These should be removed from HTML output, but keeping subelements
-html_skip = ["numericalresponse", "customresponse", "schematicresponse", "formularesponse", "text", "externalresponse", 'symbolicresponse']
+html_problem_semantics = ["responseparam", "answer", "script","hintgroup"]
 
-# removed in MC
-## These should be transformed
-#html_special_response = {"textline":inputtypes.textline.render,
-#                         "schematic":inputtypes.schematic.render,
-#                         "textbox":inputtypes.textbox.render,
-#                         "formulainput":inputtypes.jstextline.render,
-#                         "solution":inputtypes.solution.render,
-#                         }
+log = logging.getLogger('mitx.' + __name__)
 
+#-----------------------------------------------------------------------------
+# main class for this module
 
 class LoncapaProblem(object):
+    '''
+    Main class for capa Problems.
+    '''
+
     def __init__(self, fileobject, id, state=None, seed=None, system=None):
+        '''
+        Initializes capa Problem.  The problem itself is defined by the XML file
+        pointed to by fileobject.
+
+        Arguments:
+
+         - filesobject  : an OSFS instance: see fs.osfs
+         - id           : string used as the identifier for this problem; often a filename (no spaces)
+         - state        : student state (represented as a dict)
+         - seed         : random number generator seed (int)
+         - system       : I4xSystem instance which provides OS, rendering, and user context 
+
+        '''
+
         ## Initialize class variables from state
-        self.seed = None
-        self.student_answers = dict()
-        self.correct_map = dict()
-        self.done = False
+        self.do_reset()
         self.problem_id = id
         self.system = system
-
-        if seed is not None:
         self.seed = seed
 
         if state:
@@ -101,7 +95,7 @@ class LoncapaProblem(object):
             if 'student_answers' in state:
                 self.student_answers = state['student_answers']
             if 'correct_map' in state:
-                self.correct_map = state['correct_map']
+                self.correct_map.set_dict(state['correct_map'])
             if 'done' in state:
                 self.done = state['done']
 
@@ -109,22 +103,30 @@ class LoncapaProblem(object):
         if not self.seed:
             self.seed = struct.unpack('i', os.urandom(4))[0]
 
-        ## Parse XML file
-        if getattr(system, 'DEBUG', False):
+        self.fileobject = fileobject    	# save problem file object, so we can use for debugging information later
+        if getattr(system, 'DEBUG', False):	# get the problem XML string from the problem file
             log.info("[courseware.capa.capa_problem.lcp.init]  fileobject = %s" % fileobject)
         file_text = fileobject.read()
-        self.fileobject = fileobject    # save it, so we can use for debugging information later
-        # Convert startouttext and endouttext to proper <text></text>
-        # TODO: Do with XML operations
-        file_text = re.sub("startouttext\s*/", "text", file_text)
+        file_text = re.sub("startouttext\s*/", "text", file_text)   # Convert startouttext and endouttext to proper <text></text>
         file_text = re.sub("endouttext\s*/", "/text", file_text)
-        self.tree = etree.XML(file_text)
 
-        self.preprocess_problem(self.tree, correct_map=self.correct_map, answer_map=self.student_answers)
-        self.context = self.extract_context(self.tree, seed=self.seed)
-        for response in self.tree.xpath('//' + "|//".join(response_types)):
-            responder = response_types[response.tag](response, self.context, self.system)
-            responder.preprocess_response()
+        self.tree = etree.XML(file_text)	# parse problem XML file into an element tree
+
+        # construct script processor context (eg for customresponse problems)
+        self.context = self._extract_context(self.tree, seed=self.seed)
+
+        # pre-parse the XML tree: modifies it to add ID's and perform some in-place transformations
+        # this also creates the dict (self.responders) of Response instances for each question in the problem.
+        # the dict has keys = xml subtree of Response, values = Response instance
+        self._preprocess_problem(self.tree)
+
+    def do_reset(self):
+        '''
+        Reset internal state to unfinished, with no answers
+        '''
+        self.student_answers = dict()
+        self.correct_map = CorrectMap()
+        self.done = False
 
     def __unicode__(self):
         return u"LoncapaProblem ({0})".format(self.fileobject)
@@ -133,25 +135,49 @@ class LoncapaProblem(object):
         ''' Stored per-user session data neeeded to:
             1) Recreate the problem
             2) Populate any student answers. '''
+
         return {'seed': self.seed,
                 'student_answers': self.student_answers,
-                'correct_map': self.correct_map,
+                'correct_map': self.correct_map.get_dict(),
                 'done': self.done}
 
     def get_max_score(self):
         '''
-        TODO: multiple points for programming problems.
+        Return maximum score for this problem.
+        We do this by counting the number of answers available for each question
+        in the problem.  If the Response for a question has a get_max_score() method
+        then we call that and add its return value to the count.  That can be
+        used to give complex problems (eg programming questions) multiple points.
         '''
-        sum = 0
-        for et in entry_types:
-            sum = sum + self.tree.xpath('count(//' + et + ')')
-        return int(sum)
+        maxscore = 0
+        for responder in self.responders.values():
+            if hasattr(responder,'get_max_score'):
+                try:
+                    maxscore += responder.get_max_score()
+                except Exception:
+                    log.debug('responder %s failed to properly return from get_max_score()' % responder) # FIXME
+                    raise
+            else:
+                try:
+                    maxscore += len(responder.get_answers())
+                except:
+                    log.debug('responder %s failed to properly return get_answers()' % responder) # FIXME
+                    raise
+        return maxscore
 
     def get_score(self):
+        '''
+        Compute score for this problem.  The score is the number of points awarded.
+        Returns an integer, from 0 to get_max_score().
+        '''
         correct = 0
         for key in self.correct_map:
-            if self.correct_map[key] == u'correct':
-                correct += 1
+            try:
+                correct += self.correct_map.get_npoints(key)
+            except Exception:
+                log.error('key=%s, correct_map = %s' % (key,self.correct_map))
+                raise
+
         if (not self.student_answers) or len(self.student_answers) == 0:
             return {'score': 0,
                     'total': self.get_max_score()}
@@ -166,42 +192,37 @@ class LoncapaProblem(object):
         of each key removed (the string before the first "_").
 
         Thus, for example, input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123
+
+        Calles the Response for each question in this problem, to do the actual grading.
         '''
         self.student_answers = answers
-        self.correct_map = dict()
-        problems_simple = self.extract_problems(self.tree)
-        for response in problems_simple:
-            grader = response_types[response.tag](response, self.context, self.system)
-            results = grader.get_score(answers)        # call the responsetype instance to do the actual grading
-            self.correct_map.update(results)
-        return self.correct_map
+        oldcmap = self.correct_map				# old CorrectMap
+        newcmap = CorrectMap()					# start new with empty CorrectMap
+        # log.debug('Responders: %s' % self.responders)
+        for responder in self.responders.values():
+            results = responder.evaluate_answers(answers,oldcmap)      # call the responsetype instance to do the actual grading
+            newcmap.update(results)
+        self.correct_map = newcmap
+        # log.debug('%s: in grade_answers, answers=%s, cmap=%s' % (self,answers,newcmap))
+        return newcmap
 
     def get_question_answers(self):
-        """Returns a dict of answer_ids to answer values. If we can't generate
+        """Returns a dict of answer_ids to answer values. If we cannot generate
         an answer (this sometimes happens in customresponses), that answer_id is
         not included. Called by "show answers" button JSON request
         (see capa_module)
         """
         answer_map = dict()
-        problems_simple = self.extract_problems(self.tree)    # purified (flat) XML tree of just response queries
-        for response in problems_simple:
-            responder = response_types[response.tag](response, self.context, self.system)    # instance of numericalresponse, customresponse,...
+        for responder in self.responders.values():
             results = responder.get_answers()
             answer_map.update(results)                # dict of (id,correct_answer)
 
-        # example for the following: <textline size="5" correct_answer="saturated" />
-        for entry in problems_simple.xpath("//" + "|//".join(response_properties + entry_types)):
-            answer = entry.get('correct_answer')        # correct answer, when specified elsewhere, eg in a textline
-            if answer:
-                answer_map[entry.get('id')] = contextualize_text(answer, self.context)
-
         # include solutions from <solution>...</solution> stanzas
-        # Tentative merge; we should figure out how we want to handle hints and solutions
         for entry in self.tree.xpath("//" + "|//".join(solution_types)):
             answer = etree.tostring(entry)
-            if answer:
-                answer_map[entry.get('id')] = answer
+            if answer: answer_map[entry.get('id')] = answer
 
+        log.debug('answer_map = %s' % answer_map)
         return answer_map
 
     def get_answer_ids(self):
@@ -209,19 +230,19 @@ class LoncapaProblem(object):
         the dicts returned by grade_answers and get_question_answers. (Though
         get_question_answers may only return a subset of these."""
         answer_ids = []
-        problems_simple = self.extract_problems(self.tree)
-        for response in problems_simple:
-            responder = response_types[response.tag](response, self.context)
-            if hasattr(responder, "answer_id"):
-                answer_ids.append(responder.answer_id)
-            # customresponse types can have multiple answer_ids
-            elif hasattr(responder, "answer_ids"):
-                answer_ids.extend(responder.answer_ids)
-
+        for responder in self.responders.values():
+            answer_ids.append(responder.get_answers().keys())
         return answer_ids
 
-    # ======= Private ========
-    def extract_context(self, tree, seed=struct.unpack('i', os.urandom(4))[0]):  # private
+    def get_html(self):
+        '''
+        Main method called externally to get the HTML to be rendered for this capa Problem.
+        '''
+        return contextualize_text(etree.tostring(self._extract_html(self.tree)), self.context)
+
+    # ======= Private Methods Below ========
+
+    def _extract_context(self, tree, seed=struct.unpack('i', os.urandom(4))[0]):  # private
         '''
         Extract content of <script>...</script> from the problem.xml file, and exec it in the
         context of this problem.  Provides ability to randomize problems, and also set
@@ -235,7 +256,6 @@ class LoncapaProblem(object):
         context['__builtins__'] = globals()['__builtins__']    	# put globals there also
         context['the_lcp'] = self                		# pass instance of LoncapaProblem in
 
-        #for script in tree.xpath('/problem/script'):
         for script in tree.findall('.//script'):
             stype = script.get('type')
             if stype:
@@ -253,158 +273,103 @@ class LoncapaProblem(object):
                 log.exception("Error while execing code: " + code)
         return context
 
-    def get_html(self):
-        return contextualize_text(etree.tostring(self.extract_html(self.tree)[0]), self.context)
+    def _extract_html(self, problemtree):  # private
+        '''
+        Main (private) function which converts Problem XML tree to HTML.
+        Calls itself recursively.
 
-    def extract_html(self, problemtree):  # private
-        ''' Helper function for get_html. Recursively converts XML tree to HTML
+        Returns Element tree of XHTML representation of problemtree.
+        Calls render_html of Response instances to render responses into XHTML.
+
+        Used by get_html.
         '''
         if problemtree.tag in html_problem_semantics:
             return
 
         problemid = problemtree.get('id')    # my ID
 
-        # used to be
-        # if problemtree.tag in html_special_response:
-
         if problemtree.tag in inputtypes.get_input_xml_tags():
-            # status is currently the answer for the problem ID for the input element,
-            # but it will turn into a dict containing both the answer and any associated message
-            # for the problem ID for the input element.
+
             status = "unsubmitted"
+            msg = ''
+            hint = ''
+            hintmode = None
             if problemid in self.correct_map:
-                status = self.correct_map[problemtree.get('id')]
+                pid = problemtree.get('id')
+                status = self.correct_map.get_correctness(pid)
+                msg = self.correct_map.get_msg(pid)
+                hint = self.correct_map.get_hint(pid)
+                hintmode = self.correct_map.get_hintmode(pid)
 
             value = ""
             if self.student_answers and problemid in self.student_answers:
                 value = self.student_answers[problemid]
 
-            #### This code is a hack. It was merged to help bring two branches
-            #### in sync, but should be replaced. msg should be passed in a
-            #### response_type
-            # prepare the response message, if it exists in correct_map
-            if 'msg' in self.correct_map:
-                msg = self.correct_map['msg']
-            elif ('msg_%s' % problemid) in self.correct_map:
-                msg = self.correct_map['msg_%s' % problemid]
-            else:
-                msg = ''
-
             # do the rendering
-            # This should be broken out into a helper function
-            # that handles all input objects
             render_object = inputtypes.SimpleInput(system=self.system,
                                                    xml=problemtree,
                                                    state={'value': value,
                                                           'status': status,
                                                           'id': problemtree.get('id'),
-                                                          'feedback': {'message': msg}
+                                                          'feedback': {'message': msg,
+                                                                       'hint' : hint,
+                                                                       'hintmode' : hintmode,
+                                                                       }
                                                           },
                                                    use='capa_input')
             return render_object.get_html()  # function(problemtree, value, status, msg) # render the special response (textline, schematic,...)
 
-        tree = Element(problemtree.tag)
+        if problemtree in self.responders:		# let each Response render itself
+            return self.responders[problemtree].render_html(self._extract_html)
+
+        tree = etree.Element(problemtree.tag)
         for item in problemtree:
-            subitems = self.extract_html(item)
-            if subitems is not None:
-                for subitem in subitems:
-                    tree.append(subitem)
-        for (key, value) in problemtree.items():
+            item_xhtml = self._extract_html(item)		# nothing special: recurse
+            if item_xhtml is not None:
+                    tree.append(item_xhtml)
+
+        if tree.tag in html_transforms:
+            tree.tag = html_transforms[problemtree.tag]['tag']
+        else:
+            for (key, value) in problemtree.items():	# copy attributes over if not innocufying
                 tree.set(key, value)
 
         tree.text = problemtree.text
         tree.tail = problemtree.tail
 
-        if problemtree.tag in html_transforms:
-            tree.tag = html_transforms[problemtree.tag]['tag']
-            # Reset attributes. Otherwise, we get metadata in HTML
-            # (e.g. answers)
-            # TODO: We should remove and not zero them.
-            # I'm not sure how to do that quickly with lxml
-            for k in tree.keys():
-                tree.set(k, "")
-
-        # TODO: Fix. This loses Element().tail
-        #if problemtree.tag in html_skip:
-        #    return tree
-        return [tree]
-
-    def preprocess_problem(self, tree, correct_map=dict(), answer_map=dict()):  # private
+        return tree
+
+    def _preprocess_problem(self, tree):  # private
         '''
         Assign IDs to all the responses
         Assign sub-IDs to all entries (textline, schematic, etc.)
         Annoted correctness and value
         In-place transformation
+
+        Also create capa Response instances for each responsetype and save as self.responders
         '''
         response_id = 1
-        for response in tree.xpath('//' + "|//".join(response_types)):
+        self.responders = {}
+        for response in tree.xpath('//' + "|//".join(response_tag_dict)):
             response_id_str = self.problem_id + "_" + str(response_id)
-            response.attrib['id'] = response_id_str
-            if response_id not in correct_map:
-                correct = 'unsubmitted'
-            response.attrib['state'] = correct
-            response_id = response_id + 1
+            response.set('id',response_id_str)				# create and save ID for this response
+            response_id += 1
+
             answer_id = 1
-            for entry in tree.xpath("|".join(['//' + response.tag + '[@id=$id]//' + x for x in (entry_types + solution_types)]),
-                                    id=response_id_str):
-                # assign one answer_id for each entry_type or solution_type
+            inputfields = tree.xpath("|".join(['//' + response.tag + '[@id=$id]//' + x for x in (entry_types + solution_types)]),
+                                    id=response_id_str)
+            for entry in inputfields:			                # assign one answer_id for each entry_type or solution_type
                 entry.attrib['response_id'] = str(response_id)
                 entry.attrib['answer_id'] = str(answer_id)
                 entry.attrib['id'] = "%s_%i_%i" % (self.problem_id, response_id, answer_id)
                 answer_id = answer_id + 1
 
+            responder = response_tag_dict[response.tag](response, inputfields, self.context, self.system) # instantiate capa Response
+            self.responders[response] = responder                               # save in list in self
+
         # <solution>...</solution> may not be associated with any specific response; give IDs for those separately
         # TODO: We should make the namespaces consistent and unique (e.g. %s_problem_%i).
         solution_id = 1
         for solution in tree.findall('.//solution'):
             solution.attrib['id'] = "%s_solution_%i" % (self.problem_id, solution_id)
             solution_id += 1
-
-    def extract_problems(self, problem_tree):
-        ''' Remove layout from the problem, and give a purified XML tree of just the problems '''
-        problem_tree = copy.deepcopy(problem_tree)
-        tree = Element('problem')
-        for response in problem_tree.xpath("//" + "|//".join(response_types)):
-            newresponse = copy.copy(response)
-            for e in newresponse:
-                newresponse.remove(e)
-            # copy.copy is needed to make xpath work right. Otherwise, it starts at the root
-            # of the tree. We should figure out if there's some work-around
-            for e in copy.copy(response).xpath("//" + "|//".join(response_properties + entry_types)):
-                newresponse.append(e)
-
-            tree.append(newresponse)
-        return tree
-
-if __name__ == '__main__':
-    problem_id = 'simpleFormula'
-    filename = 'simpleFormula.xml'
-
-    problem_id = 'resistor'
-    filename = 'resistor.xml'
-
-    lcp = LoncapaProblem(filename, problem_id)
-
-    context = lcp.extract_context(lcp.tree)
-    problem = lcp.extract_problems(lcp.tree)
-    print lcp.grade_problems({'resistor_2_1': '1.0', 'resistor_3_1': '2.0'})
-    #print lcp.grade_problems({'simpleFormula_2_1':'3*x^3'})
-#numericalresponse(problem, context)
-
-#print etree.tostring((lcp.tree))
-    print '============'
-    print
-#print etree.tostring(lcp.extract_problems(lcp.tree))
-    print lcp.get_html()
-#print extract_context(tree)
-
-
-
-    # def handle_fr(self, element):
-    #     problem={"answer":self.contextualize_text(answer),
-    #              "type":"formularesponse",
-    #              "tolerance":evaluator({},{},self.contextualize_text(tolerance)),
-    #              "sample_range":dict(zip(variables, sranges)),
-    #              "samples_count": numsamples,
-    #              "id":id,
-    #     self.questions[self.lid]=problem

common/lib/capa/correctmap.py

+#-----------------------------------------------------------------------------
+# class used to store graded responses to CAPA questions
+#
+# Used by responsetypes and capa_problem
+
+class CorrectMap(object):
+    '''
+    Stores map between answer_id and response evaluation result for each question
+    in a capa problem.  The response evaluation result for each answer_id includes
+    (correctness, npoints, msg, hint, hintmode).
+
+    - correctness : either 'correct' or 'incorrect'
+    - npoints     : None, or integer specifying number of points awarded for this answer_id
+    - msg         : string (may have HTML) giving extra message response (displayed below textline or textbox)
+    - hint        : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg)
+    - hintmode    : one of (None,'on_request','always') criteria for displaying hint
+
+    Behaves as a dict.
+    '''
+    def __init__(self,*args,**kwargs):
+        self.cmap = dict()		# start with empty dict
+        self.items = self.cmap.items
+        self.keys = self.cmap.keys
+        self.set(*args,**kwargs)
+
+    def __getitem__(self, *args, **kwargs):
+        return self.cmap.__getitem__(*args, **kwargs)
+
+    def __iter__(self):
+        return self.cmap.__iter__()
+
+    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None):
+        if answer_id is not None:
+            self.cmap[answer_id] = {'correctness': correctness,
+                                    'npoints': npoints,
+                                    'msg': msg,
+                                    'hint' : hint,
+                                    'hintmode' : hintmode,
+                                    }
+
+    def __repr__(self):
+        return repr(self.cmap)
+
+    def get_dict(self):
+        '''
+        return dict version of self
+        '''
+        return self.cmap
+
+    def set_dict(self,correct_map):
+        '''
+        set internal dict to provided correct_map dict
+        for graceful migration, if correct_map is a one-level dict, then convert it to the new
+        dict of dicts format.
+        '''
+        if correct_map and not (type(correct_map[correct_map.keys()[0]])==dict):
+            self.__init__()							# empty current dict
+            for k in correct_map: self.set(k,correct_map[k])			# create new dict entries
+        else:
+            self.cmap = correct_map
+
+    def is_correct(self,answer_id):
+        if answer_id in self.cmap: return self.cmap[answer_id]['correctness'] == 'correct'
+        return None
+
+    def get_npoints(self,answer_id):
+        if self.is_correct(answer_id):
+            npoints = self.cmap[answer_id].get('npoints',1)	# default to 1 point if correct
+            return npoints or 1
+        return 0						# if not correct, return 0
+
+    def set_property(self,answer_id,property,value):
+        if answer_id in self.cmap: self.cmap[answer_id][property] = value
+        else: self.cmap[answer_id] = {property:value}
+
+    def get_property(self,answer_id,property,default=None):
+        if answer_id in self.cmap: return self.cmap[answer_id].get(property,default)
+        return default
+
+    def get_correctness(self,answer_id):
+        return self.get_property(answer_id,'correctness')
+
+    def get_msg(self,answer_id):
+        return self.get_property(answer_id,'msg','')
+
+    def get_hint(self,answer_id):
+        return self.get_property(answer_id,'hint','')
+
+    def get_hintmode(self,answer_id):
+        return self.get_property(answer_id,'hintmode',None)
+
+    def set_hint_and_mode(self,answer_id,hint,hintmode):
+        '''
+          - hint     : (string) HTML text for hint
+          - hintmode : (string) mode for hint display ('always' or 'on_request')
+        '''
+        self.set_property(answer_id,'hint',hint)
+        self.set_property(answer_id,'hintmode',hintmode)
+
+    def update(self,other_cmap):
+        '''
+        Update this CorrectMap with the contents of another CorrectMap
+        '''
+        if not isinstance(other_cmap,CorrectMap):
+            raise Exception('CorrectMap.update called with invalid argument %s' % other_cmap)
+        self.cmap.update(other_cmap.get_dict())
+
+
+    

common/lib/capa/eia.py

+""" Standard resistor codes.
+http://en.wikipedia.org/wiki/Electronic_color_code
+"""
 E6=[10,15,22,33,47,68]
 E12=[10,12,15,18,22,27,33,39,47,56,68,82]
 E24=[10,12,15,18,22,27,33,39,47,56,68,82,11,13,16,20,24,30,36,43,51,62,75,91]

common/lib/capa/inputtypes.py

@@ -32,100 +32,71 @@ def get_input_xml_tags():
     return SimpleInput.get_xml_tags()
 
 class SimpleInput():# XModule
-    ''' Type for simple inputs -- plain HTML with a form element
-    State is a dictionary with optional keys: 
+    '''
+    Type for simple inputs -- plain HTML with a form element
+    '''
+
+    xml_tags = {} ## Maps tags to functions
+
+    def __init__(self, system, xml, item_id = None, track_url=None, state=None, use = 'capa_input'):
+        '''
+        Instantiate a SimpleInput class.  Arguments:
+
+        - system    : I4xSystem instance which provides OS, rendering, and user context 
+        - xml       : Element tree of this Input element
+        - item_id   : id for this input element (assigned by capa_problem.LoncapProblem) - string
+        - track_url : URL used for tracking - string
+        - state     : a dictionary with optional keys: 
                       * Value
                       * ID
                       * Status (answered, unanswered, unsubmitted)
                       * Feedback (dictionary containing keys for hints, errors, or other 
                         feedback from previous attempt)
+        - use        :
         '''
 
-    xml_tags = {} ## Maps tags to functions
-    
-    @classmethod
-    def get_xml_tags(c):
-        return c.xml_tags.keys()
-
-    @classmethod
-    def get_uses(c):
-        return ['capa_input', 'capa_transform']
-
-    def get_html(self):
-        return self.xml_tags[self.tag](self.xml, self.value, self.status, self.system.render_template, self.msg)
-
-    def __init__(self, system, xml, item_id = None, track_url=None, state=None, use = 'capa_input'):
         self.xml = xml
         self.tag = xml.tag
-        if not state:
-            state = {}
+        self.system = system
+        if not state: state = {}
+
         ## ID should only come from one place. 
         ## If it comes from multiple, we use state first, XML second, and parameter
         ## third. Since we don't make this guarantee, we can swap this around in 
         ## the future if there's a more logical order. 
-        if item_id:
-            self.id = item_id
-        if xml.get('id'):
-            self.id = xml.get('id')
-        if 'id' in state:
-            self.id = state['id']
-        self.system = system
+        if item_id:       self.id = item_id
+        if xml.get('id'): self.id = xml.get('id')
+        if 'id' in state: self.id = state['id']
 
         self.value = ''
         if 'value' in state:
             self.value = state['value']
 
         self.msg = ''
-        if 'feedback' in state and 'message' in state['feedback']:
-            self.msg = state['feedback']['message']
+        feedback = state.get('feedback')
+        if feedback is not None:
+            self.msg = feedback.get('message','')
+            self.hint = feedback.get('hint','')
+            self.hintmode = feedback.get('hintmode',None)
+            
+            # put hint above msg if to be displayed
+            if self.hintmode == 'always':
+                self.msg = self.hint + ('<br/.>' if self.msg else '') + self.msg
  
         self.status = 'unanswered'
         if 'status' in state:
             self.status = state['status']
 
-## TODO
-# class SimpleTransform():
-#     ''' Type for simple XML to HTML transforms. Examples:
-#     * Math tags, which go from LON-CAPA-style m-tags to MathJAX
-#     '''
-#     xml_tags = {} ## Maps tags to functions
-    
-#     @classmethod
-#     def get_xml_tags(c):
-#         return c.xml_tags.keys()
-
-#     @classmethod
-#     def get_uses(c):
-#         return ['capa_transform']
-
-#     def get_html(self):
-#         return self.xml_tags[self.tag](self.xml, self.value, self.status, self.msg)
-
-#     def __init__(self, system, xml, item_id = None, track_url=None, state=None, use = 'capa_input'):
-#         self.xml = xml
-#         self.tag = xml.tag
-#         if not state:
-#             state = {}
-#         if item_id:
-#             self.id = item_id
-#         if xml.get('id'):
-#             self.id = xml.get('id')
-#         if 'id' in state:
-#             self.id = state['id']
-#         self.system = system
-
-#         self.value = ''
-#         if 'value' in state:
-#             self.value = state['value']
-
-#         self.msg = ''
-#         if 'feedback' in state and 'message' in state['feedback']:
-#             self.msg = state['feedback']['message']
-
-#         self.status = 'unanswered'
-#         if 'status' in state:
-#             self.status = state['status']
+    @classmethod
+    def get_xml_tags(c):
+        return c.xml_tags.keys()
 
+    @classmethod
+    def get_uses(c):
+        return ['capa_input', 'capa_transform']
+
+    def get_html(self):
+        return self.xml_tags[self.tag](self.xml, self.value, self.status, self.system.render_template, self.msg)
 
 def register_render_function(fn, names=None, cls=SimpleInput):
     if names is None:
@@ -136,9 +107,6 @@ def register_render_function(fn, names=None, cls=SimpleInput):
         return fn
     return wrapped
 
-
-
-
 #-----------------------------------------------------------------------------
 
 @register_render_function
@@ -201,16 +169,20 @@ def choicegroup(element, value, status, render_template, msg=''):
     return etree.XML(html)
 
 @register_render_function
-def textline(element, value, state, render_template, msg=""):
+def textline(element, value, status, render_template, msg=""):
     '''
     Simple text line input, with optional size specification.
     '''
     if element.get('math') or element.get('dojs'):		# 'dojs' flag is temporary, for backwards compatibility with 8.02x
-        return SimpleInput.xml_tags['textline_dynamath'](element,value,state,msg)
+        return SimpleInput.xml_tags['textline_dynamath'](element,value,status,render_template,msg)
     eid=element.get('id')
+    if eid is None:
+        msg = 'textline has no id: it probably appears outside of a known response type'
+        msg += "\nSee problem XML source line %s" % getattr(element,'sourceline','<unavailable>')
+        raise Exception(msg)
     count = int(eid.split('_')[-2])-1 # HACK
     size = element.get('size')
-    context = {'id':eid, 'value':value, 'state':state, 'count':count, 'size': size, 'msg': msg}
+    context = {'id':eid, 'value':value, 'state':status, 'count':count, 'size': size, 'msg': msg}
     html = render_template("textinput.html", context)
     return etree.XML(html)
 

common/lib/capa/responsetypes.py

@@ -21,44 +21,252 @@ import abc
 
 # specific library imports
 from calc import evaluator, UndefinedVariable
-from util import contextualize_text
+from correctmap import CorrectMap
+from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	# uses Beautiful Soup!!! FIXME?
 
-log = logging.getLogger(__name__)
+log = logging.getLogger('mitx.' + __name__)
 
-def compare_with_tolerance(v1, v2, tol):
-    ''' Compare v1 to v2 with maximum tolerance tol
-    tol is relative if it ends in %; otherwise, it is absolute
+#-----------------------------------------------------------------------------
+# Exceptions
+
+class LoncapaProblemError(Exception):
     '''
-    relative = "%" in tol
-    if relative: 
-        tolerance_rel = evaluator(dict(),dict(),tol[:-1]) * 0.01
-        tolerance = tolerance_rel * max(abs(v1), abs(v2))
-    else: 
-        tolerance = evaluator(dict(),dict(),tol)
-    return abs(v1-v2) <= tolerance
+    Error in specification of a problem
+    '''
+    pass
+
+class ResponseError(Exception):
+    '''
+    Error for failure in processing a response
+    '''
+    pass
+
+class StudentInputError(Exception):
+    pass
+
+#-----------------------------------------------------------------------------
+#
+# Main base class for CAPA responsetypes
+
+class LoncapaResponse(object):
+    '''
+    Base class for CAPA responsetypes.  Each response type (ie a capa question,
+    which is part of a capa problem) is represented as a subclass,
+    which should provide the following methods:
+
+      - get_score           : evaluate the given student answers, and return a CorrectMap
+      - get_answers         : provide a dict of the expected answers for this problem
+      
+    Each subclass must also define the following attributes:
+
+      - response_tag         : xhtml tag identifying this response (used in auto-registering)
 
-class GenericResponse(object):
+    In addition, these methods are optional:
+
+      - get_max_score        : if defined, this is called to obtain the maximum score possible for this question
+      - setup_response       : find and note the answer input field IDs for the response; called by __init__
+      - check_hint_condition : check to see if the student's answers satisfy a particular condition for a hint to be displayed
+      - render_html          : render this Response as HTML (must return XHTML compliant string)
+      - __unicode__          : unicode representation of this Response
+
+    Each response type may also specify the following attributes:
+
+      - max_inputfields      : (int) maximum number of answer input fields (checked in __init__ if not None)
+      - allowed_inputfields  : list of allowed input fields (each a string) for this Response
+      - required_attributes  : list of required attributes (each a string) on the main response XML stanza
+      - hint_tag             : xhtml tag identifying hint associated with this response inside hintgroup
+
+    '''
     __metaclass__=abc.ABCMeta # abc = Abstract Base Class
 
+    response_tag = None
+    hint_tag = None
+
+    max_inputfields = None
+    allowed_inputfields = []
+    required_attributes = []
+    
+    def __init__(self, xml, inputfields, context, system=None):
+        '''
+        Init is passed the following arguments:
+
+          - xml         : ElementTree of this Response
+          - inputfields : ordered list of ElementTrees for each input entry field in this Response
+          - context     : script processor context
+          - system      : I4xSystem instance which provides OS, rendering, and user context 
+
+        '''
+        self.xml = xml
+        self.inputfields = inputfields
+        self.context = context
+        self.system = system
+
+        for abox in inputfields:
+            if abox.tag not in self.allowed_inputfields:
+                msg = "%s: cannot have input field %s" % (unicode(self),abox.tag)
+                msg += "\nSee XML source line %s" % getattr(xml,'sourceline','<unavailable>')
+                raise LoncapaProblemError(msg)
+
+        if self.max_inputfields and len(inputfields)>self.max_inputfields:
+            msg = "%s: cannot have more than %s input fields" % (unicode(self),self.max_inputfields)
+            msg += "\nSee XML source line %s" % getattr(xml,'sourceline','<unavailable>')
+            raise LoncapaProblemError(msg)
+
+        for prop in self.required_attributes:
+            if not xml.get(prop):
+                msg = "Error in problem specification: %s missing required attribute %s" % (unicode(self),prop)
+                msg += "\nSee XML source line %s" % getattr(xml,'sourceline','<unavailable>')
+                raise LoncapaProblemError(msg)
+
+        self.answer_ids = [x.get('id') for x in self.inputfields]	# ordered list of answer_id values for this response
+        if self.max_inputfields==1:
+            self.answer_id = self.answer_ids[0]		# for convenience
+
+        self.default_answer_map = {}			# dict for default answer map (provided in input elements)
+        for entry in self.inputfields:
+            answer = entry.get('correct_answer')        
+            if answer:
+                self.default_answer_map[entry.get('id')] = contextualize_text(answer, self.context)
+
+        if hasattr(self,'setup_response'):
+            self.setup_response()
+
+    def render_html(self,renderer):
+        '''
+        Return XHTML Element tree representation of this Response.
+
+        Arguments:
+
+          - renderer : procedure which produces HTML given an ElementTree
+        '''
+        tree = etree.Element('span')			# render ourself as a <span> + our content
+        for item in self.xml:
+            item_xhtml = renderer(item)			# call provided procedure to do the rendering
+            if item_xhtml is not None: tree.append(item_xhtml)
+        tree.tail = self.xml.tail
+        return tree
+
+    def evaluate_answers(self,student_answers,old_cmap):
+        '''
+        Called by capa_problem.LoncapaProblem to evaluate student answers, and to
+        generate hints (if any).
+
+        Returns the new CorrectMap, with (correctness,msg,hint,hintmode) for each answer_id.
+        '''
+        new_cmap = self.get_score(student_answers)
+        self.get_hints(student_answers, new_cmap, old_cmap)
+        # log.debug('new_cmap = %s' % new_cmap)
+        return new_cmap
+
+    def get_hints(self, student_answers, new_cmap, old_cmap):
+        '''
+        Generate adaptive hints for this problem based on student answers, the old CorrectMap,
+        and the new CorrectMap produced by get_score.
+
+        Does not return anything.
+
+        Modifies new_cmap, by adding hints to answer_id entries as appropriate.
+        '''
+        hintgroup = self.xml.find('hintgroup')
+        if hintgroup is None: return
+
+        # hint specified by function?
+        hintfn = hintgroup.get('hintfn')
+        if hintfn:
+            '''
+            Hint is determined by a function defined in the <script> context; evaluate that function to obtain
+            list of hint, hintmode for each answer_id.
+
+            The function should take arguments (answer_ids, student_answers, new_cmap, old_cmap)
+            and it should modify new_cmap as appropriate.
+
+            We may extend this in the future to add another argument which provides a callback procedure
+            to a social hint generation system.
+
+            '''
+            if not hintfn in self.context:
+                msg = 'missing specified hint function %s in script context' % hintfn
+                msg += "\nSee XML source line %s" % getattr(self.xml,'sourceline','<unavailable>')
+                raise LoncapaProblemError(msg)
+
+            try:
+                self.context[hintfn](self.answer_ids, student_answers, new_cmap, old_cmap)
+            except Exception, err:
+                msg = 'Error %s in evaluating hint function %s' % (err,hintfn)
+                msg += "\nSee XML source line %s" % getattr(self.xml,'sourceline','<unavailable>')
+                raise ResponseError(msg)
+            return
+
+        # hint specified by conditions and text dependent on conditions (a-la Loncapa design)
+        # see http://help.loncapa.org/cgi-bin/fom?file=291
+        #
+        # Example:
+        #
+        # <formularesponse samples="x@-5:5#11" id="11" answer="$answer">
+        #   <textline size="25" />
+        #   <hintgroup>
+        #     <formulahint samples="x@-5:5#11" answer="$wrongans" name="inversegrad"></formulahint>
+        #     <hintpart on="inversegrad">
+        #       <text>You have inverted the slope in the question.  The slope is
+        #             (y2-y1)/(x2 - x1) you have the slope as (x2-x1)/(y2-y1).</text>
+        #     </hintpart>
+        #   </hintgroup>     
+        # </formularesponse>
+
+        if self.hint_tag is not None and hintgroup.find(self.hint_tag) is not None and hasattr(self,'check_hint_condition'):
+            rephints = hintgroup.findall(self.hint_tag)
+            hints_to_show = self.check_hint_condition(rephints,student_answers)
+            hintmode = hintgroup.get('mode','always')	# can be 'on_request' or 'always' (default)
+            for hintpart in hintgroup.findall('hintpart'):
+                if hintpart.get('on') in hints_to_show:
+                    hint_text = hintpart.find('text').text
+                    aid = self.answer_ids[-1]		# make the hint appear after the last answer box in this response
+                    new_cmap.set_hint_and_mode(aid,hint_text,hintmode)
+            log.debug('after hint: new_cmap = %s' % new_cmap)
+
     @abc.abstractmethod
     def get_score(self, student_answers):
+        '''
+        Return a CorrectMap for the answers expected vs given.  This includes
+        (correctness, npoints, msg) for each answer_id.
+
+        Arguments:
+
+         - student_answers : dict of (answer_id,answer) where answer = student input (string)
+         - old_cmap        : previous CorrectMap (may be empty); useful for analyzing or recording history of responses
+
+        '''
         pass
 
     @abc.abstractmethod
     def get_answers(self):
+        '''
+        Return a dict of (answer_id,answer_text) for each answer for this question.
+        '''
         pass
 
-    #not an abstract method because plenty of responses will not want to preprocess anything, and we should not require that they override this method.
-    def preprocess_response(self):
+    def check_hint_condition(self,hxml_set,student_answers):
+        '''
+        Return a list of hints to show.
+
+          - hxml_set        : list of Element trees, each specifying a condition to be satisfied for a named hint condition
+          - student_answers : dict of student answers
+
+        Returns a list of names of hint conditions which were satisfied.  Those are used to determine which hints are displayed.
+        '''
         pass
 
-#Every response type needs methods "get_score" and "get_answers"     
+    def setup_response(self):
+        pass
+
+    def __unicode__(self):
+        return u'LoncapaProblem Response %s' % self.xml.tag
 
 #-----------------------------------------------------------------------------
 
-class MultipleChoiceResponse(GenericResponse):
+class MultipleChoiceResponse(LoncapaResponse):
     # TODO: handle direction and randomize
     snippets = [{'snippet': '''<multiplechoiceresponse direction="vertical" randomize="yes">
      <choicegroup type="MultipleChoice">
@@ -69,30 +277,20 @@ class MultipleChoiceResponse(GenericResponse):
      </choicegroup>
     </multiplechoiceresponse>
     '''}]
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
-        self.correct_choices = xml.xpath('//*[@id=$id]//choice[@correct="true"]',
-                                    id=xml.get('id'))
-        self.correct_choices = [choice.get('name') for choice in self.correct_choices]
-        self.context = context
 
-        self.answer_field = xml.find('choicegroup')	# assumes only ONE choicegroup within this response
-        self.answer_id = xml.xpath('//*[@id=$id]//choicegroup/@id',
-                                   id=xml.get('id'))
-        if not len(self.answer_id) == 1:
-            raise Exception("should have exactly one choice group per multiplechoicceresponse")
-        self.answer_id=self.answer_id[0]
+    response_tag = 'multiplechoiceresponse'
+    max_inputfields = 1
+    allowed_inputfields = ['choicegroup']
 
-    def get_score(self, student_answers):
-        if self.answer_id in student_answers and student_answers[self.answer_id] in self.correct_choices:
-            return {self.answer_id:'correct'}
-        else:
-            return {self.answer_id:'incorrect'}
+    def setup_response(self):
+        self.mc_setup_response()	# call secondary setup for MultipleChoice questions, to set name attributes
 
-    def get_answers(self):
-        return {self.answer_id:self.correct_choices}
+        # define correct choices (after calling secondary setup)
+        xml = self.xml
+        cxml = xml.xpath('//*[@id=$id]//choice[@correct="true"]',id=xml.get('id'))
+        self.correct_choices = [choice.get('name') for choice in cxml]
 
-    def preprocess_response(self):
+    def mc_setup_response(self):
         '''
         Initialize name attributes in <choice> stanzas in the <choicegroup> in this response.
         '''
@@ -108,8 +306,24 @@ class MultipleChoiceResponse(GenericResponse):
                 else:
                     choice.set("name", "choice_"+choice.get("name"))
 
+    def get_score(self, student_answers):
+        '''
+        grade student response.
+        '''
+        # log.debug('%s: student_answers=%s, correct_choices=%s' % (unicode(self),student_answers,self.correct_choices))
+        if self.answer_id in student_answers and student_answers[self.answer_id] in self.correct_choices:
+            return CorrectMap(self.answer_id,'correct')
+        else:
+            return CorrectMap(self.answer_id,'incorrect')
+
+    def get_answers(self):
+        return {self.answer_id:self.correct_choices}
+
 class TrueFalseResponse(MultipleChoiceResponse):
-    def preprocess_response(self):
+
+    response_tag = 'truefalseresponse'
+
+    def mc_setup_response(self):
         i=0
         for response in self.xml.xpath("choicegroup"):
             response.set("type", "TrueFalse")
@@ -125,13 +339,13 @@ class TrueFalseResponse(MultipleChoiceResponse):
         answers = set(student_answers.get(self.answer_id, []))
         
         if correct == answers:
-            return { self.answer_id : 'correct'}
+            return CorrectMap( self.answer_id , 'correct')
         
-        return {self.answer_id : 'incorrect'}
+        return CorrectMap(self.answer_id ,'incorrect')
 
 #-----------------------------------------------------------------------------
 
-class OptionResponse(GenericResponse):
+class OptionResponse(LoncapaResponse):
     '''
     TODO: handle direction and randomize
     '''
@@ -140,34 +354,42 @@ class OptionResponse(GenericResponse):
         <optioninput options="('Up','Down')" correct="Down"><text>The location of the earth</text></optioninput>
     </optionresponse>'''}]
 
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
-        self.answer_fields = xml.findall('optioninput')
-        self.context = context
+    response_tag = 'optionresponse'
+    hint_tag = 'optionhint'
+    allowed_inputfields = ['optioninput']
+
+    def setup_response(self):
+        self.answer_fields = self.inputfields
 
     def get_score(self, student_answers):
-        cmap = {}
+        # log.debug('%s: student_answers=%s' % (unicode(self),student_answers))
+        cmap = CorrectMap()
         amap = self.get_answers()
         for aid in amap:
             if aid in student_answers and student_answers[aid]==amap[aid]:
-                cmap[aid] = 'correct'
+                cmap.set(aid,'correct')
             else:
-                cmap[aid] = 'incorrect'
+                cmap.set(aid,'incorrect')
         return cmap
 
     def get_answers(self):
         amap = dict([(af.get('id'),af.get('correct')) for af in self.answer_fields])
+        # log.debug('%s: expected answers=%s' % (unicode(self),amap))
         return amap
 
 #-----------------------------------------------------------------------------
 
-class NumericalResponse(GenericResponse):
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
-	if not xml.get('answer'):
-            msg = "Error in problem specification: numericalresponse missing required answer attribute\n"
-            msg += "See XML source line %s" % getattr(xml,'sourceline','<unavailable>')
-	    raise Exception,msg
+class NumericalResponse(LoncapaResponse):
+
+    response_tag = 'numericalresponse'
+    hint_tag = 'numericalhint'
+    allowed_inputfields = ['textline']
+    required_attributes = ['answer']
+    max_inputfields = 1
+
+    def setup_response(self):
+        xml = self.xml
+        context = self.context
         self.correct_answer = contextualize_text(xml.get('answer'), context)
         try:
             self.tolerance_xml = xml.xpath('//*[@id=$id]//responseparam[@type="tolerance"]/@default',
@@ -182,7 +404,7 @@ class NumericalResponse(GenericResponse):
             self.answer_id = None
 
     def get_score(self, student_answers):
-        ''' Display HTML for a numeric response '''
+        '''Grade a numeric response '''
         student_answer = student_answers[self.answer_id]
         try:
             correct = compare_with_tolerance (evaluator(dict(),dict(),student_answer), complex(self.correct_answer), self.tolerance)
@@ -193,16 +415,54 @@ class NumericalResponse(GenericResponse):
             raise StudentInputError('Invalid input -- please use a number only')
 
         if correct:
-            return {self.answer_id:'correct'}
+            return CorrectMap(self.answer_id,'correct')
         else:
-            return {self.answer_id:'incorrect'}
+            return CorrectMap(self.answer_id,'incorrect')
+
+    # TODO: add check_hint_condition(self,hxml_set,student_answers)
 
     def get_answers(self):
         return {self.answer_id:self.correct_answer}
 
 #-----------------------------------------------------------------------------
 
-class CustomResponse(GenericResponse):
+class StringResponse(LoncapaResponse):
+
+    response_tag = 'stringresponse'
+    hint_tag = 'stringhint'
+    allowed_inputfields = ['textline']
+    required_attributes = ['answer']
+    max_inputfields = 1
+
+    def setup_response(self):
+        self.correct_answer = contextualize_text(self.xml.get('answer'), self.context).strip()
+
+    def get_score(self, student_answers):
+        '''Grade a string response '''
+        student_answer = student_answers[self.answer_id].strip()
+        correct = self.check_string(self.correct_answer,student_answer)
+        return CorrectMap(self.answer_id,'correct' if correct else 'incorrect')
+
+    def check_string(self,expected,given):
+        if self.xml.get('type')=='ci': return given.lower() == expected.lower()
+        return given == expected
+
+    def check_hint_condition(self,hxml_set,student_answers):
+        given = student_answers[self.answer_id].strip()
+        hints_to_show = []
+        for hxml in hxml_set:
+            name = hxml.get('name')
+            correct_answer = contextualize_text(hxml.get('answer'),self.context).strip()
+            if self.check_string(correct_answer,given): hints_to_show.append(name)
+        log.debug('hints_to_show = %s' % hints_to_show)
+        return hints_to_show
+
+    def get_answers(self):
+        return {self.answer_id:self.correct_answer}
+
+#-----------------------------------------------------------------------------
+
+class CustomResponse(LoncapaResponse):
     '''
     Custom response.  The python code to be run should be in <answer>...</answer>
     or in a <script>...</script>
@@ -241,16 +501,11 @@ def sympy_check2():
     <responseparam description="Numerical Tolerance" type="tolerance" default="0.00001" name="tol"/>
   </customresponse>'''}]
 
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
-        self.system = system
-        ## CRITICAL TODO: Should cover all entrytypes
-        ## NOTE: xpath will look at root of XML tree, not just 
-        ## what's in xml. @id=id keeps us in the right customresponse. 
-        self.answer_ids = xml.xpath('//*[@id=$id]//textline/@id',
-                                    id=xml.get('id'))
-        self.answer_ids += [x.get('id') for x in xml.findall('textbox')]	# also allow textbox inputs
-        self.context = context
+    response_tag = 'customresponse'
+    allowed_inputfields = ['textline','textbox']
+
+    def setup_response(self):
+        xml = self.xml
 
         # if <customresponse> has an "expect" (or "answer") attribute then save that
         self.expect = xml.get('expect') or xml.get('answer')
@@ -271,15 +526,17 @@ def sympy_check2():
             cfn = xml.get('cfn')
             if cfn:
                 log.debug("cfn = %s" % cfn)
-                if cfn in context:
-                    self.code = context[cfn]
+                if cfn in self.context:
+                    self.code = self.context[cfn]
                 else:
-                    print "can't find cfn in context = ",context
+                    msg = "%s: can't find cfn %s in context" % (unicode(self),cfn)
+                    msg += "\nSee XML source line %s" % getattr(self.xml,'sourceline','<unavailable>')
+                    raise LoncapaProblemError(msg)
 
         if not self.code:
             if answer is None:
                 # raise Exception,"[courseware.capa.responsetypes.customresponse] missing code checking script! id=%s" % self.myid
-                print "[courseware.capa.responsetypes.customresponse] missing code checking script! id=%s" % self.myid
+                log.error("[courseware.capa.responsetypes.customresponse] missing code checking script! id=%s" % self.myid)
                 self.code = ''
             else:
                 answer_src = answer.get('src')
@@ -294,6 +551,8 @@ def sympy_check2():
         of each key removed (the string before the first "_").
         '''
 
+        log.debug('%s: student_answers=%s' % (unicode(self),student_answers))
+
         idset = sorted(self.answer_ids)				# ordered list of answer id's
         try:
             submission = [student_answers[k] for k in idset]	# ordered list of answers
@@ -301,7 +560,7 @@ def sympy_check2():
             msg = '[courseware.capa.responsetypes.customresponse] error getting student answer from %s' % student_answers
             msg += '\n idset = %s, error = %s' % (idset,err)
             log.error(msg)
-            raise Exception,msg
+            raise Exception(msg)
 
         # global variable in context which holds the Presentation MathML from dynamic math input
         dynamath = [ student_answers.get(k+'_dynamath',None) for k in idset ]	# ordered list of dynamath responses
@@ -364,7 +623,7 @@ def sympy_check2():
                 log.error("oops in customresponse (cfn) error %s" % err)
                 # print "context = ",self.context
                 log.error(traceback.format_exc())
-                raise Exception,"oops in customresponse (cfn) error %s" % err
+                raise Exception("oops in customresponse (cfn) error %s" % err)
             log.debug("[courseware.capa.responsetypes.customresponse.get_score] ret = %s" % ret)
             if type(ret)==dict:
                 correct = ['correct']*len(idset) if ret['ok'] else ['incorrect']*len(idset)
@@ -386,28 +645,26 @@ def sympy_check2():
                 correct = ['correct']*len(idset) if ret else ['incorrect']*len(idset)
 
         # build map giving "correct"ness of the answer(s)
-        #correct_map = dict(zip(idset, self.context['correct']))
-        correct_map = {}
+        correct_map = CorrectMap()
         for k in range(len(idset)):
-            correct_map[idset[k]] = correct[k]
-            correct_map['msg_%s' % idset[k]] = messages[k]
+            correct_map.set(idset[k], correct[k], msg=messages[k])
         return correct_map
 
     def get_answers(self):
         '''
         Give correct answer expected for this response.
 
-        capa_problem handles correct_answers from entry objects like textline, and that
-        is what should be used when this response has multiple entry objects.
+        use default_answer_map from entry elements (eg textline),
+        when this response has multiple entry objects.
 
         but for simplicity, if an "expect" attribute was given by the content author
-        ie <customresponse expect="foo" ...> then return it now.
+        ie <customresponse expect="foo" ...> then that.
         '''
         if len(self.answer_ids)>1:
-            return {}
+            return self.default_answer_map
         if self.expect:
             return {self.answer_ids[0] : self.expect}
-        return {}
+        return self.default_answer_map
 
 #-----------------------------------------------------------------------------
 
@@ -425,16 +682,18 @@ class SymbolicResponse(CustomResponse):
       Your input should be typed in as a list of lists, eg <tt>[[1,2],[3,4]]</tt>.
       </text>
     </problem>'''}]
-    def __init__(self, xml, context, system=None):
-        xml.set('cfn','symmath_check')
-        code = "from symmath import *"
-        exec code in context,context
-        CustomResponse.__init__(self,xml,context,system)
 
+    response_tag = 'symbolicresponse'
+
+    def setup_response(self):
+        self.xml.set('cfn','symmath_check')
+        code = "from symmath import *"
+        exec code in self.context,self.context
+        CustomResponse.setup_response(self)
 
 #-----------------------------------------------------------------------------
 
-class ExternalResponse(GenericResponse):
+class ExternalResponse(LoncapaResponse):
     '''
     Grade the students input using an external server.
     
@@ -480,15 +739,14 @@ main()
     </answer>
   </externalresponse>'''}]
 
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
+    response_tag = 'externalresponse'
+    allowed_inputfields = ['textline','textbox']
+
+    def setup_response(self):
+        xml = self.xml
         self.url = xml.get('url') or "http://eecs1.mit.edu:8889/pyloncapa"	# FIXME - hardcoded URL
-        self.answer_ids = xml.xpath('//*[@id=$id]//textbox/@id|//*[@id=$id]//textline/@id',
-                                    id=xml.get('id'))
-        self.context = context
-        answer = xml.xpath('//*[@id=$id]//answer',
-                           id=xml.get('id'))[0]
 
+        answer = xml.xpath('//*[@id=$id]//answer',id=xml.get('id'))[0]	# FIXME - catch errors
         answer_src = answer.get('src')
         if answer_src is not None:
             self.code = self.system.filesystem.open('src/'+answer_src).read()
@@ -519,28 +777,30 @@ main()
         except Exception,err:
             msg = 'Error %s - cannot connect to external server url=%s' % (err,self.url)
             log.error(msg)
-            raise Exception, msg
+            raise Exception(msg)
 
         if self.system.DEBUG: log.info('response = %s' % r.text)
 
         if (not r.text ) or (not r.text.strip()):
-            raise Exception,'Error: no response from external server url=%s' % self.url
+            raise Exception('Error: no response from external server url=%s' % self.url)
 
         try:
             rxml = etree.fromstring(r.text)         # response is XML; prase it
         except Exception,err:
             msg = 'Error %s - cannot parse response from external server r.text=%s' % (err,r.text)
             log.error(msg)
-            raise Exception, msg
+            raise Exception(msg)
 
         return rxml
 
     def get_score(self, student_answers):
+        idset = sorted(self.answer_ids)
+        cmap = CorrectMap()
         try:
-            submission = [student_answers[k] for k in sorted(self.answer_ids)]
+            submission = [student_answers[k] for k in idset]
         except Exception,err:
             log.error('Error %s: cannot get student answer for %s; student_answers=%s' % (err,self.answer_ids,student_answers))
-            raise Exception,err
+            raise Exception(err)
 
         self.context.update({'submission':submission})
 
@@ -551,9 +811,9 @@ main()
         except Exception, err:
             log.error('Error %s' % err)
             if self.system.DEBUG:
-                correct_map = dict(zip(sorted(self.answer_ids), ['incorrect'] * len(self.answer_ids) ))
-                correct_map['msg_%s' % self.answer_ids[0]] = '<font color="red" size="+2">%s</font>' % str(err).replace('<','&lt;')
-                return correct_map
+                cmap.set_dict(dict(zip(sorted(self.answer_ids), ['incorrect'] * len(idset) )))
+                cmap.set_property(self.answer_ids[0],'msg','<font color="red" size="+2">%s</font>' % str(err).replace('<','&lt;'))
+                return cmap
 
         ad = rxml.find('awarddetail').text
         admap = {'EXACT_ANS':'correct',         # TODO: handle other loncapa responses
@@ -563,13 +823,13 @@ main()
         if ad in admap:
             self.context['correct'][0] = admap[ad]
 
-        # self.context['correct'] = ['correct','correct']
-        correct_map = dict(zip(sorted(self.answer_ids), self.context['correct']))
-        
-        # store message in correct_map
-        correct_map['msg_%s' % self.answer_ids[0]] = rxml.find('message').text.replace('&nbsp;','&#160;')  
+        # create CorrectMap
+        for key in idset:
+            idx = idset.index(key)
+            msg = rxml.find('message').text.replace('&nbsp;','&#160;') if idx==0 else None
+            cmap.set(key, self.context['correct'][idx], msg=msg)
 
-        return  correct_map
+        return cmap
 
     def get_answers(self):
         '''
@@ -587,15 +847,13 @@ main()
             
         if not (len(exans)==len(self.answer_ids)):
             log.error('Expected %d answers from external server, only got %d!' % (len(self.answer_ids),len(exans)))
-            raise Exception,'Short response from external server'
+            raise Exception('Short response from external server')
         return dict(zip(self.answer_ids,exans))
 
-class StudentInputError(Exception):
-    pass
 
 #-----------------------------------------------------------------------------
 
-class FormulaResponse(GenericResponse):
+class FormulaResponse(LoncapaResponse):
     '''
     Checking of symbolic math response using numerical sampling.
     '''
@@ -617,8 +875,15 @@ class FormulaResponse(GenericResponse):
 
     </problem>'''}]
 
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
+    response_tag = 'formularesponse'
+    hint_tag = 'formulahint'
+    allowed_inputfields = ['textline']
+    required_attributes = ['answer']
+    max_inputfields = 1
+
+    def setup_response(self):
+        xml = self.xml
+        context = self.context
         self.correct_answer = contextualize_text(xml.get('answer'), context)
         self.samples = contextualize_text(xml.get('samples'), context)
         try:
@@ -626,16 +891,8 @@ class FormulaResponse(GenericResponse):
                                            id=xml.get('id'))[0]
             self.tolerance = contextualize_text(self.tolerance_xml, context)
         except Exception:
-            self.tolerance = 0
+            self.tolerance = '0.00001'
 
-        try:
-            self.answer_id = xml.xpath('//*[@id=$id]//textline/@id',
-                                       id=xml.get('id'))[0]
-        except Exception:
-            self.answer_id = None
-            raise Exception, "[courseware.capa.responsetypes.FormulaResponse] Error: missing answer_id!!"
-
-        self.context = context
         ts = xml.get('type')
         if ts is None:
             typeslist = []
@@ -648,12 +905,16 @@ class FormulaResponse(GenericResponse):
         else: # Default
             self.case_sensitive = False
 
-
     def get_score(self, student_answers):
-        variables=self.samples.split('@')[0].split(',')
-        numsamples=int(self.samples.split('@')[1].split('#')[1])
+        given = student_answers[self.answer_id]
+        correctness = self.check_formula(self.correct_answer, given, self.samples)
+        return CorrectMap(self.answer_id, correctness)
+
+    def check_formula(self,expected, given, samples):
+        variables=samples.split('@')[0].split(',')
+        numsamples=int(samples.split('@')[1].split('#')[1])
         sranges=zip(*map(lambda x:map(float, x.split(",")), 
-                         self.samples.split('@')[1].split('#')[0].split(':')))
+                         samples.split('@')[1].split('#')[0].split(':')))
 
         ranges=dict(zip(variables, sranges))
         for i in range(numsamples):
@@ -663,23 +924,26 @@ class FormulaResponse(GenericResponse):
                 value = random.uniform(*ranges[var])
                 instructor_variables[str(var)] = value
                 student_variables[str(var)] = value
-            instructor_result = evaluator(instructor_variables,dict(),self.correct_answer, cs = self.case_sensitive)
+            #log.debug('formula: instructor_vars=%s, expected=%s' % (instructor_variables,expected))
+            instructor_result = evaluator(instructor_variables,dict(),expected, cs = self.case_sensitive)
             try: 
-                #print student_variables,dict(),student_answers[self.answer_id]
-                student_result = evaluator(student_variables,dict(),
-                                           student_answers[self.answer_id], 
+                #log.debug('formula: student_vars=%s, given=%s' % (student_variables,given))
+                student_result = evaluator(student_variables,
+                                           dict(),
+                                           given, 
                                            cs = self.case_sensitive)
             except UndefinedVariable as uv:
+                log.debug('formularesponse: undefined variable in given=%s' % given)
                 raise StudentInputError(uv.message+" not permitted in answer")
-            except:
+            except Exception, err:
                 #traceback.print_exc()
+                log.debug('formularesponse: error %s in formula' % err)
                 raise StudentInputError("Error in formula")
             if numpy.isnan(student_result) or numpy.isinf(student_result):
-                return {self.answer_id:"incorrect"}
+                return "incorrect"
             if not compare_with_tolerance(student_result, instructor_result, self.tolerance):
-                return {self.answer_id:"incorrect"}
- 
-        return {self.answer_id:"correct"}
+                return "incorrect"
+        return "correct"
 
     def strip_dict(self, d):
         ''' Takes a dict. Returns an identical dict, with all non-word
@@ -691,19 +955,35 @@ class FormulaResponse(GenericResponse):
                     isinstance(d[k], numbers.Number)])
         return d
 
+    def check_hint_condition(self,hxml_set,student_answers):
+        given = student_answers[self.answer_id]
+        hints_to_show = []
+        for hxml in hxml_set:
+            samples = hxml.get('samples')
+            name = hxml.get('name')
+            correct_answer = contextualize_text(hxml.get('answer'),self.context)
+            try:
+                correctness = self.check_formula(correct_answer, given, samples)
+            except Exception:
+                correctness = 'incorrect'
+            if correctness=='correct':
+                hints_to_show.append(name)
+        log.debug('hints_to_show = %s' % hints_to_show)
+        return hints_to_show
+
     def get_answers(self):
         return {self.answer_id:self.correct_answer}
 
 #-----------------------------------------------------------------------------
 
-class SchematicResponse(GenericResponse):
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
-        self.answer_ids = xml.xpath('//*[@id=$id]//schematic/@id',
-                                    id=xml.get('id'))
-        self.context = context
-        answer = xml.xpath('//*[@id=$id]//answer',
-                           id=xml.get('id'))[0]
+class SchematicResponse(LoncapaResponse):
+
+    response_tag = 'schematicresponse'
+    allowed_inputfields = ['schematic']
+
+    def setup_response(self):
+        xml = self.xml
+        answer = xml.xpath('//*[@id=$id]//answer', id=xml.get('id'))[0]
         answer_src = answer.get('src')
         if answer_src is not None:
             self.code = self.system.filestore.open('src/'+answer_src).read() # Untested; never used
@@ -715,16 +995,17 @@ class SchematicResponse(GenericResponse):
         submission = [json.loads(student_answers[k]) for k in sorted(self.answer_ids)]
         self.context.update({'submission':submission})
         exec self.code in global_context, self.context
-        return  zip(sorted(self.answer_ids), self.context['correct'])
+        cmap = CorrectMap()
+        cmap.set_dict(dict(zip(sorted(self.answer_ids), self.context['correct'])))
+        return cmap
 
     def get_answers(self):
-        # Since this is explicitly specified in the problem, this will 
-        # be handled by capa_problem
-        return {}
+        # use answers provided in input elements
+        return self.default_answer_map
 
 #-----------------------------------------------------------------------------
 
-class ImageResponse(GenericResponse):
+class ImageResponse(LoncapaResponse):
     """
     Handle student response for image input: the input is a click on an image,
     which produces an [x,y] coordinate pair.  The click is correct if it falls
@@ -740,14 +1021,15 @@ class ImageResponse(GenericResponse):
       <imageinput src="image2.jpg" width="210" height="130" rectangle="(12,12)-(40,60)" />
     </imageresponse>'''}]
 
-    def __init__(self, xml, context, system=None):
-        self.xml = xml
-        self.context = context
-        self.ielements = xml.findall('imageinput')
+    response_tag = 'imageresponse'
+    allowed_inputfields = ['imageinput']
+
+    def setup_response(self):
+        self.ielements = self.inputfields
         self.answer_ids = [ie.get('id')  for ie in self.ielements]
 
     def get_score(self, student_answers):
-        correct_map = {}
+        correct_map = CorrectMap()
         expectedset = self.get_answers()
 
         for aid in self.answer_ids:	# loop through IDs of <imageinput> fields in our stanza
@@ -759,21 +1041,28 @@ class ImageResponse(GenericResponse):
             if not m:
                 msg = 'Error in problem specification! cannot parse rectangle in %s' % (etree.tostring(self.ielements[aid],
                                                                                                        pretty_print=True))
-                raise Exception,'[capamodule.capa.responsetypes.imageinput] '+msg
+                raise Exception('[capamodule.capa.responsetypes.imageinput] '+msg)
             (llx,lly,urx,ury) = [int(x) for x in m.groups()]
                 
             # parse given answer
             m = re.match('\[([0-9]+),([0-9]+)]',given.strip().replace(' ',''))
             if not m:
-                raise Exception,'[capamodule.capa.responsetypes.imageinput] error grading %s (input=%s)' % (aid,given)
+                raise Exception('[capamodule.capa.responsetypes.imageinput] error grading %s (input=%s)' % (aid,given))
             (gx,gy) = [int(x) for x in m.groups()]
             
             # answer is correct if (x,y) is within the specified rectangle
             if (llx <= gx <= urx) and (lly <= gy <= ury):
-                correct_map[aid] = 'correct'
+                correct_map.set(aid, 'correct')
             else:
-                correct_map[aid] = 'incorrect'
+                correct_map.set(aid, 'incorrect')
         return correct_map
 
     def get_answers(self):
         return dict([(ie.get('id'),ie.get('rectangle')) for ie in self.ielements])
+
+#-----------------------------------------------------------------------------
+# TEMPORARY: List of all response subclasses
+# FIXME: To be replaced by auto-registration
+
+__all__ = [ NumericalResponse, FormulaResponse, CustomResponse, SchematicResponse, MultipleChoiceResponse, TrueFalseResponse, ExternalResponse, ImageResponse, OptionResponse, SymbolicResponse, StringResponse ]
+

common/lib/capa/templates/textinput_dynamath.html

@@ -2,15 +2,12 @@
 ### version of textline.html which does dynammic math
 ###
 <section class="text-input-dynamath">
-<table style="display:inline; vertical-align:middle;"><tr><td>
- <input type="text" name="input_${id}" id="input_${id}" value="${value}" 
-	 % if size:
-	 size="${size}"
-	 % endif 
-	 onkeyup="DoUpdateMath('${id}')"
-	 />
-   </td><td>
-
+  <table style="display:inline; vertical-align:middle;">
+    <tr>
+      <td>
+        <input type="text" name="input_${id}" id="input_${id}" value="${value}" class="math" size="${size if size else ''}" />
+      </td>
+      <td>
         <span id="answer_${id}"></span>
 
         % if state == 'unsubmitted':
@@ -22,29 +19,18 @@
         % elif state == 'incomplete':
         <span class="incorrect" id="status_${id}"></span>
         % endif
-
-   </td></tr><tr><td>
+      </td>
+    </tr>
+    <tr>
+      <td>
         <span id="display_${id}">`{::}`</span>
-    </td><td>
+      </td>
+      <td>
         <textarea style="display:none" id="input_${id}_dynamath" name="input_${id}_dynamath">  </textarea>
-    </td></tr>
+      </td>
+    </tr>
   </table>
-##   
-## javascript for dynamic math: add this math element to the MathJax rendering queue
-## also adds to global jaxset js array
-##   
-<script type="text/javascript">
-         MathJax.Hub.queue.Push(function () {
-	      math = MathJax.Hub.getAllJax("display_${id}")[0];
-	      if (math){
-		  jaxset["${id}"] = math;
-		  math.Text(document.getElementById("input_${id}").value);
-                  // UpdateMathML(math,"${id}");
-	      }
-	  });
-</script>
   % if msg:
-    <br/>
-    <span class="debug">${msg|n}</span>
+      <span class="message">${msg|n}</span>
   % endif
 </section>

common/lib/capa/util.py

+from calc import evaluator, UndefinedVariable
+
+#-----------------------------------------------------------------------------
+#
+# Utility functions used in CAPA responsetypes
+
+def compare_with_tolerance(v1, v2, tol):
+    ''' Compare v1 to v2 with maximum tolerance tol
+    tol is relative if it ends in %; otherwise, it is absolute
+
+     - v1    :  student result (number)
+     - v2    :  instructor result (number)
+     - tol   :  tolerance (string or number)
+
+    '''
+    relative = tol.endswith('%')
+    if relative: 
+        tolerance_rel = evaluator(dict(),dict(),tol[:-1]) * 0.01
+        tolerance = tolerance_rel * max(abs(v1), abs(v2))
+    else: 
+        tolerance = evaluator(dict(),dict(),tol)
+    return abs(v1-v2) <= tolerance
+
 def contextualize_text(text, context): # private
     ''' Takes a string with variables. E.g. $a+$b. 
     Does a substitution of those variables from the context '''

common/lib/xmodule/capa_module.py

@@ -13,6 +13,7 @@ from lxml import etree
 from x_module import XModule, XModuleDescriptor
 from capa.capa_problem import LoncapaProblem
 from capa.responsetypes import StudentInputError
+
 log = logging.getLogger("mitx.courseware")
 
 #-----------------------------------------------------------------------------
@@ -280,6 +281,7 @@ class Module(XModule):
 
     def answer_available(self):
         ''' Is the user allowed to see an answer?
+        TODO: simplify. 
         '''
         if self.show_answer == '':
             return False
@@ -365,18 +367,17 @@ class Module(XModule):
         self.attempts = self.attempts + 1
         self.lcp.done=True
 
-        success = 'correct'
-        for i in correct_map:
-            if correct_map[i]!='correct':
+        success = 'correct'				# success = correct if ALL questions in this problem are correct
+        for answer_id in correct_map:
+            if not correct_map.is_correct(answer_id):
                 success = 'incorrect'
 
-        event_info['correct_map']=correct_map
+        event_info['correct_map']=correct_map.get_dict()	# log this in the tracker
         event_info['success']=success
-
         self.tracker('save_problem_check', event_info)
 
         try:
-            html = self.get_problem_html(encapsulate=False)
+            html = self.get_problem_html(encapsulate=False)	# render problem into HTML
         except Exception,err:
             log.error('failed to generate html')
             raise Exception,err
@@ -430,16 +431,9 @@ class Module(XModule):
             self.tracker('reset_problem_fail', event_info)
             return "Refresh the page and make an attempt before resetting."
 
-        self.lcp.done=False
-        self.lcp.answers=dict()
-        self.lcp.correct_map=dict()
-        self.lcp.student_answers = dict()
-
-
+        self.lcp.do_reset()		# call method in LoncapaProblem to reset itself
         if self.rerandomize == "always":
-            self.lcp.context=dict()
-            self.lcp.questions=dict() # Detailed info about questions in problem instance. TODO: Should be by id and not lid.
-            self.lcp.seed=None
+            self.lcp.seed=None		# reset random number generator seed (note the self.lcp.get_state() in next line)
  
         self.lcp=LoncapaProblem(self.filestore.open(self.filename), self.item_id, self.lcp.get_state(), system=self.system)
 

common/lib/xmodule/test_files/formularesponse_with_hint.xml

+<problem>
+<script type="loncapa/python">
+# from loncapa import *
+x1 = 4 # lc_random(2,4,1)
+y1 = 5 # lc_random(3,7,1)
+
+x2 = 10 # lc_random(x1+1,9,1)
+y2 = 20 # lc_random(y1+1,15,1)
+
+m = (y2-y1)/(x2-x1)
+b = y1 - m*x1
+answer = "%s*x+%s" % (m,b)
+answer = answer.replace('+-','-')
+
+inverted_m = (x2-x1)/(y2-y1)
+inverted_b = b
+wrongans = "%s*x+%s" % (inverted_m,inverted_b)
+wrongans = wrongans.replace('+-','-')
+</script>
+
+<text>
+<p>Hints can be provided to students, based on the last response given, as well as the history of responses given.  Here is an example of a hint produced by a Formula Response problem.</p>
+
+<p>
+What is the equation of the line which passess through ($x1,$y1) and
+($x2,$y2)?</p>
+
+<p>The correct answer is <tt>$answer</tt>. A common error is to invert the equation for the slope.  Enter <tt>
+$wrongans</tt> to see a hint.</p>
+
+</text>
+
+<formularesponse samples="x@-5:5#11" id="11" answer="$answer">
+ <responseparam description="Numerical Tolerance" type="tolerance" default="0.001" name="tol" /> 
+ <text>y = <textline size="25" /></text>
+  <hintgroup>
+   <formulahint samples="x@-5:5#11" answer="$wrongans" name="inversegrad">
+   </formulahint>
+   <hintpart on="inversegrad">
+    <text>You have inverted the slope in the question.</text>
+   </hintpart>
+  </hintgroup>     
+</formularesponse>
+</problem>
+

common/lib/xmodule/test_files/stringresponse_with_hint.xml

+<problem >
+ <text><h2>Example: String Response Problem</h2>
+ <br/>
+ </text>
+ 
+ <text>Which US state has Lansing as its capital?</text>
+  <stringresponse answer="Michigan" type="ci">
+  <textline size="20" />
+  <hintgroup>
+   <stringhint answer="wisconsin" type="cs" name="wisc">
+   </stringhint>
+   <stringhint answer="minnesota" type="cs" name="minn">
+   </stringhint>
+   <hintpart on="wisc">
+    <text>The state capital of Wisconsin is Madison.</text>
+   </hintpart>
+   <hintpart on="minn">
+    <text>The state capital of Minnesota is St. Paul.</text>
+   </hintpart>
+   <hintpart on="default">
+    <text>The state you are looking for is also known as the 'Great Lakes State'</text>
+   </hintpart>     
+  </hintgroup>
+ </stringresponse>
+</problem>

common/lib/xmodule/tests.py

 #
-# unittests for courseware
+# unittests for xmodule (and capa)
 #
 # Note: run this using a like like this:
 #
-# django-admin.py test --settings=envs.test_ike --pythonpath=. courseware
+# django-admin.py test --settings=lms.envs.test_ike --pythonpath=. common/lib/xmodule
 
 import unittest
 import os
@@ -28,12 +28,13 @@ class I4xSystem(object):
         self.track_function = lambda x: None
         self.render_function = lambda x: {} # Probably incorrect
         self.exception404 = Exception
+        self.DEBUG = True
     def __repr__(self):
         return repr(self.__dict__)
     def __str__(self):
         return str(self.__dict__)
 
-i4xs = I4xSystem
+i4xs = I4xSystem()
 
 class ModelsTest(unittest.TestCase):
     def setUp(self):
@@ -96,31 +97,31 @@ class MultiChoiceTest(unittest.TestCase):
         multichoice_file = os.path.dirname(__file__)+"/test_files/multichoice.xml"
         test_lcp = lcp.LoncapaProblem(open(multichoice_file), '1', system=i4xs)
         correct_answers = {'1_2_1':'choice_foil3'}
-        self.assertEquals(test_lcp.grade_answers(correct_answers)['1_2_1'], 'correct')
+        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
         false_answers = {'1_2_1':'choice_foil2'}
-        self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
 
     def test_MC_bare_grades(self):
         multichoice_file = os.path.dirname(__file__)+"/test_files/multi_bare.xml"
         test_lcp = lcp.LoncapaProblem(open(multichoice_file), '1', system=i4xs)
         correct_answers = {'1_2_1':'choice_2'}
-        self.assertEquals(test_lcp.grade_answers(correct_answers)['1_2_1'], 'correct')
+        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
         false_answers = {'1_2_1':'choice_1'}
-        self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
         
     def test_TF_grade(self):
         truefalse_file =  os.path.dirname(__file__)+"/test_files/truefalse.xml"
         test_lcp = lcp.LoncapaProblem(open(truefalse_file), '1', system=i4xs)
         correct_answers = {'1_2_1':['choice_foil2', 'choice_foil1']}
-        self.assertEquals(test_lcp.grade_answers(correct_answers)['1_2_1'], 'correct')
+        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
         false_answers = {'1_2_1':['choice_foil1']}
-        self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
         false_answers = {'1_2_1':['choice_foil1', 'choice_foil3']}
-        self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
         false_answers = {'1_2_1':['choice_foil3']}
-        self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
         false_answers = {'1_2_1':['choice_foil1', 'choice_foil2', 'choice_foil3']}
-        self.assertEquals(test_lcp.grade_answers(false_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
         
 class ImageResponseTest(unittest.TestCase):
     def test_ir_grade(self):
@@ -131,8 +132,8 @@ class ImageResponseTest(unittest.TestCase):
         test_answers = {'1_2_1':'[500,20]',
                         '1_2_2':'[250,300]',
                         }
-        self.assertEquals(test_lcp.grade_answers(test_answers)['1_2_1'], 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers)['1_2_2'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_2'), 'incorrect')
         
 class SymbolicResponseTest(unittest.TestCase):
     def test_sr_grade(self):
@@ -220,8 +221,8 @@ class SymbolicResponseTest(unittest.TestCase):
   </mstyle>
 </math>''',
                         }
-        self.assertEquals(test_lcp.grade_answers(correct_answers)['1_2_1'], 'correct')
-        self.assertEquals(test_lcp.grade_answers(wrong_answers)['1_2_1'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(wrong_answers).get_correctness('1_2_1'), 'incorrect')
         
 class OptionResponseTest(unittest.TestCase):
     '''
@@ -237,8 +238,37 @@ class OptionResponseTest(unittest.TestCase):
         test_answers = {'1_2_1':'True',
                         '1_2_2':'True',
                         }
-        self.assertEquals(test_lcp.grade_answers(test_answers)['1_2_1'], 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers)['1_2_2'], 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_2'), 'incorrect')
+
+class FormulaResponseWithHintTest(unittest.TestCase):
+    '''
+    Test Formula response problem with a hint
+    This problem also uses calc.
+    '''
+    def test_or_grade(self):
+        problem_file = os.path.dirname(__file__)+"/test_files/formularesponse_with_hint.xml"
+        test_lcp = lcp.LoncapaProblem(open(problem_file), '1', system=i4xs)
+        correct_answers = {'1_2_1':'2.5*x-5.0'}
+        test_answers = {'1_2_1':'0.4*x-5.0'}
+        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
+        cmap = test_lcp.grade_answers(test_answers)
+        self.assertEquals(cmap.get_correctness('1_2_1'), 'incorrect')
+        self.assertTrue('You have inverted' in cmap.get_hint('1_2_1'))
+
+class StringResponseWithHintTest(unittest.TestCase):
+    '''
+    Test String response problem with a hint
+    '''
+    def test_or_grade(self):
+        problem_file = os.path.dirname(__file__)+"/test_files/stringresponse_with_hint.xml"
+        test_lcp = lcp.LoncapaProblem(open(problem_file), '1', system=i4xs)
+        correct_answers = {'1_2_1':'Michigan'}
+        test_answers = {'1_2_1':'Minnesota'}
+        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
+        cmap = test_lcp.grade_answers(test_answers)
+        self.assertEquals(cmap.get_correctness('1_2_1'), 'incorrect')
+        self.assertTrue('St. Paul' in cmap.get_hint('1_2_1'))
 
 #-----------------------------------------------------------------------------
 # Grading tests

create-dev-env.sh

-#!/bin/bash
+#!/usr/bin/env bash
 set -e
+
+# posix compliant sanity check
+if [ -z $BASH ] || [  $BASH = "/bin/sh" ]; then
+    echo "Please use the bash interpreter to run this script"
+    exit 1
+fi
+
 trap "ouch" ERR
 
 ouch() {
@@ -14,6 +21,7 @@ ouch() {
     script again with the -v flag.  
 
 EOL
+    printf '\E[0m'
 
 }
 error() {
@@ -28,6 +36,7 @@ usage() {
     Usage: $PROG [-c] [-v] [-h]
     
             -c        compile scipy and numpy
+            -s        --system-site-packages for virtualenv
             -v        set -x + spew
             -h        this
 
@@ -48,21 +57,45 @@ EO
 
 clone_repos() {
     cd "$BASE"
+    
+    if [[ -d "$BASE/mitx/.git" ]]; then
+        output "Pulling mitx"
+        cd "$BASE/mitx"
+        git pull >>$LOG
+    else
         output "Cloning mitx"
         if [[ -d "$BASE/mitx" ]]; then
             mv "$BASE/mitx" "${BASE}/mitx.bak.$$"
         fi
         git clone git@github.com:MITx/mitx.git >>$LOG 
+    fi
+    
+    cd "$BASE"
+    if [[ -d "$BASE/askbot-devel/.git" ]]; then
+        output "Pulling askbot-devel"
+        cd "$BASE/askbot-devel"
+        git pull >>$LOG
+    else
         output "Cloning askbot-devel"
         if [[ -d "$BASE/askbot-devel" ]]; then
             mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$"
         fi
         git clone git@github.com:MITx/askbot-devel >>$LOG 
+    fi
+    
+    cd "$BASE"
+    if [[ -d "$BASE/data/.hg" ]]; then
+        output "Pulling data"
+        cd "$BASE/data"
+        hg pull >>$LOG
+        hg update >>$LOG
+    else
         output "Cloning data"
         if [[ -d "$BASE/data" ]]; then
             mv "$BASE/data" "${BASE}/data.bak.$$"
         fi
         hg clone ssh://hg-content@gp.mitx.mit.edu/data >>$LOG 
+    fi
 }
 
 PROG=${0##*/}
@@ -81,7 +114,7 @@ if [[ $EUID -eq 0 ]]; then
     usage
     exit 1
 fi
-ARGS=$(getopt "cvh" "$*")
+ARGS=$(getopt "cvhs" "$*")
 if [[ $? != 0 ]]; then
     usage
     exit 1
@@ -93,6 +126,10 @@ while true; do
             compile=true
             shift
             ;;
+        -s)
+            systempkgs=true
+            shift
+            ;;
         -v)
             set -x
             verbose=true
@@ -123,7 +160,7 @@ cat<<EO
 
   To compile scipy and numpy from source use the -c option
 
-  STDOUT is redirected to /var/tmp/install.log, run
+  Most of STDOUT is redirected to /var/tmp/install.log, run
   $ tail -f /var/tmp/install.log
   to monitor progress
 
@@ -211,8 +248,13 @@ esac
 output "Installing rvm and ruby"
 curl -sL get.rvm.io | bash -s stable
 source $RUBY_DIR/scripts/rvm
-rvm install $RUBY_VER
-virtualenv "$PYTHON_DIR"
+# skip the intro 
+LESS="-E" rvm install $RUBY_VER
+if [[ -n $systempkgs ]]; then
+    virtualenv --system-site-packages "$PYTHON_DIR"
+else
+    virtualenv "$PYTHON_DIR"
+fi
 source $PYTHON_DIR/bin/activate
 output "Installing gem bundler"
 gem install bundler
@@ -251,22 +293,37 @@ mkdir "$BASE/log" || true
 mkdir "$BASE/db" || true
 
 cat<<END
-   
    Success!!
 
-   To start using Django you will need
-   to activate the local Python and Ruby
-   environment:
+   To start using Django you will need to activate the local Python 
+   and Ruby environment (at this time rvm only supports bash) :
 
         $ source $RUBY_DIR/scripts/rvm
         $ source $PYTHON_DIR/bin/activate
   
-   To initialize and start a local instance of Django:
+   To initialize Django
         
         $ cd $BASE/mitx
-        $ django-admin.py syncdb --settings=envs.dev --pythonpath=.
-        $ django-admin.py migrate --settings=envs.dev --pythonpath=.
-        $ django-admin.py runserver --settings=envs.dev --pythonpath=.   
+        $ rake django-admin[syncdb]
+        $ rake django-admin[migrate]
+
+   To start the Django on port 8000
+
+        $ rake lms
+   
+   Or to start Django on a different <port#>
+
+        $ rake django-admin[runserver,lms,dev,<port#>]   
+
+  If the Django development server starts properly you 
+  should see:
+
+      Development server is running at http://127.0.0.1:<port#>/
+      Quit the server with CONTROL-C.
+
+  Connect your browser to http://127.0.0.1:<port#> to 
+  view the Django site.
+
 
 END
 exit 0

doc/README

+This directory contains some high level documentation for the code.  We should strive to keep it up-to-date, but don't take it as the absolute truth.
+
+A good place to start is 'overview.md'

doc/code_standards.txt

+Scope
+
+This document describes code quality standards for the i4x
+system. 
+
+1. Coding Standards
+
+Code falls into four categories:
+
+* Deployed. Running on a live server. 
+* Production. Intended for deployment. 
+* Scaffolding. Intended to define interfaces for future work, and
+  minimal implementations to support further development. 
+* Prototype. Experimental new features. 
+
+1.1 Deployed
+
+The standards for deployed code are identical to production. In
+general, we tend to do either:
+
+1) Perform a final verification QA cycle on changed parts of code
+before deploying.
+2) Use code on a staging or internal server for a week before
+deploying.
+
+1.2 Production
+
+All production code must be peer-reviewed. The code must meet the
+following standards:
+
+1) Test Suite. Code must have reasonable, although not complete, test
+   coverage.
+2) Consistent. Code must follow PEP8
+3) Clean Abstractions. 
+4) Future-Compatible. Code must not be incompatible with the
+   long-term vision of either the codebase or of edX. 
+5) Properly Documented
+6) Maintainable and deployable
+7) Robust. 
+
+All code paths must be manually or automatically verified. 
+
+1.3 Scaffolding
+
+All scaffolding code should be peer-reviewed. The code must meet the
+following standards:
+
+1) Testable. We do not require test coverage, but we do require the
+   code to be structured such that it is possible to build tests.
+2) Consistent. Code must follow PEP8
+3) Clean abstractions or obvious throw-away code. One of the goals 
+   of scaffolding is to define proper abstractions. 
+4) Future-Compatible. Code must not be incompatible with the
+   long-term vision of either the codebase or of edX. 
+5) Somewhat documented
+6) Unpluggable. There should be a setting to disable scaffolding code. 
+   By default, and by policy, it should never be enabled on production 
+   servers.
+7) Purpose. The scaffolding must provide a clean reason for existence
+   (e.g. define a specific interface, etc.)
+
+1.4 Prototype
+
+Prototype code should live in a separate branch. It should strive
+to follow PEP8, be readable, testable, and future-proof, but we have
+no hard standards. 
+
+2. Process Standards
+
+* Code should be integrated in small pull requests. Large commits
+  should be broken down into small commits for integration.
+* Every piece of production and deployed code must be reviewed prior
+  to integration.
+* Anyone on the edX team competent to review a piece of code may
+  review it (this may change as the team grows). 
+* Each contributor is responsible for finding a person to review their
+  code. If it is not clear to the contributor who is appropriate, each
+  project has an owner
+
+3. Documentation Standards
+
+* Whenever possible, documentation should live in code. 
+* When impossible, it should live in the github repo. 
+* Discussion should live on github, Basecamp or Pivotal, depending on
+  context.

doc/overview.md

+# Documentation for edX code (mitx repo)
+
+This document explains the general structure of the edX platform, and defines some of the acronyms and terms you'll see flying around in the code.
+
+## Assumptions:
+
+You should be familiar with the following.  If you're not, go read some docs...
+
+ - python
+ - django
+ - javascript
+ - html, xml -- xpath, xslt
+ - css
+ - git
+ - mako templates -- we use these instead of django templates, because they support embedding real python.
+ 
+## Other relevant terms
+
+ - CAPA -- lon-capa.org -- content management system that has defined a standard for online learning and assessment materials.  Many of our materials follow this standard.
+    - TODO: add more details / link to relevant docs.  lon-capa.org is not immediately intuitive.  
+    - lcp = loncapa problem
+
+
+## Parts of the system
+
+  - LMS -- Learning Management System.   The student-facing parts of the system.  Handles student accounts, displaying videos, tutorials, exercies, problems, etc. 
+
+  - CMS -- Course Management System.  The instructor-facing parts of the system.  Allows instructors to see and modify their course, add lectures, problems, reorder things, etc.
+
+  - Askbot -- the discussion forums.  We have a custom fork of this project.  We're also hoping to replace it with something better later.  (e.g. need support for multiple classes, etc)
+
+  - Data.  In the data/ dir.  There is currently a single `course.xml` file that describes an entire course.  Speaking of which...
+
+  - Courses.  A course is broken up into Chapters ("week 1", "week 2", etc).  A chapter is broken up into Sections ("Lecture 1", "Simple Circuits Exercises", "HW1", etc).  A section can contain modules: Problems, Html, Videos, Verticals, or Sequences.
+     - Problems: specified in problem files.  May have python scripts embedded to both generate random parameters and check answers.  Also allows specifying things like tolerance or precision in answers
+     - Html: any html - often description, or links to outside resources
+     - Videos: links to youtube or elsewhere
+     - Verticals: a nesting tag: collect several videos, problems, html modules and display them vertically.
+     - Sequences: a sequence of modules, displayed with a horizontal navigation bar, displaying one component at a time.
+     - see `data/course.xml` for more examples
+
+
+## High Level Entities in the code
+
+### Common libraries 
+
+- x_modules -- generic learning modules. *x* can be sequence, video, template, html, vertical, capa, etc.  These are the things that one puts inside sections in the course structure.  Modules know how to render themselves to html, how to score themselves, and handle ajax calls from the front end. 
+      - x_modules take a 'system context' parameter, which helps isolate xmodules from any particular application, so they can be used in many places. The modules should make no references to Django (though there are still a few left).  The system context knows how to render things, track events, complain about 404s, etc. 
+      - TODO: document the system context interface--it's different in `x_module.XModule.__init__` and in `x_module tests.py` (do this in the code, not here)
+      - in `common/lib/xmodule`
+
+- capa modules -- defines `LoncapaProblem` and many related things.  
+    - in `common/lib/capa`
+
+### LMS 
+
+The LMS is a django site, with root in `lms/`.  It runs in many different environments--the settings files are in `lms/envs`. 
+
+- We use the Django Auth system, including the is_staff and is_superuser flags.  User profiles and related code lives in `lms/djangoapps/student/`.   There is support for groups of students (e.g. 'want emails about future courses', 'have unenrolled', etc) in `lms/djangoapps/student/models.py`.
+
+- `StudentModule` -- keeps track of where a particular student is in a module (problem, video, html)--what's their grade, have they started, are they done, etc.  [This is only partly implemented so far.]
+    - `lms/djangoapps/courseware/models.py`
+
+- Core rendering path:
+  - `lms/urls.py` points to `courseware.views.index`, which gets module info from the course xml file, pulls list of `StudentModule` objects for this user (to avoid multiple db hits).  
+
+  - Calls `render_accordion` to render the "accordion"--the display of the course structure.
+
+  - To render the current module, calls `module_render.py:render_x_module()`, which gets the `StudentModule` instance, and passes the `StudentModule` state and other system context to the module constructor the get an instance of the appropriate module class for this user.
+
+  - calls the module's `.get_html()` method.  If the module has nested submodules, render_x_module() will be called again for each.
+  
+  - ajax calls go to `module_render.py:modx_dispatch()`, which passes it to the module's `handle_ajax()` function, and then updates the grade and state if they changed.
+
+  - [This diagram](https://github.com/MITx/mitx/wiki/MITx-Architecture) visually shows how the clients communicate with problems + modules.
+  
+- See `lms/urls.py` for the wirings of urls to views.  
+
+- Tracking: there is support for basic tracking of client-side events in `lms/djangoapps/track`.  
+
+### Other modules
+
+- Wiki -- in `lms/djangoapps/simplewiki`.  Has some markdown extentions for embedding circuits, videos, etc.
+
+## Testing
+
+See `testing.md`.
+
+## TODO:
+
+- update lms/envs/README.txt
+
+- describe our production environment
+
+- describe the front-end architecture, tools, etc.  Starting point: `lms/static`
+
+---
+Note: this file uses markdown.  To convert to html, run:
+
+    markdown2 overview.md > overview.html

doc/testing.md

+# Testing
+
+Testing is good.  Here is some useful info about how we set up tests--
+
+### Backend code:
+
+- TODO
+
+### Frontend code:
+
+- TODO
+

lms/djangoapps/courseware/module_render.py

@@ -26,8 +26,29 @@ class I4xSystem(object):
     This is an abstraction such that x_modules can function independent 
     of the courseware (e.g. import into other types of courseware, LMS, 
     or if we want to have a sandbox server for user-contributed content)
+
+    I4xSystem objects are passed to x_modules to provide access to system
+    functionality.
+    '''
+    def __init__(self, ajax_url, track_function, render_function,
+                 render_template, filestore=None):
+        '''
+        Create a closure around the system environment.
+
+        ajax_url - the url where ajax calls to the encapsulating module go.
+        track_function - function of (event_type, event), intended for logging
+                         or otherwise tracking the event.
+                         TODO: Not used, and has inconsistent args in different
+                         files.  Update or remove.
+        render_function - function that takes (module_xml) and renders it,
+                          returning a dictionary with a context for rendering the
+                          module to html.  Dictionary will contain keys 'content'
+                          and 'type'.
+        render_template - a function that takes (template_file, context), and returns
+                          rendered html.
+        filestore - A filestore ojbect.  Defaults to an instance of OSFS based at
+                    settings.DATA_DIR.
         '''
-    def __init__(self, ajax_url, track_function, render_function, render_template, filestore=None):
         self.ajax_url = ajax_url
         self.track_function = track_function
         if not filestore: 
@@ -35,37 +56,47 @@ class I4xSystem(object):
         else:
             self.filestore = filestore
             if settings.DEBUG:
-                log.info("[courseware.module_render.I4xSystem] filestore path = %s" % filestore)
+                log.info("[courseware.module_render.I4xSystem] filestore path = %s",
+                         filestore)
         self.render_function = render_function
         self.render_template = render_template
         self.exception404 = Http404
         self.DEBUG = settings.DEBUG
 
-    def get(self,attr):			# uniform access to attributes (like etree)
+    def get(self, attr):
+        '''	provide uniform access to attributes (like etree).'''
         return self.__dict__.get(attr)
-    def set(self,attr,val):		# uniform access to attributes (like etree)
+    
+    def set(self,attr,val):
+        '''provide uniform access to attributes (like etree)'''
         self.__dict__[attr] = val
+
     def __repr__(self):
         return repr(self.__dict__)
+
     def __str__(self):
         return str(self.__dict__)
 
-def object_cache(cache, user, module_type, module_id):
-    # We don't look up on user -- all queries include user
-    # Additional lookup would require a DB hit the way Django 
-    # is broken. 
+def smod_cache_lookup(cache, module_type, module_id):
+    '''
+    Look for a student module with the given type and id in the cache.
+
+    cache -- list of student modules
+
+    returns first found object, or None
+    '''
     for o in cache: 
-        if o.module_type == module_type and \
-                o.module_id == module_id:
+        if o.module_type == module_type and o.module_id == module_id:
             return o
     return None
 
 def make_track_function(request):
     ''' We want the capa problem (and other modules) to be able to
     track/log what happens inside them without adding dependencies on
-    Django or the rest of the codebase. We do this by passing a
-    tracking function to them. This generates a closure for each request 
-    that gives a clean interface on both sides. 
+    Django or the rest of the codebase.
+
+    To do this in a clean way, we pass a tracking function to the module,
+    which calls it to log events.
     '''
     import track.views
 
@@ -80,67 +111,86 @@ def grade_histogram(module_id):
     from django.db import connection
     cursor = connection.cursor()
 
-    cursor.execute("select courseware_studentmodule.grade,COUNT(courseware_studentmodule.student_id) from courseware_studentmodule where courseware_studentmodule.module_id=%s group by courseware_studentmodule.grade", [module_id])
+    q = """SELECT courseware_studentmodule.grade,
+                  COUNT(courseware_studentmodule.student_id)
+    FROM courseware_studentmodule
+    WHERE courseware_studentmodule.module_id=%s
+    GROUP BY courseware_studentmodule.grade"""
+    # Passing module_id this way prevents sql-injection.
+    cursor.execute(q, [module_id])
 
     grades = list(cursor.fetchall())
-    grades.sort(key=lambda x:x[0]) # Probably not necessary
-    if (len(grades) == 1 and grades[0][0] is None):
+    grades.sort(key=lambda x: x[0])          # Add ORDER BY to sql query?
+    if len(grades) == 1 and grades[0][0] is None:
         return []
     return grades
 
-def get_module(user, request, xml_module, module_object_preload, position=None):
-    module_type=xml_module.tag
-    module_class=xmodule.get_module_class(module_type)
-    module_id=xml_module.get('id') #module_class.id_attribute) or "" 
+def get_module(user, request, module_xml, student_module_cache, position=None):
+    ''' Get an instance of the xmodule class corresponding to module_xml,
+    setting the state based on an existing StudentModule, or creating one if none
+    exists.
 
-    # Grab state from database
-    smod = object_cache(module_object_preload, 
-                        user, 
-                        module_type, 
-                        module_id)
+    Arguments:
+      - user                  : current django User
+      - request               : current django HTTPrequest
+      - module_xml            : lxml etree of xml subtree for the requested module
+      - student_module_cache  : list of StudentModule objects, one of which may
+                                match this module type and id
+      - position   	          : extra information from URL for user-specified
+                                position within module
 
-    if not smod: # If nothing in the database...
-        state=None
-    else:
-        state = smod.state
+    Returns:
+      - a tuple (xmodule instance, student module, module type).
+    '''
+    module_type = module_xml.tag
+    module_class = xmodule.get_module_class(module_type)
+    module_id = module_xml.get('id')
 
-    # get coursename if stored
+    # Grab xmodule state from StudentModule cache
+    smod = smod_cache_lookup(student_module_cache, module_type, module_id)
+    state = smod.state if smod else None
+
+    # get coursename if present in request
     coursename = multicourse_settings.get_coursename_from_request(request)
 
     if coursename and settings.ENABLE_MULTICOURSE:
-        xp = multicourse_settings.get_course_xmlpath(coursename)	# path to XML for the course
+        # path to XML for the course
+        xp = multicourse_settings.get_course_xmlpath(coursename)
         data_root = settings.DATA_DIR + xp
     else:
         data_root = settings.DATA_DIR
 
-    # Create a new instance
-    ajax_url = settings.MITX_ROOT_URL + '/modx/'+module_type+'/'+module_id+'/'
+    # Setup system context for module instance
+    ajax_url = settings.MITX_ROOT_URL + '/modx/' + module_type + '/' + module_id + '/'
+    def render_function(module_xml):
+        return render_x_module(user, request, module_xml, student_module_cache, position)
     
     system = I4xSystem(track_function = make_track_function(request), 
-                       render_function = lambda x: render_x_module(user, request, x, module_object_preload, position), 
+                       render_function = render_function, 
                        render_template = render_to_string,
                        ajax_url = ajax_url,
                        filestore = OSFS(data_root),
                        )
-    system.set('position',position)	# pass URL specified position along to module, through I4xSystem
-    instance=module_class(system, 
-                          etree.tostring(xml_module), 
+    # pass position specified in URL to module through I4xSystem
+    system.set('position', position) 
+    instance = module_class(system, 
+                            etree.tostring(module_xml), 
                             module_id, 
                             state=state)
     
-    # If instance wasn't already in the database, and this
-    # isn't a guest user, create it
+    # If StudentModule for this instance wasn't already in the database,
+    # and this isn't a guest user, create it.
     if not smod and user.is_authenticated():
-        smod=StudentModule(student=user, 
-                           module_type = module_type,
-                           module_id=module_id, 
-                           state=instance.get_state())
+        smod = StudentModule(student=user, module_type = module_type,
+                           module_id=module_id, state=instance.get_state())
         smod.save()
-        module_object_preload.append(smod)
+        # Add to cache. The caller and the system context have references
+        # to it, so the change persists past the return
+        student_module_cache.append(smod)
 
     return (instance, smod, module_type)
 
-def render_x_module(user, request, xml_module, module_object_preload, position=None):
+def render_x_module(user, request, module_xml, student_module_cache, position=None):
     ''' Generic module for extensions. This renders to HTML.
 
     modules include sequential, vertical, problem, video, html
@@ -151,64 +201,83 @@ def render_x_module(user, request, xml_module, module_object_preload, position=N
 
       - user                  : current django User
       - request               : current django HTTPrequest
-      - xml_module            : lxml etree of xml subtree for the current module
-      - module_object_preload : list of StudentModule objects, one of which may match this module type and id
+      - module_xml            : lxml etree of xml subtree for the current module
+      - student_module_cache : list of StudentModule objects, one of which may match this module type and id
       - position   	      : extra information from URL for user-specified position within module
 
     Returns:
 
-      -  dict which is context for HTML rendering of the specified module
-
+      - dict which is context for HTML rendering of the specified module.  Will have
+      key 'content', and will have 'type' key if passed a valid module.
     '''
-    if xml_module==None :
-        return {"content":""}
+    if module_xml is None :
+        return {"content": ""}
 
-    (instance, smod, module_type) = get_module(user, request, xml_module, module_object_preload, position)
+    (instance, smod, module_type) = get_module(
+        user, request, module_xml, student_module_cache, position)
 
-    # Grab content
     content = instance.get_html()
 
     # special extra information about each problem, only for users who are staff 
-    if user.is_staff:
-        module_id = xml_module.get('id')
+    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
+        module_id = module_xml.get('id')
         histogram = grade_histogram(module_id)
         render_histogram = len(histogram) > 0
-        content=content+render_to_string("staff_problem_info.html", {'xml':etree.tostring(xml_module), 
-                                                                     'module_id' : module_id,
+        staff_context = {'xml': etree.tostring(module_xml), 
+                         'module_id': module_id,
                          'histogram': json.dumps(histogram),
-                                                                     'render_histogram' : render_histogram})
+                         'render_histogram': render_histogram}
+        content += render_to_string("staff_problem_info.html", staff_context)
 
-    content = {'content':content,
-               'type':module_type}
-
-    return content
+    context = {'content': content, 'type': module_type}
+    return context
 
 def modx_dispatch(request, module=None, dispatch=None, id=None):
-    ''' Generic view for extensions. This is where AJAX calls go.'''
+    ''' Generic view for extensions. This is where AJAX calls go.
+
+    Arguments:
+
+      - request -- the django request.
+      - module -- the type of the module, as used in the course configuration xml.
+                  e.g. 'problem', 'video', etc
+      - dispatch -- the command string to pass through to the module's handle_ajax call
+           (e.g. 'problem_reset').  If this string contains '?', only pass
+           through the part before the first '?'.
+      - id -- the module id.  Used to look up the student module.
+            e.g. filenamexformularesponse
+    '''
+    # ''' (fix emacs broken parsing)
     if not request.user.is_authenticated():
         return redirect('/')
 
+    # python concats adjacent strings
+    error_msg = ("We're sorry, this module is temporarily unavailable."
+                 "Our staff is working to fix it as soon as possible")
+
+
     # Grab the student information for the module from the database
     s = StudentModule.objects.filter(student=request.user, 
                                      module_id=id)
-    #s = StudentModule.get_with_caching(request.user, id)
-    if len(s) == 0 or s is None:
-        log.debug("Couldnt find module for user and id " + str(module) + " " + str(request.user) + " "+ str(id))
+
+    # s = StudentModule.get_with_caching(request.user, id)
+    if s is None or len(s) == 0:
+        log.debug("Couldn't find module '%s' for user '%s' and id '%s'",
+                  module, request.user, id)
         raise Http404
     s = s[0]
 
     oldgrade = s.grade
     oldstate = s.state
 
-    dispatch=dispatch.split('?')[0]
+    # If there are arguments, get rid of them
+    if '?' in dispatch:
+        dispatch = dispatch.split('?')[0]
 
-    ajax_url = settings.MITX_ROOT_URL + '/modx/'+module+'/'+id+'/'
-
-    # get coursename if stored
+    ajax_url = '{root}/modx/{module}/{id}'.format(root = settings.MITX_ROOT_URL,
+                                                  module=module, id=id)
     coursename = multicourse_settings.get_coursename_from_request(request)
-
     if coursename and settings.ENABLE_MULTICOURSE:
-        xp = multicourse_settings.get_course_xmlpath(coursename)	# path to XML for the course
+        xp = multicourse_settings.get_course_xmlpath(coursename)
         data_root = settings.DATA_DIR + xp
     else:
         data_root = settings.DATA_DIR
@@ -217,11 +286,13 @@ def modx_dispatch(request, module=None, dispatch=None, id=None):
     try:
         xml = content_parser.module_xml(request.user, module, 'id', id, coursename)
     except:
-        log.exception("Unable to load module during ajax call. module=%s, dispatch=%s, id=%s" % (module, dispatch, id))
+        log.exception(
+            "Unable to load module during ajax call. module=%s, dispatch=%s, id=%s",
+            module, dispatch, id)
         if accepts(request, 'text/html'):
             return render_to_response("module-error.html", {})
         else:
-            response = HttpResponse(json.dumps({'success': "We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible"}))
+            response = HttpResponse(json.dumps({'success': error_msg}))
         return response
 
     # Create the module
@@ -233,24 +304,23 @@ def modx_dispatch(request, module=None, dispatch=None, id=None):
                        )
 
     try:
-        instance=xmodule.get_module_class(module)(system, 
-                                                             xml, 
-                                                             id, 
-                                                             state=oldstate)
+        module_class = xmodule.get_module_class(module)
+        instance = module_class(system, xml, id, state=oldstate)
     except:
         log.exception("Unable to load module instance during ajax call")
         if accepts(request, 'text/html'):
             return render_to_response("module-error.html", {})
         else:
-            response = HttpResponse(json.dumps({'success': "We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible"}))
+            response = HttpResponse(json.dumps({'success': error_msg}))
         return response
 
     # Let the module handle the AJAX
-    ajax_return=instance.handle_ajax(dispatch, request.POST)
+    ajax_return = instance.handle_ajax(dispatch, request.POST)
+
     # Save the state back to the database
-    s.state=instance.get_state()
+    s.state = instance.get_state()
     if instance.get_score(): 
-        s.grade=instance.get_score()['score']
+        s.grade = instance.get_score()['score']
     if s.grade != oldgrade or s.state != oldstate:
         s.save()
     # Return whatever the module wanted to return to the client/caller

lms/djangoapps/courseware/views.py

@@ -41,18 +41,18 @@ def gradebook(request):
     coursename = multicourse_settings.get_coursename_from_request(request)
 
     student_objects = User.objects.all()[:100]
-    student_info = [{'username' :s.username,
-                     'id' : s.id,
+    student_info = [{'username': s.username,
+                     'id': s.id,
                      'email': s.email,
-                     'grade_info' : grades.grade_sheet(s,coursename), 
-                     'realname' : UserProfile.objects.get(user = s).name
+                     'grade_info': grades.grade_sheet(s, coursename),
+                     'realname': UserProfile.objects.get(user = s).name
                      } for s in student_objects]
 
-    return render_to_response('gradebook.html',{'students':student_info})
+    return render_to_response('gradebook.html', {'students': student_info})
 
 @login_required
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def profile(request, student_id = None):
+def profile(request, student_id=None):
     ''' User profile. Show username, location, etc, as well as grades .
         We need to allow the user to change some of these settings .'''
 
@@ -67,36 +67,41 @@ def profile(request, student_id = None):
 
     coursename = multicourse_settings.get_coursename_from_request(request)
 
-    context={'name':user_info.name,
-             'username':student.username,
-             'location':user_info.location,
-             'language':user_info.language,
-             'email':student.email,
-             'format_url_params' : content_parser.format_url_params,
-             'csrf':csrf(request)['csrf_token']
+    context = {'name': user_info.name,
+               'username': student.username,
+               'location': user_info.location,
+               'language': user_info.language,
+               'email': student.email,
+               'format_url_params': content_parser.format_url_params,
+               'csrf': csrf(request)['csrf_token']
                }
-    context.update(grades.grade_sheet(student,coursename))
+    context.update(grades.grade_sheet(student, coursename))
 
     return render_to_response('profile.html', context)
 
-def render_accordion(request,course,chapter,section):
+
+def render_accordion(request, course, chapter, section):
     ''' Draws navigation bar. Takes current position in accordion as
         parameter. Returns (initialization_javascript, content)'''
     if not course:
         course = "6.002 Spring 2012"
 
-    toc=content_parser.toc_from_xml(content_parser.course_file(request.user,course), chapter, section)
-    active_chapter=1
+    toc = content_parser.toc_from_xml(
+        content_parser.course_file(request.user, course), chapter, section)
+
+    active_chapter = 1
     for i in range(len(toc)):
         if toc[i]['active']:
-            active_chapter=i
-    context=dict([['active_chapter',active_chapter],
-                  ['toc',toc], 
-                  ['course_name',course],
-                  ['format_url_params',content_parser.format_url_params],
-                  ['csrf',csrf(request)['csrf_token']]] + \
+            active_chapter = i
+
+    context=dict([('active_chapter', active_chapter),
+                  ('toc', toc),
+                  ('course_name', course),
+                  ('format_url_params', content_parser.format_url_params),
+                  ('csrf', csrf(request)['csrf_token'])] +
                      template_imports.items())
-    return render_to_string('accordion.html',context)
+    return render_to_string('accordion.html', context)
+
 
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def render_section(request, section):
@@ -122,13 +127,13 @@ def render_section(request, section):
     module_ids = dom.xpath("//@id")
 
     if user.is_authenticated():
-        module_object_preload = list(StudentModule.objects.filter(student=user, 
+        student_module_cache = list(StudentModule.objects.filter(student=user,
                                                                   module_id__in=module_ids))
     else:
-        module_object_preload = []
+        student_module_cache = []
 
     try:
-        module = render_x_module(user, request, dom, module_object_preload)
+        module = render_x_module(user, request, dom, student_module_cache)
     except:
         log.exception("Unable to load module")
         context.update({
@@ -138,18 +143,67 @@ def render_section(request, section):
         return render_to_response('courseware.html', context)
 
     context.update({
-        'init':module.get('init_js', ''),
-        'content':module['content'],
+        'init': module.get('init_js', ''),
+        'content': module['content'],
     })
 
     result = render_to_response('courseware.html', context)
     return result
 
+def get_course(request, course):
+    ''' Figure out what the correct course is.
+
+    Needed to preserve backwards compatibility with non-multi-course version.
+    TODO: Can this go away once multicourse becomes standard?
+    '''
+
+    if course==None:
+        if not settings.ENABLE_MULTICOURSE:
+            course = "6.002 Spring 2012"
+        elif 'coursename' in request.session:
+            course = request.session['coursename']
+        else:
+            course = settings.COURSE_DEFAULT
+    return course
+
+def get_module_xml(user, course, chapter, section):
+    ''' Look up the module xml for the given course/chapter/section path.
+
+    Takes the user to look up the course file.
+
+    Returns None if there was a problem, or the lxml etree for the module.
+    '''
+    try:
+        # this is the course.xml etree
+        dom = content_parser.course_file(user, course)
+    except:
+        log.exception("Unable to parse courseware xml")
+        return None
+
+    # this is the module's parent's etree
+    path = "//course[@name=$course]/chapter[@name=$chapter]//section[@name=$section]"
+    dom_module = dom.xpath(path, course=course, chapter=chapter, section=section)
+
+    module_wrapper = dom_module[0] if len(dom_module) > 0 else None
+    if module_wrapper is None:
+        module = None
+    elif module_wrapper.get("src"):
+        module = content_parser.section_file(
+            user=user, section=module_wrapper.get("src"), coursename=course)
+    else:
+        # Copy the element out of the module's etree
+        module = etree.XML(etree.tostring(module_wrapper[0]))
+    return module
+
 
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def index(request, course=None, chapter="Using the System", section="Hints",position=None): 
+def index(request, course=None, chapter=None, section=None,
+          position=None):
     ''' Displays courseware accordion, and any associated content.
+    If course, chapter, and section aren't all specified, just returns
+    the accordion.  If they are specified, returns an error if they don't
+    point to a valid module.
 
     Arguments:
 
@@ -162,110 +216,113 @@ def index(request, course=None, chapter="Using the System", section="Hints",posi
     Returns:
 
      - HTTPresponse
+    '''
+    def clean(s):
+        ''' Fixes URLs -- we convert spaces to _ in URLs to prevent
+        funny encoding characters and keep the URLs readable.  This undoes
+        that transformation.
 
+        TODO: Properly replace underscores.  (Q: what is properly?)
         '''
-    user = request.user
-    if not settings.COURSEWARE_ENABLED:
-        return redirect('/')
+        return s.replace('_', ' ')
 
-    if course==None:
-        if not settings.ENABLE_MULTICOURSE:
-            course = "6.002 Spring 2012"
-        elif 'coursename' in request.session:
-            course = request.session['coursename']
+    def get_submodule_ids(module_xml):
+        '''
+        Get a list with ids of the modules within this module.
+        '''
+        return module_xml.xpath("//@id")
+
+    def preload_student_modules(module_xml):
+        '''
+        Find any StudentModule objects for this user that match
+        one of the given module_ids.  Used as a cache to avoid having
+        each rendered module hit the db separately.
+
+        Returns the list, or None on error.
+        '''
+        if request.user.is_authenticated():
+            module_ids = get_submodule_ids(module_xml)
+            return list(StudentModule.objects.filter(student=request.user,
+                                                     module_id__in=module_ids))
         else:
-            course = settings.COURSE_DEFAULT
+            return []
 
-    # Fixes URLs -- we don't get funny encoding characters from spaces
-    # so they remain readable
-    ## TODO: Properly replace underscores
-    course=course.replace("_"," ")
-    chapter=chapter.replace("_"," ")
-    section=section.replace("_"," ")
+    def get_module_context():
+        '''
+        Look up the module object and render it.  If all goes well, returns
+        {'init': module-init-js, 'content': module-rendered-content}
 
-    # use multicourse module to determine if "course" is valid
-    #if course!=settings.COURSE_NAME.replace('_',' '):
-    if not multicourse_settings.is_valid_course(course):
-        return redirect('/')
+        If there's an error, returns
+        {'content': module-error message}
+        '''
+        # Can't modify variables of outer scope, so need new ones
+        chapter_ = clean(chapter)
+        section_ = clean(section)
 
-    request.session['coursename'] = course		# keep track of current course being viewed in django's request.session
+        user = request.user
         
-    try:
-        # this is the course.xml etree 
-        dom = content_parser.course_file(user,course)	# also pass course to it, for course-specific XML path
-    except:
-        log.exception("Unable to parse courseware xml")
-        return render_to_response('courseware-error.html', {})
+        module_xml = get_module_xml(user, course, chapter_, section_)
+        if module_xml is None:
+            log.exception("couldn't get module_xml: course/chapter/section: '%s/%s/%s'",
+                          course, chapter_, section_)
+            return {'content' : render_to_string("module-error.html", {})}
         
-    # this is the module's parent's etree
-    dom_module = dom.xpath("//course[@name=$course]/chapter[@name=$chapter]//section[@name=$section]", 
-                           course=course, chapter=chapter, section=section)
+        student_module_cache = preload_student_modules(module_xml)
 
-    #print "DM", dom_module
+        try:
+            module_context = render_x_module(user, request, module_xml,
+                                             student_module_cache, position)
+        except:
+            log.exception("Unable to load module")
+            return {'content' : render_to_string("module-error.html", {})}
 
-    if len(dom_module) == 0:
-        module_wrapper = None
-    else:
-        module_wrapper = dom_module[0]
+        return {'init': module_context.get('init_js', ''),
+                'content': module_context['content']}
     
-    if module_wrapper is None:
-        module = None
-    elif module_wrapper.get("src"):
-        module = content_parser.section_file(user=user, section=module_wrapper.get("src"), coursename=course)
-    else:
-        # this is the module's etree
-        module = etree.XML(etree.tostring(module_wrapper[0])) # Copy the element out of the tree
+    if not settings.COURSEWARE_ENABLED:
+        return redirect('/')
 
-    module_ids = []
-    if module is not None:
-        module_ids = module.xpath("//@id", 
-                                  course=course, chapter=chapter, section=section)
+    course = clean(get_course(request, course))
+    if not multicourse_settings.is_valid_course(course):
+        return redirect('/')
 
-    if user.is_authenticated():
-        module_object_preload = list(StudentModule.objects.filter(student=user, 
-                                                                  module_id__in=module_ids))
-    else:
-        module_object_preload = []
+    # keep track of current course being viewed in django's request.session
+    request.session['coursename'] = course
 
     context = {
         'csrf': csrf(request)['csrf_token'],
         'accordion': render_accordion(request, course, chapter, section),
-        'COURSE_TITLE':multicourse_settings.get_course_title(course),
-    }
-
-    try:
-        module_context = render_x_module(user, request, module, module_object_preload, position)
-    except:
-        log.exception("Unable to load module")
-        context.update({
+        'COURSE_TITLE': multicourse_settings.get_course_title(course),
         'init': '',
-            'content': render_to_string("module-error.html", {}),
-        })
-        return render_to_response('courseware.html', context)
+        'content': ''
+    }
 
-    context.update({
-        'init': module_context.get('init_js', ''),
-        'content': module_context['content'],
-    })
+    look_for_module = chapter is not None and section is not None
+    if look_for_module:
+        context.update(get_module_context())
 
     result = render_to_response('courseware.html', context)
     return result
 
 def jump_to(request, probname=None):
     '''
-    Jump to viewing a specific problem.  The problem is specified by a problem name - currently the filename (minus .xml)
-    of the problem.  Maybe this should change to a more generic tag, eg "name" given as an attribute in <problem>.
+    Jump to viewing a specific problem.  The problem is specified by a
+    problem name - currently the filename (minus .xml) of the problem.
+    Maybe this should change to a more generic tag, eg "name" given as
+    an attribute in <problem>.
 
-    We do the jump by (1) reading course.xml to find the first instance of <problem> with the given filename, then
-    (2) finding the parent element of the problem, then (3) rendering that parent element with a specific computed position
-    value (if it is <sequential>).
+    We do the jump by (1) reading course.xml to find the first
+    instance of <problem> with the given filename, then (2) finding
+    the parent element of the problem, then (3) rendering that parent
+    element with a specific computed position value (if it is
+    <sequential>).
 
     '''
     # get coursename if stored
     coursename = multicourse_settings.get_coursename_from_request(request)
 
     # begin by getting course.xml tree
-    xml = content_parser.course_file(request.user,coursename)
+    xml = content_parser.course_file(request.user, coursename)
 
     # look for problem of given name
     pxml = xml.xpath('//problem[@filename="%s"]' % probname)
@@ -279,12 +336,16 @@ def jump_to(request, probname=None):
     section = None
     branch = parent
     for k in range(4):	# max depth of recursion
-        if branch.tag=='section': section = branch.get('name')
-        if branch.tag=='chapter': chapter = branch.get('name')
+        if branch.tag == 'section':
+            section = branch.get('name')
+        if branch.tag == 'chapter':
+            chapter = branch.get('name')
         branch = branch.getparent()
 
     position = None
-    if parent.tag=='sequential':
-        position = parent.index(pxml)+1	# position in sequence
+    if parent.tag == 'sequential':
+        position = parent.index(pxml) + 1	# position in sequence
 
-    return index(request,course=coursename,chapter=chapter,section=section,position=position)
+    return index(request,
+                 course=coursename, chapter=chapter,
+                 section=section, position=position)

lms/djangoapps/student/views.py

@@ -89,7 +89,7 @@ def login_user(request, error=""):
 
 @ensure_csrf_cookie
 def logout_user(request):
-    ''' HTTP request to log in the user. Redirects to marketing page'''
+    ''' HTTP request to log out the user. Redirects to marketing page'''
     logout(request)
     return redirect('/')
 

lms/envs/common.py

@@ -37,6 +37,7 @@ PERFSTATS = False
 MITX_FEATURES = {
     'SAMPLE' : False,
     'USE_DJANGO_PIPELINE' : True,
+    'DISPLAY_HISTOGRAMS_TO_STAFF' : True,
 }
 
 # Used for A/B testing
@@ -287,13 +288,15 @@ PIPELINE_CSS = {
     }
 }
 
+PIPELINE_ALWAYS_RECOMPILE = ['sass/application.scss', 'sass/marketing.scss', 'sass/marketing-ie.scss', 'sass/print.scss']
+
 PIPELINE_JS = {
     'application': {
         'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '') for pth in glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee')],
         'output_filename': 'js/application.js'
     },
     'spec': {
-        'source_filenames': [pth.replace('static/', '') for pth in glob2.glob('static/coffee/spec/**/*.coffee')],
+        'source_filenames': [pth.replace(PROJECT_ROOT / 'static/', '') for pth in glob2.glob(PROJECT_ROOT / 'static/coffee/spec/**/*.coffee')],
         'output_filename': 'js/spec.js'
     }
 }

lms/envs/dev_ike.py

@@ -27,7 +27,8 @@ DEBUG = True
 ENABLE_MULTICOURSE = True     # set to False to disable multicourse display (see lib.util.views.mitxhome)
 QUICKEDIT = True
 
-MITX_FEATURES['USE_DJANGO_PIPELINE'] = False
+# MITX_FEATURES['USE_DJANGO_PIPELINE'] = False
+MITX_FEATURES['DISPLAY_HISTOGRAMS_TO_STAFF'] = False
 
 COURSE_SETTINGS =  {'6.002_Spring_2012': {'number' : '6.002x',
                                           'title'  :  'Circuits and Electronics',

lms/envs/test.py

@@ -20,7 +20,7 @@ INSTALLED_APPS = [
 
 # Nose Test Runner
 INSTALLED_APPS += ['django_nose']
-NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', '--cover-inclusive']
+NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html', '--cover-inclusive', '--cover-html-dir', os.environ['NOSE_COVER_HTML_DIR']]
 for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
     NOSE_ARGS += ['--cover-package', app]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

lms/lib/dogfood/README.md

+This is a library for edx4edx, allowing users to practice writing problems. 

lms/lib/dogfood/check.py

@@ -5,7 +5,7 @@ import string
 import traceback
 
 from django.conf import settings
-import courseware.capa.capa_problem as lcp
+import capa.capa_problem as lcp
 from dogfood.views import update_problem
 
 def GenID(length=8, chars=string.letters + string.digits):
@@ -44,9 +44,9 @@ def check_problem_code(ans,the_lcp,correct_answers,false_answers):
         fp = the_lcp.system.filestore.open('problems/%s.xml' % pfn)
         test_lcp = lcp.LoncapaProblem(fp, '1', system=the_lcp.system)
     
-        if not (test_lcp.grade_answers(correct_answers)['1_2_1']=='correct'):
+        if not (test_lcp.grade_answers(correct_answers).get_correctness('1_2_1')=='correct'):
             is_ok = False
-        if (test_lcp.grade_answers(false_answers)['1_2_1']=='correct'):
+        if (test_lcp.grade_answers(false_answers).get_correctness('1_2_1')=='correct'):
             is_ok = False
     except Exception,err:
         is_ok = False

lms/lib/dogfood/views.py

@@ -21,7 +21,6 @@ from django.http import HttpResponse
 from django.shortcuts import redirect
 from mitxmako.shortcuts import render_to_response, render_to_string
 
-import courseware.capa.calc
 import track.views
 from lxml import etree
 
@@ -34,7 +33,8 @@ from util.cache import cache
 from util.views import accepts
 
 import courseware.content_parser as content_parser
-import courseware.modules
+#import courseware.modules
+import xmodule
 
 log = logging.getLogger("mitx.courseware")
 
@@ -184,7 +184,7 @@ def quickedit(request, id=None, qetemplate='quickedit.html',coursename=None):
                            filestore = OSFS(settings.DATA_DIR + xp),
                            #role = 'staff' if request.user.is_staff else 'student',		# TODO: generalize this
                            )
-        instance=courseware.modules.get_module_class(module)(system, 
+        instance=xmodule.get_module_class(module)(system, 
                                                              xml, 
                                                              id,
                                                              state=None)

lms/static/coffee/fixtures/problem_content.html

@@ -6,6 +6,10 @@
   <section class="action">
     <input type="hidden" name="problem_id" value="1">
 
+    <input type="text" name="input_example_1" id="input_example_1" value="" class="math" />
+    <span id="display_example_1"></span>
+    <span id="input_example_1_dynamath"></span>
+
     <input class="check" type="button" value="Check">
     <input class="reset" type="button" value="Reset">
     <input class="save" type="button" value="Save">

lms/static/coffee/spec/calculator_spec.coffee

@@ -33,16 +33,16 @@ describe 'Calculator', ->
   describe 'toggle', ->
     it 'toggle the calculator and focus the input', ->
       spyOn $.fn, 'focus'
-      @calculator.toggle()
+      @calculator.toggle(jQuery.Event("click"))
 
       expect($('li.calc-main')).toHaveClass('open')
       expect($('#calculator_wrapper #calculator_input').focus).toHaveBeenCalled()
 
     it 'toggle the close button on the calculator button', ->
-      @calculator.toggle()
+      @calculator.toggle(jQuery.Event("click"))
       expect($('.calc')).toHaveClass('closed')
 
-      @calculator.toggle()
+      @calculator.toggle(jQuery.Event("click"))
       expect($('.calc')).not.toHaveClass('closed')
 
   describe 'helpToggle', ->

lms/static/coffee/spec/courseware_spec.coffee

@@ -5,16 +5,6 @@ describe 'Courseware', ->
       Courseware.start()
       expect(window.Navigation).toHaveBeenCalled()
 
-    it 'create the calculator', ->
-      spyOn(window, 'Calculator')
-      Courseware.start()
-      expect(window.Calculator).toHaveBeenCalled()
-
-    it 'creates the FeedbackForm', ->
-      spyOn(window, 'FeedbackForm')
-      Courseware.start()
-      expect(window.FeedbackForm).toHaveBeenCalled()
-
     it 'binds the Logger', ->
       spyOn(Logger, 'bind')
       Courseware.start()

lms/static/coffee/spec/helper.coffee

@@ -30,16 +30,17 @@ jasmine.stubRequests = ->
 
 jasmine.stubYoutubePlayer = ->
   YT.Player = -> jasmine.createSpyObj 'YT.Player', ['cueVideoById', 'getVideoEmbedCode',
-    'getCurrentTime', 'getPlayerState', 'loadVideoById', 'playVideo', 'pauseVideo', 'seekTo']
+    'getCurrentTime', 'getPlayerState', 'getVolume', 'setVolume', 'loadVideoById',
+    'playVideo', 'pauseVideo', 'seekTo']
 
-jasmine.stubVideoPlayer = (context, enableParts) ->
+jasmine.stubVideoPlayer = (context, enableParts, createPlayer=true) ->
   enableParts = [enableParts] unless $.isArray(enableParts)
 
   suite = context.suite
   currentPartName = suite.description while suite = suite.parentSuite
   enableParts.push currentPartName
 
-  for part in ['VideoCaption', 'VideoSpeedControl', 'VideoProgressSlider']
+  for part in ['VideoCaption', 'VideoSpeedControl', 'VideoVolumeControl', 'VideoProgressSlider']
     unless $.inArray(part, enableParts) >= 0
       spyOn window, part
 
@@ -48,6 +49,7 @@ jasmine.stubVideoPlayer = (context, enableParts) ->
   YT.Player = undefined
   context.video = new Video 'example', '.75:abc123,1.0:def456'
   jasmine.stubYoutubePlayer()
+  if createPlayer
     return new VideoPlayer context.video
 
 spyOn(window, 'onunload')

lms/static/coffee/spec/histogram_spec.coffee

@@ -11,7 +11,7 @@ describe 'Histogram', ->
 
   describe 'calculate', ->
     beforeEach ->
-      @histogram = new Histogram(1, [[1, 1], [2, 2], [3, 3]])
+      @histogram = new Histogram(1, [[null, 1], [1, 1], [2, 2], [3, 3]])
 
     it 'store the correct value for data', ->
       expect(@histogram.data).toEqual [[1, Math.log(2)], [2, Math.log(3)], [3, Math.log(4)]]

lms/static/coffee/spec/modules/problem_spec.coffee

 describe 'Problem', ->
   beforeEach ->
     # Stub MathJax
-    window.MathJax = { Hub: { Queue: -> } }
+    window.MathJax =
+      Hub: jasmine.createSpyObj('MathJax.Hub', ['getAllJax', 'Queue'])
+      Callback: jasmine.createSpyObj('MathJax.Callback', ['After'])
+    @stubbedJax = root: jasmine.createSpyObj('jax.root', ['toMathML'])
+    MathJax.Hub.getAllJax.andReturn [@stubbedJax]
     window.update_schematics = ->
 
     loadFixtures 'problem.html'
@@ -25,8 +29,8 @@ describe 'Problem', ->
 
   describe 'bind', ->
     beforeEach ->
-      spyOn MathJax.Hub, 'Queue'
       spyOn window, 'update_schematics'
+      MathJax.Hub.getAllJax.andReturn [@stubbedJax]
       @problem = new Problem 1, '/problem/url/'
 
     it 'set mathjax typeset', ->
@@ -50,6 +54,12 @@ describe 'Problem', ->
     it 'bind the save button', ->
       expect($('section.action input.save')).toHandleWith 'click', @problem.save
 
+    it 'bind the math input', ->
+      expect($('input.math')).toHandleWith 'keyup', @problem.refreshMath
+
+    it 'display the math input', ->
+      expect(@stubbedJax.root.toMathML).toHaveBeenCalled()
+
   describe 'render', ->
     beforeEach ->
       @problem = new Problem 1, '/problem/url/'
@@ -223,6 +233,30 @@ describe 'Problem', ->
       @problem.save()
       expect(window.alert).toHaveBeenCalledWith 'Saved'
 
+  describe 'refreshMath', ->
+    beforeEach ->
+      @problem = new Problem 1, '/problem/url/'
+      @stubbedJax.root.toMathML.andReturn '<MathML>'
+      $('#input_example_1').val 'E=mc^2'
+
+    describe 'when there is no exception', ->
+      beforeEach ->
+        @problem.refreshMath target: $('#input_example_1').get(0)
+
+      it 'should convert and display the MathML object', ->
+        expect(MathJax.Hub.Queue).toHaveBeenCalledWith ['Text', @stubbedJax, 'E=mc^2']
+
+      it 'should display debug output in hidden div', ->
+        expect($('#input_example_1_dynamath')).toHaveValue '<MathML>'
+
+    describe 'when there is an exception', ->
+      beforeEach ->
+        @stubbedJax.root.toMathML.andThrow {restart: true}
+        @problem.refreshMath target: $('#input_example_1').get(0)
+
+      it 'should queue up the exception', ->
+        expect(MathJax.Callback.After).toHaveBeenCalledWith [@problem.refreshMath, @stubbedJax], true
+
   describe 'refreshAnswers', ->
     beforeEach ->
       @problem = new Problem 1, '/problem/url/'

lms/static/coffee/spec/modules/sequence_spec.coffee

@@ -24,7 +24,6 @@ describe 'Sequence', ->
       expect(titles).toEqual ['Video 1', 'Video 2', 'Sample Problem']
 
     it 'bind the page events', ->
-      expect(@sequence.element).toHandleWith 'contentChanged', @sequence.toggleArrows
       expect($('#sequence-list a')).toHandleWith 'click', @sequence.goto
 
     it 'render the active sequence content', ->
@@ -76,6 +75,7 @@ describe 'Sequence', ->
       spyOn $, 'postWithPrefix'
       @sequence = new Sequence '1', @items, 'sequence'
       spyOnEvent @sequence.element, 'contentChanged'
+      spyOn(@sequence, 'toggleArrows').andCallThrough()
 
     describe 'with a different position than the current one', ->
       beforeEach ->
@@ -105,6 +105,9 @@ describe 'Sequence', ->
       it 'update the position', ->
         expect(@sequence.position).toEqual 1
 
+      it 're-update the arrows', ->
+        expect(@sequence.toggleArrows).toHaveBeenCalled()
+
       it 'trigger contentChanged event', ->
         expect('contentChanged').toHaveBeenTriggeredOn @sequence.element
 

lms/static/coffee/spec/modules/video/video_player_spec.coffee

 describe 'VideoPlayer', ->
   beforeEach ->
-    jasmine.stubVideoPlayer @
+    jasmine.stubVideoPlayer @, [], false
 
   afterEach ->
     YT.Player = undefined
@@ -11,7 +11,10 @@ describe 'VideoPlayer', ->
       spyOn YT, 'Player'
       $.fn.qtip.andCallFake ->
         $(this).data('qtip', true)
-      $('.video').append $('<div class="hide-subtitles" />')
+      $('.video').append $('<div class="add-fullscreen" /><div class="hide-subtitles" />')
+
+    describe 'always', ->
+      beforeEach ->
         @player = new VideoPlayer @video
 
       it 'instanticate current time to zero', ->
@@ -67,13 +70,35 @@ describe 'VideoPlayer', ->
         expect($(document)).toHandleWith 'keyup', @player.bindExitFullScreen
 
       it 'bind to fullscreen switching button', ->
+        console.debug $('.add-fullscreen')
         expect($('.add-fullscreen')).toHandleWith 'click', @player.toggleFullScreen
 
     describe 'when not on a touch based device', ->
+      beforeEach ->
+        spyOn(window, 'onTouchBasedDevice').andReturn false
+        $('.add-fullscreen, .hide-subtitles').removeData 'qtip'
+        @player = new VideoPlayer @video
+
       it 'add the tooltip to fullscreen and subtitle button', ->
         expect($('.add-fullscreen')).toHaveData 'qtip'
         expect($('.hide-subtitles')).toHaveData 'qtip'
 
+      it 'create video volume control', ->
+        expect(window.VideoVolumeControl).toHaveBeenCalledWith @player
+
+    describe 'when on a touch based device', ->
+      beforeEach ->
+        spyOn(window, 'onTouchBasedDevice').andReturn true
+        $('.add-fullscreen, .hide-subtitles').removeData 'qtip'
+        @player = new VideoPlayer @video
+
+      it 'does not add the tooltip to fullscreen and subtitle button', ->
+        expect($('.add-fullscreen')).not.toHaveData 'qtip'
+        expect($('.hide-subtitles')).not.toHaveData 'qtip'
+
+      it 'does not create video volume control', ->
+        expect(window.VideoVolumeControl).not.toHaveBeenCalled()
+
   describe 'onReady', ->
     beforeEach ->
       @video.embed()
@@ -387,3 +412,17 @@ describe 'VideoPlayer', ->
 
     it 'delegate to the video', ->
       expect(@player.currentSpeed()).toEqual '3.0'
+
+  describe 'volume', ->
+    beforeEach ->
+      @player = new VideoPlayer @video
+      @player.player.getVolume.andReturn 42
+
+    describe 'without value', ->
+      it 'return current volume', ->
+        expect(@player.volume()).toEqual 42
+
+    describe 'with value', ->
+      it 'set player volume', ->
+        @player.volume(60)
+        expect(@player.player.setVolume).toHaveBeenCalledWith(60)

lms/static/coffee/spec/modules/video/video_progress_slider_spec.coffee

@@ -18,7 +18,7 @@ describe 'VideoProgressSlider', ->
           stop: @slider.onStop
 
       it 'build the seek handle', ->
-        expect(@slider.handle).toBe '.ui-slider-handle'
+        expect(@slider.handle).toBe '.slider .ui-slider-handle'
         expect($.fn.qtip).toHaveBeenCalledWith
           content: "0:00"
           position:

lms/static/coffee/spec/modules/video/video_speed_control_spec.coffee

@@ -3,8 +3,6 @@ describe 'VideoSpeedControl', ->
     @player = jasmine.stubVideoPlayer @
     $('.speeds').remove()
 
-  afterEach ->
-
   describe 'constructor', ->
     describe 'always', ->
       beforeEach ->

lms/static/coffee/spec/modules/video/video_volume_control_spec.coffee

+describe 'VideoVolumeControl', ->
+  beforeEach ->
+    @player = jasmine.stubVideoPlayer @
+    $('.volume').remove()
+
+  describe 'constructor', ->
+    beforeEach ->
+      spyOn($.fn, 'slider')
+      @volumeControl = new VideoVolumeControl @player
+
+    it 'initialize previousVolume to 100', ->
+      expect(@volumeControl.previousVolume).toEqual 100
+
+    it 'render the volume control', ->
+      expect($('.secondary-controls').html()).toContain """
+        <div class="volume">
+          <a href="#"></a>
+          <div class="volume-slider-container">
+            <div class="volume-slider"></div>
+          </div>
+        </div>
+      """
+
+    it 'create the slider', ->
+      expect($.fn.slider).toHaveBeenCalledWith
+        orientation: "vertical"
+        range: "min"
+        min: 0
+        max: 100
+        value: 100
+        change: @volumeControl.onChange
+        slide: @volumeControl.onChange
+
+    it 'bind the volume control', ->
+      expect($(@player)).toHandleWith 'ready', @volumeControl.onReady
+      expect($('.volume>a')).toHandleWith 'click', @volumeControl.toggleMute
+
+      expect($('.volume')).not.toHaveClass 'open'
+      $('.volume').mouseenter()
+      expect($('.volume')).toHaveClass 'open'
+      $('.volume').mouseleave()
+      expect($('.volume')).not.toHaveClass 'open'
+
+  describe 'onReady', ->
+    beforeEach ->
+      @volumeControl = new VideoVolumeControl @player
+      spyOn $.fn, 'slider'
+      spyOn(@player, 'volume').andReturn 60
+      @volumeControl.onReady()
+
+    it 'set the max value of the slider', ->
+      expect($.fn.slider).toHaveBeenCalledWith 'option', 'max', 60
+
+  describe 'onChange', ->
+    beforeEach ->
+      spyOn @player, 'volume'
+      @volumeControl = new VideoVolumeControl @player
+
+    describe 'when the new volume is more than 0', ->
+      beforeEach ->
+        @volumeControl.onChange undefined, value: 60
+
+      it 'set the player volume', ->
+        expect(@player.volume).toHaveBeenCalledWith 60
+
+      it 'remote muted class', ->
+        expect($('.volume')).not.toHaveClass 'muted'
+
+    describe 'when the new volume is 0', ->
+      beforeEach ->
+        @volumeControl.onChange undefined, value: 0
+
+      it 'set the player volume', ->
+        expect(@player.volume).toHaveBeenCalledWith 0
+
+      it 'add muted class', ->
+        expect($('.volume')).toHaveClass 'muted'
+
+  describe 'toggleMute', ->
+    beforeEach ->
+      spyOn @player, 'volume'
+      @volumeControl = new VideoVolumeControl @player
+
+    describe 'when the current volume is more than 0', ->
+      beforeEach ->
+        @player.volume.andReturn 60
+        @volumeControl.toggleMute()
+
+      it 'save the previous volume', ->
+        expect(@volumeControl.previousVolume).toEqual 60
+
+      it 'set the player volume', ->
+        expect(@player.volume).toHaveBeenCalledWith 0
+
+    describe 'when the current volume is 0', ->
+      beforeEach ->
+        @player.volume.andReturn 0
+        @volumeControl.previousVolume = 60
+        @volumeControl.toggleMute()
+
+      it 'set the player volume to previous volume', ->
+        expect(@player.volume).toHaveBeenCalledWith 60

lms/static/coffee/src/calculator.coffee

@@ -6,7 +6,8 @@ class @Calculator
     $('div.help-wrapper a').hover(@helpToggle).click (e) ->
       e.preventDefault()
 
-  toggle: ->
+  toggle: (event) ->
+    event.preventDefault()
     $('li.calc-main').toggleClass 'open'
     $('#calculator_wrapper #calculator_input').focus()
     if $('.calc.closed').length

lms/static/coffee/src/courseware.coffee

@@ -4,8 +4,6 @@ class @Courseware
   constructor: ->
     Courseware.prefix = $("meta[name='path_prefix']").attr('content')
     new Navigation
-    new Calculator
-    new FeedbackForm
     Logger.bind()
     @bind()
     @render()

lms/static/coffee/src/histogram.coffee

@@ -8,6 +8,7 @@ class @Histogram
 
   calculate: ->
     for [score, count] in @rawData
+      continue if score == null
       log_count = Math.log(count + 1)
       @data.push [score, log_count]
       @xTicks.push [score, score.toString()]

lms/static/coffee/src/main.coffee

@@ -17,6 +17,8 @@ $ ->
   $("a[rel*=leanModal]").leanModal()
   $('#csrfmiddlewaretoken').attr 'value', $.cookie('csrftoken')
 
+  new Calculator
+  new FeedbackForm
   if $('body').hasClass('courseware')
     Courseware.start()
 

lms/static/coffee/src/modules/problem.coffee

@@ -15,6 +15,7 @@ class @Problem
     @$('section.action input.reset').click @reset
     @$('section.action input.show').click @show
     @$('section.action input.save').click @save
+    @$('input.math').keyup(@refreshMath).each(@refreshMath)
 
   render: (content) ->
     if content
@@ -44,14 +45,14 @@ class @Problem
         $.each response, (key, value) =>
           if $.isArray(value)
             for choice in value
-              @$("label[for='input_#{key}_#{choice}']").attr
-                correct_answer: 'true'
+              @$("label[for='input_#{key}_#{choice}']").attr correct_answer: 'true'
           else
-            @$("#answer_#{key}").text(value)
+            @$("#answer_#{key}, #solution_#{key}").html(value)
+        MathJax.Hub.Queue ["Typeset", MathJax.Hub]
         @$('.show').val 'Hide Answer'
         @element.addClass 'showed'
     else
-      @$('[id^=answer_]').text ''
+      @$('[id^=answer_], [id^=solution_]').text ''
       @$('[correct_answer]').attr correct_answer: null
       @element.removeClass 'showed'
       @$('.show').val 'Show Answer'
@@ -62,6 +63,20 @@ class @Problem
       if response.success
         alert 'Saved'
 
+  refreshMath: (event, element) =>
+    element = event.target unless element
+    target = "display_#{element.id.replace(/^input_/, '')}"
+
+    if jax = MathJax.Hub.getAllJax(target)[0]
+      MathJax.Hub.Queue ['Text', jax, $(element).val()]
+
+      try
+        output = jax.root.toMathML ''
+        $("##{element.id}_dynamath").val(output)
+      catch exception
+        throw exception unless exception.restart
+        MathJax.Callback.After [@refreshMath, jax], exception.restart
+
   refreshAnswers: =>
     @$('input.schematic').each (index, element) ->
       element.schematic.update_value()

lms/static/coffee/src/modules/sequence.coffee

@@ -9,7 +9,6 @@ class @Sequence
     $(selector, @element)
 
   bind: ->
-    @element.bind 'contentChanged', @toggleArrows
     @$('#sequence-list a').click @goto
 
   buildNavigation: ->
@@ -43,6 +42,7 @@ class @Sequence
 
       MathJax.Hub.Queue(["Typeset", MathJax.Hub])
       @position = new_position
+      @toggleArrows()
       @element.trigger 'contentChanged'
 
   goto: (event) =>

lms/static/coffee/src/modules/video/video_player.coffee

@@ -30,6 +30,7 @@ class @VideoPlayer
   render: ->
     new VideoControl @
     new VideoCaption @, @video.youtubeId('1.0')
+    new VideoVolumeControl @ unless onTouchBasedDevice()
     new VideoSpeedControl @, @video.speeds
     new VideoProgressSlider @
     @player = new YT.Player @video.id,
@@ -132,3 +133,9 @@ class @VideoPlayer
 
   currentSpeed: ->
     @video.speed
+
+  volume: (value) ->
+    if value?
+      @player.setVolume value
+    else
+      @player.getVolume()

lms/static/coffee/src/modules/video/video_progress_slider.coffee

@@ -17,7 +17,7 @@ class @VideoProgressSlider
     @buildHandle()
 
   buildHandle: ->
-    @handle = @$('.ui-slider-handle')
+    @handle = @$('.slider .ui-slider-handle')
     @handle.qtip
       content: "#{Time.format(@slider.slider('value'))}"
       position:

lms/static/coffee/src/modules/video/video_volume_control.coffee

+class @VideoVolumeControl
+  constructor: (@player) ->
+    @previousVolume = 100
+    @render()
+    @bind()
+
+  $: (selector) ->
+    @player.$(selector)
+
+  bind: ->
+    $(@player).bind('ready', @onReady)
+    @$('.volume').mouseenter ->
+      $(this).addClass('open')
+    @$('.volume').mouseleave ->
+      $(this).removeClass('open')
+    @$('.volume>a').click(@toggleMute)
+
+  render: ->
+    @$('.secondary-controls').prepend """
+      <div class="volume">
+        <a href="#"></a>
+        <div class="volume-slider-container">
+          <div class="volume-slider"></div>
+        </div>
+      </div>
+      """
+    @slider = @$('.volume-slider').slider
+      orientation: "vertical"
+      range: "min"
+      min: 0
+      max: 100
+      value: 100
+      change: @onChange
+      slide: @onChange
+
+  onReady: =>
+    @slider.slider 'option', 'max', @player.volume()
+
+  onChange: (event, ui) =>
+    @player.volume ui.value
+    @$('.secondary-controls .volume').toggleClass 'muted', ui.value == 0
+
+  toggleMute: =>
+    if @player.volume() > 0
+      @previousVolume = @player.volume()
+      @slider.slider 'option', 'value', 0
+    else
+      @slider.slider 'option', 'value', @previousVolume

lms/static/sass/courseware/_video.scss

@@ -286,6 +286,87 @@ section.course-content {
             }
           }
 
+          div.volume {
+            float: left;
+            position: relative;
+
+            &.open {
+              .volume-slider-container {
+                display: block;
+                opacity: 1;
+              }
+            }
+
+            &.muted {
+              &>a {
+                background: url('../images/mute.png') 10px center no-repeat;
+              }
+            }
+
+            > a {
+              background: url('../images/volume.png') 10px center no-repeat;
+              border-right: 1px solid #000;
+              @include box-shadow(1px 0 0 #555, inset 1px 0 0 #555);
+              @include clearfix();
+              color: #fff;
+              cursor: pointer;
+              display: block;
+              height: 46px;
+              margin-right: 0;
+              padding-left: 15px;
+              position: relative;
+              @include transition();
+              -webkit-font-smoothing: antialiased;
+              width: 30px;
+
+              &:hover, &:active, &:focus {
+                background-color: #444;
+              }
+            }
+
+            .volume-slider-container {
+              @include box-shadow(inset 1px 0 0 #555, 0 3px 0 #444);
+              @include transition();
+              background-color: #444;
+              border: 1px solid #000;
+              bottom: 46px;
+              display: none;
+              opacity: 0;
+              position: absolute;
+              width: 45px;
+              height: 125px;
+              margin-left: -1px;
+              z-index: 10;
+
+              .volume-slider {
+                height: 100px;
+                border: 0;
+                width: 5px;
+                margin: 14px auto;
+                background: #666;
+                border: 1px solid #000;
+                @include box-shadow(0 1px 0 #333);
+
+                a.ui-slider-handle {
+                  background: $mit-red url(../images/slider-handle.png) center center no-repeat;
+                  @include background-size(50%);
+                  border: 1px solid darken($mit-red, 20%);
+                  @include border-radius(15px);
+                  @include box-shadow(inset 0 1px 0 lighten($mit-red, 10%));
+                  cursor: pointer;
+                  height: 15px;
+                  left: -6px;
+                  @include transition(height 2.0s ease-in-out, width 2.0s ease-in-out);
+                  width: 15px;
+                }
+
+                .ui-slider-range {
+                  background: #ddd;
+                }
+              }
+            }
+          }
+
           a.add-fullscreen {
             background: url(../images/fullscreen.png) center no-repeat;
             border-right: 1px solid #000;

lms/templates/mathjax_include.html

@@ -5,37 +5,8 @@
 ##
 ## This enables ASCIIMathJAX, and is used by js_textbox
 
- <script type="text/x-mathjax-config">
-
-//  (function () {
-    var QUEUE = MathJax.Hub.queue;  // shorthand for the queue
-    var math = null;
-    var jaxset = {};                // associative array of the element jaxs for the math output.
-    var mmlset = {};		// associative array of mathml from each jax
-
-      // constructs mathML of the specified jax element
-        function toMathML(jax,callback) {
-          var mml;
-          try {
-            mml = jax.root.toMathML("");
-          } catch(err) {
-            if (!err.restart) {throw err} // an actual error
-            return MathJax.Callback.After([toMathML,jax,callback],err.restart);
-          }
-          MathJax.Callback(callback)(mml);
-        }
-
-      // function to queue in MathJax to get put the MathML expression in in the right document element
-      function UpdateMathML(jax,id) {
-              	toMathML(jax,function (mml) {
-                    // document.getElementById(id+'_dynamath').value=math.originalText+ "\n\n=>\n\n"+ mml;
-                    delem = document.getElementById("input_" + id + "_dynamath");
-		    if (delem) { delem.value=mml; };
-		    mmlset[id] = mml;
-              	})
-      }
-
-MathJax.Hub.Config({
+<script type="text/x-mathjax-config">
+  MathJax.Hub.Config({
     tex2jax: {
       inlineMath: [
         ["\\(","\\)"],
@@ -46,41 +17,10 @@ MathJax.Hub.Config({
         ['[mathjax]','[/mathjax]']
       ]
     }
-});
-
-    //
-    //  The onchange event handler that typesets the
-    //  math entered by the user
-    //
-    window.UpdateMath = function (Am,id) {
-      QUEUE.Push(["Text",jaxset[id],Am]);
-      QUEUE.Push(UpdateMathML(jaxset[id],id));
-    }
-
-//  })();
-
-function DoUpdateMath(inputId) {
-  var str = document.getElementById("input_"+inputId).value;
-
-  // make sure the input field is in the jaxset
-  if ($.inArray(inputId,jaxset) == -1){
-      //alert('missing '+inputId);
-    if (document.getElementById("display_" + inputId)){
-	  MathJax.Hub.queue.Push(function () {
-	      math = MathJax.Hub.getAllJax("display_" + inputId)[0];
-	      if (math){
-		  jaxset[inputId] = math;
-	      }
   });
-      };
- }
-
-  UpdateMath(str,inputId)
-}
-
-  </script>
+</script>
 
-  <!-- This must appear after all mathjax-config blocks, so it is after the imports from the other templates.
+<!-- This must appear after all mathjax-config blocks, so it is after the imports from the other templates.
      It can't be run through static.url because MathJax uses crazy url introspection to do lazy loading of
      MathJax extension libraries -->
-  <script type="text/javascript" src="/static/js/mathjax-MathJax-c9db6ac/MathJax.js?config=TeX-MML-AM_HTMLorMML-full"></script>
+<script type="text/javascript" src="/static/js/mathjax-MathJax-c9db6ac/MathJax.js?config=TeX-MML-AM_HTMLorMML-full"></script>

rakefile

@@ -44,17 +44,17 @@ task :default => [:pep8, :pylint, :test]
 directory REPORT_DIR
 directory LMS_REPORT_DIR
 
-desc "Run pep8 on all of djangoapps"
-task :pep8 => LMS_REPORT_DIR do
-    sh("pep8 --ignore=E501 lms/djangoapps | tee #{LMS_REPORT_DIR}/pep8.report")
+desc "Run pep8 on all libraries"
+task :pep8 => REPORT_DIR do
+    sh("pep8 --ignore=E501 lms/djangoapps common/lib/* | tee #{REPORT_DIR}/pep8.report")
 end
 
-desc "Run pylint on all of djangoapps"
-task :pylint => LMS_REPORT_DIR do
-    ENV['PYTHONPATH'] = 'lms/djangoapps'
-    Dir["lms/djangoapps/*"].each do |app|
+desc "Run pylint on all libraries"
+task :pylint => REPORT_DIR do
+    Dir["lms/djangoapps/*", "common/lib/*"].each do |app|
+        ENV['PYTHONPATH'] = File.dirname(app)
         app = File.basename(app)
-        sh("pylint -f parseable #{app} | tee #{LMS_REPORT_DIR}/#{app}.pylint.report")
+        sh("pylint --rcfile=.pylintrc -f parseable #{app} | tee #{REPORT_DIR}/#{app}.pylint.report")
     end
 end
 
@@ -66,6 +66,7 @@ end
     desc "Run all django tests on our djangoapps for the #{system}"
     task task_name => report_dir do
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
+        ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover")
         sh(django_admin(system, :test, 'test', *Dir["#{system}/djangoapps/*"].each))
     end
     task :test => task_name
@@ -83,13 +84,13 @@ end
 Dir["common/lib/*"].each do |lib|
     task_name = "test_#{lib}"
 
-    report_dir = File.join(REPORT_DIR, task_name)
+    report_dir = File.join(REPORT_DIR, task_name.gsub('/', '_'))
     directory report_dir
 
     desc "Run tests for common lib #{lib}"
-    task task_name do
+    task task_name => report_dir do
         ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
-        sh("nosetests #{lib}")
+        sh("nosetests #{lib} --cover-erase --with-xunit --with-xcoverage --cover-html --cover-inclusive --cover-package #{File.basename(lib)} --cover-html-dir #{File.join(report_dir, "cover")}")
     end
     task :test => task_name
 end

requirements.txt

@@ -25,3 +25,6 @@ newrelic
 glob2
 pymongo
 -e common/lib/xmodule
+django_nose
+nosexcover
+rednose