Merge pull request #1574 from MITx/fix/will/cs191_matrix_style_bug

Fix/will/cs191 matrix style bug

common/lib/capa/capa/capa_problem.py

@@ -510,7 +510,9 @@ class LoncapaProblem(object):
 
         # let each Response render itself
         if problemtree in self.responders:
-            return self.responders[problemtree].render_html(self._extract_html)
+            overall_msg = self.correct_map.get_overall_message()
+            return self.responders[problemtree].render_html(self._extract_html,
+                                                response_msg=overall_msg)
 
         # let each custom renderer render itself:
         if problemtree.tag in customrender.registry.registered_tags():

common/lib/capa/capa/correctmap.py

@@ -27,6 +27,7 @@ class CorrectMap(object):
         self.cmap = dict()
         self.items = self.cmap.items
         self.keys = self.cmap.keys
+        self.overall_message = ""
         self.set(*args, **kwargs)
 
     def __getitem__(self, *args, **kwargs):
@@ -104,16 +105,21 @@ class CorrectMap(object):
         return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
 
     def get_queuetime_str(self, answer_id):
-        return self.cmap[answer_id]['queuestate']['time']
+        if self.cmap[answer_id]['queuestate']:
+            return self.cmap[answer_id]['queuestate']['time']
+        else:
+            return None
 
     def get_npoints(self, answer_id):
-        npoints = self.get_property(answer_id, 'npoints')
-        if npoints is not None:
-            return npoints
-        elif self.is_correct(answer_id):
-            return 1
-         # if not correct and no points have been assigned, return 0
-        return 0
+        """ Return the number of points for an answer:
+            If the answer is correct, return the assigned
+            number of points (default: 1 point)
+            Otherwise, return 0 points """
+        if self.is_correct(answer_id):
+            npoints = self.get_property(answer_id, 'npoints')
+            return npoints if npoints is not None else 1
+        else:
+            return 0
 
     def set_property(self, answer_id, property, value):
         if answer_id in self.cmap:
@@ -153,3 +159,15 @@ class CorrectMap(object):
         if not isinstance(other_cmap, CorrectMap):
             raise Exception('CorrectMap.update called with invalid argument %s' % other_cmap)
         self.cmap.update(other_cmap.get_dict())
+        self.set_overall_message(other_cmap.get_overall_message())
+
+
+    def set_overall_message(self, message_str):
+        """ Set a message that applies to the question as a whole,
+            rather than to individual inputs. """
+        self.overall_message = str(message_str) if message_str else ""
+
+    def get_overall_message(self):
+        """ Retrieve a message that applies to the question as a whole.
+        If no message is available, returns the empty string """
+        return self.overall_message

common/lib/capa/capa/responsetypes.py

@@ -174,13 +174,14 @@ class LoncapaResponse(object):
         '''
         return sum(self.maxpoints.values())
 
-    def render_html(self, renderer):
+    def render_html(self, renderer, response_msg=''):
         '''
         Return XHTML Element tree representation of this Response.
 
         Arguments:
 
           - renderer : procedure which produces HTML given an ElementTree
+          - response_msg: a message displayed at the end of the Response
         '''
         # render ourself as a <span> + our content
         tree = etree.Element('span')
@@ -195,6 +196,11 @@ class LoncapaResponse(object):
             if item_xhtml is not None:
                 tree.append(item_xhtml)
         tree.tail = self.xml.tail
+
+        # Add a <div> for the message at the end of the response
+        if response_msg:
+            tree.append(self._render_response_msg_html(response_msg))
+
         return tree
 
     def evaluate_answers(self, student_answers, old_cmap):
@@ -319,6 +325,29 @@ class LoncapaResponse(object):
     def __unicode__(self):
         return u'LoncapaProblem Response %s' % self.xml.tag
 
+    def _render_response_msg_html(self, response_msg):
+        """ Render a <div> for a message that applies to the entire response.
+
+        *response_msg* is a string, which may contain XHTML markup
+        
+        Returns an etree element representing the response message <div> """
+        # First try wrapping the text in a <div> and parsing
+        # it as an XHTML tree
+        try:
+            response_msg_div = etree.XML('<div>%s</div>' % str(response_msg))
+
+        # If we can't do that, create the <div> and set the message
+        # as the text of the <div>
+        except:
+            response_msg_div = etree.Element('div')
+            response_msg_div.text = str(response_msg)
+
+
+        # Set the css class of the message <div>
+        response_msg_div.set("class", "response_message")
+
+        return response_msg_div
+
 
 #-----------------------------------------------------------------------------
 
@@ -965,6 +994,7 @@ def sympy_check2():
         # not expecting 'unknown's
         correct = ['unknown'] * len(idset)
         messages = [''] * len(idset)
+        overall_message = ""
 
         # put these in the context of the check function evaluator
         # note that this doesn't help the "cfn" version - only the exec version
@@ -996,6 +1026,10 @@ def sympy_check2():
             # the list of messages to be filled in by the check function
             'messages': messages,
 
+            # a message that applies to the entire response
+            # instead of a particular input
+            'overall_message': overall_message,
+
             # any options to be passed to the cfn
             'options': self.xml.get('options'),
             'testdat': 'hello world',
@@ -1010,6 +1044,7 @@ def sympy_check2():
                 exec self.code in self.context['global_context'], self.context
                 correct = self.context['correct']
                 messages = self.context['messages']
+                overall_message = self.context['overall_message']
             except Exception as err:
                 print "oops in customresponse (code) error %s" % err
                 print "context = ", self.context
@@ -1044,34 +1079,100 @@ def sympy_check2():
                 log.error(traceback.format_exc())
                 raise Exception("oops in customresponse (cfn) error %s" % err)
             log.debug("[courseware.capa.responsetypes.customresponse.get_score] ret = %s" % ret)
+
             if type(ret) == dict:
-                correct = ['correct'] * len(idset) if ret['ok'] else ['incorrect'] * len(idset)
-                msg = ret['msg']
-
-                if 1:
-                    # try to clean up message html
-                    msg = '<html>' + msg + '</html>'
-                    msg = msg.replace('&#60;', '&lt;')
-                    #msg = msg.replace('&lt;','<')
-                    msg = etree.tostring(fromstring_bs(msg, convertEntities=None),
-                                         pretty_print=True)
-                    #msg = etree.tostring(fromstring_bs(msg),pretty_print=True)
-                    msg = msg.replace('&#13;', '')
-                    #msg = re.sub('<html>(.*)</html>','\\1',msg,flags=re.M|re.DOTALL)   # python 2.7
-                    msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
-
-                messages[0] = msg
+
+                # One kind of dictionary the check function can return has the
+                # form {'ok': BOOLEAN, 'msg': STRING}
+                # If there are multiple inputs, they all get marked
+                # to the same correct/incorrect value
+                if 'ok' in ret:
+                    correct = ['correct'] * len(idset) if ret['ok'] else ['incorrect'] * len(idset)
+                    msg = ret.get('msg', None)
+                    msg = self.clean_message_html(msg)
+
+                    # If there is only one input, apply the message to that input
+                    # Otherwise, apply the message to the whole problem
+                    if len(idset) > 1:
+                        overall_message = msg
+                    else:
+                        messages[0] = msg
+
+
+                # Another kind of dictionary the check function can return has
+                # the form:
+                # {'overall_message': STRING,
+                #  'input_list': [{ 'ok': BOOLEAN, 'msg': STRING }, ...] }
+                # 
+                # This allows the function to return an 'overall message'
+                # that applies to the entire problem, as well as correct/incorrect
+                # status and messages for individual inputs
+                elif 'input_list' in ret:
+                    overall_message = ret.get('overall_message', '')
+                    input_list = ret['input_list']
+
+                    correct = []
+                    messages = []
+                    for input_dict in input_list:
+                        correct.append('correct' if input_dict['ok'] else 'incorrect')
+                        msg = self.clean_message_html(input_dict['msg']) if 'msg' in input_dict else None
+                        messages.append(msg)
+
+                # Otherwise, we do not recognize the dictionary
+                # Raise an exception
+                else:
+                    log.error(traceback.format_exc())
+                    raise Exception("CustomResponse: check function returned an invalid dict")
+
+            # The check function can return a boolean value,
+            # indicating whether all inputs should be marked
+            # correct or incorrect
             else:
                 correct = ['correct'] * len(idset) if ret else ['incorrect'] * len(idset)
 
         # build map giving "correct"ness of the answer(s)
         correct_map = CorrectMap()
+
+        overall_message = self.clean_message_html(overall_message)
+        correct_map.set_overall_message(overall_message)
+
         for k in range(len(idset)):
             npoints = self.maxpoints[idset[k]] if correct[k] == 'correct' else 0
             correct_map.set(idset[k], correct[k], msg=messages[k],
                             npoints=npoints)
         return correct_map
 
+    def clean_message_html(self, msg):
+
+        # If *msg* is an empty string, then the code below
+        # will return "</html>".  To avoid this, we first check
+        # that *msg* is a non-empty string.
+        if msg:
+
+            # When we parse *msg* using etree, there needs to be a root
+            # element, so we wrap the *msg* text in <html> tags
+            msg = '<html>' + msg + '</html>'
+
+            # Replace < characters
+            msg = msg.replace('&#60;', '&lt;')
+
+            # Use etree to prettify the HTML
+            msg = etree.tostring(fromstring_bs(msg, convertEntities=None),
+                                 pretty_print=True)
+
+            msg = msg.replace('&#13;', '')
+
+            # Remove the <html> tags we introduced earlier, so we're
+            # left with just the prettified message markup
+            msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
+
+            # Strip leading and trailing whitespace
+            return msg.strip()
+
+        # If we start with an empty string, then return an empty string
+        else:
+            return ""
+
     def get_answers(self):
         '''
         Give correct answer expected for this response.

common/lib/capa/capa/tests/test_correctmap.py

+import unittest
+from capa.correctmap import CorrectMap
+import datetime
+
+class CorrectMapTest(unittest.TestCase):
+
+    def setUp(self):
+        self.cmap = CorrectMap()
+
+    def test_set_input_properties(self):
+
+        # Set the correctmap properties for two inputs
+        self.cmap.set(answer_id='1_2_1',
+                    correctness='correct',
+                    npoints=5,
+                    msg='Test message',
+                    hint='Test hint',
+                    hintmode='always',
+                    queuestate={'key':'secretstring',
+                                'time':'20130228100026'})
+
+        self.cmap.set(answer_id='2_2_1',
+                    correctness='incorrect',
+                    npoints=None,
+                    msg=None,
+                    hint=None,
+                    hintmode=None,
+                    queuestate=None)
+
+        # Assert that each input has the expected properties
+        self.assertTrue(self.cmap.is_correct('1_2_1'))
+        self.assertFalse(self.cmap.is_correct('2_2_1'))
+
+        self.assertEqual(self.cmap.get_correctness('1_2_1'), 'correct')
+        self.assertEqual(self.cmap.get_correctness('2_2_1'), 'incorrect')
+
+        self.assertEqual(self.cmap.get_npoints('1_2_1'), 5)
+        self.assertEqual(self.cmap.get_npoints('2_2_1'), 0)
+
+        self.assertEqual(self.cmap.get_msg('1_2_1'), 'Test message')
+        self.assertEqual(self.cmap.get_msg('2_2_1'), None)
+
+        self.assertEqual(self.cmap.get_hint('1_2_1'), 'Test hint')
+        self.assertEqual(self.cmap.get_hint('2_2_1'), None)
+
+        self.assertEqual(self.cmap.get_hintmode('1_2_1'), 'always')
+        self.assertEqual(self.cmap.get_hintmode('2_2_1'), None)
+
+        self.assertTrue(self.cmap.is_queued('1_2_1'))
+        self.assertFalse(self.cmap.is_queued('2_2_1'))
+
+        self.assertEqual(self.cmap.get_queuetime_str('1_2_1'), '20130228100026')
+        self.assertEqual(self.cmap.get_queuetime_str('2_2_1'), None)
+
+        self.assertTrue(self.cmap.is_right_queuekey('1_2_1', 'secretstring'))
+        self.assertFalse(self.cmap.is_right_queuekey('1_2_1', 'invalidstr'))
+        self.assertFalse(self.cmap.is_right_queuekey('1_2_1', ''))
+        self.assertFalse(self.cmap.is_right_queuekey('1_2_1', None))
+
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', 'secretstring'))
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', 'invalidstr'))
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', ''))
+        self.assertFalse(self.cmap.is_right_queuekey('2_2_1', None))
+
+
+    def test_get_npoints(self):
+        # Set the correctmap properties for 4 inputs
+        # 1) correct, 5 points
+        # 2) correct, None points
+        # 3) incorrect, 5 points
+        # 4) incorrect, None points
+        # 5) correct, 0 points
+        self.cmap.set(answer_id='1_2_1',
+                    correctness='correct',
+                    npoints=5)
+
+        self.cmap.set(answer_id='2_2_1',
+                    correctness='correct',
+                    npoints=None)
+
+        self.cmap.set(answer_id='3_2_1',
+                    correctness='incorrect',
+                    npoints=5)
+
+        self.cmap.set(answer_id='4_2_1',
+                    correctness='incorrect',
+                    npoints=None)
+
+        self.cmap.set(answer_id='5_2_1',
+                    correctness='correct',
+                    npoints=0)
+
+        # Assert that we get the expected points
+        # If points assigned and correct --> npoints
+        # If no points assigned and correct --> 1 point
+        # Otherwise --> 0 points
+        self.assertEqual(self.cmap.get_npoints('1_2_1'), 5)
+        self.assertEqual(self.cmap.get_npoints('2_2_1'), 1)
+        self.assertEqual(self.cmap.get_npoints('3_2_1'), 0)
+        self.assertEqual(self.cmap.get_npoints('4_2_1'), 0)
+        self.assertEqual(self.cmap.get_npoints('5_2_1'), 0)
+
+
+    def test_set_overall_message(self):
+
+        # Default is an empty string string
+        self.assertEqual(self.cmap.get_overall_message(), "")
+
+        # Set a message that applies to the whole question
+        self.cmap.set_overall_message("Test message")
+
+        # Retrieve the message
+        self.assertEqual(self.cmap.get_overall_message(), "Test message")
+
+        # Setting the message to None --> empty string
+        self.cmap.set_overall_message(None)
+        self.assertEqual(self.cmap.get_overall_message(), "")
+
+    def test_update_from_correctmap(self):
+        # Initialize a CorrectMap with some properties
+        self.cmap.set(answer_id='1_2_1',
+                    correctness='correct',
+                    npoints=5,
+                    msg='Test message',
+                    hint='Test hint',
+                    hintmode='always',
+                    queuestate={'key':'secretstring',
+                                'time':'20130228100026'})
+
+        self.cmap.set_overall_message("Test message")
+
+        # Create a second cmap, then update it to have the same properties
+        # as the first cmap
+        other_cmap = CorrectMap()
+        other_cmap.update(self.cmap)
+        
+        # Assert that it has all the same properties
+        self.assertEqual(other_cmap.get_overall_message(),
+                        self.cmap.get_overall_message())
+
+        self.assertEqual(other_cmap.get_dict(),
+                        self.cmap.get_dict())
+
+
+    def test_update_from_invalid(self):
+        # Should get an exception if we try to update() a CorrectMap
+        # with a non-CorrectMap value
+        invalid_list = [None, "string", 5, datetime.datetime.today()]
+
+        for invalid in invalid_list:
+            with self.assertRaises(Exception):
+                self.cmap.update(invalid)

common/lib/capa/capa/tests/test_html_render.py

+import unittest
+from lxml import etree
+import os
+import textwrap
+import json
+import mock
+
+from capa.capa_problem import LoncapaProblem
+from response_xml_factory import StringResponseXMLFactory, CustomResponseXMLFactory
+from . import test_system
+
+class CapaHtmlRenderTest(unittest.TestCase):
+
+    def test_include_html(self):
+        # Create a test file to include
+        self._create_test_file('test_include.xml',
+                                '<test>Test include</test>')
+
+        # Generate some XML with an <include>
+        xml_str = textwrap.dedent("""
+            <problem>
+                <include file="test_include.xml"/>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = LoncapaProblem(xml_str, '1', system=test_system)
+    
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the include file was embedded in the problem
+        test_element = rendered_html.find("test")
+        self.assertEqual(test_element.tag, "test")
+        self.assertEqual(test_element.text, "Test include")
+
+
+    def test_process_outtext(self):
+        # Generate some XML with <startouttext /> and <endouttext />
+        xml_str = textwrap.dedent("""
+            <problem>
+            <startouttext/>Test text<endouttext/>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = LoncapaProblem(xml_str, '1', system=test_system)
+    
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the <startouttext /> and <endouttext />
+        # were converted to <span></span> tags
+        span_element = rendered_html.find('span')
+        self.assertEqual(span_element.text, 'Test text')
+
+    def test_render_script(self):
+        # Generate some XML with a <script> tag
+        xml_str = textwrap.dedent("""
+            <problem>
+                <script>test=True</script>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = LoncapaProblem(xml_str, '1', system=test_system)
+    
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the script element has been removed from the rendered HTML
+        script_element = rendered_html.find('script')
+        self.assertEqual(None, script_element)
+
+    def test_render_response_xml(self):
+        # Generate some XML for a string response
+        kwargs = {'question_text': "Test question",
+                    'explanation_text': "Test explanation",
+                    'answer': 'Test answer',
+                    'hints': [('test prompt', 'test_hint', 'test hint text')]}
+        xml_str = StringResponseXMLFactory().build_xml(**kwargs)
+
+        # Mock out the template renderer
+        test_system.render_template = mock.Mock()
+        test_system.render_template.return_value = "<div>Input Template Render</div>"
+
+        # Create the problem and render the HTML
+        problem = LoncapaProblem(xml_str, '1', system=test_system)
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect problem has been turned into a <div>
+        self.assertEqual(rendered_html.tag, "div")
+
+        # Expect question text is in a <p> child
+        question_element = rendered_html.find("p")
+        self.assertEqual(question_element.text, "Test question")
+
+        # Expect that the response has been turned into a <span>
+        response_element = rendered_html.find("span")
+        self.assertEqual(response_element.tag, "span")
+
+        # Expect that the response <span> 
+        # that contains a <div> for the textline
+        textline_element = response_element.find("div")
+        self.assertEqual(textline_element.text, 'Input Template Render')
+        
+        # Expect a child <div> for the solution
+        # with the rendered template
+        solution_element = rendered_html.find("div")
+        self.assertEqual(solution_element.text, 'Input Template Render')
+
+        # Expect that the template renderer was called with the correct
+        # arguments, once for the textline input and once for
+        # the solution
+        expected_textline_context = {'status': 'unsubmitted', 
+                                        'value': '', 
+                                        'preprocessor': None, 
+                                        'msg': '', 
+                                        'inline': False, 
+                                        'hidden': False, 
+                                        'do_math': False, 
+                                        'id': '1_2_1', 
+                                        'size': None}
+
+        expected_solution_context = {'id': '1_solution_1'}
+
+        expected_calls = [mock.call('textline.html', expected_textline_context),
+                mock.call('solutionspan.html', expected_solution_context)]
+
+        self.assertEqual(test_system.render_template.call_args_list,
+                            expected_calls)
+
+
+    def test_render_response_with_overall_msg(self):
+        # CustomResponse script that sets an overall_message
+        script=textwrap.dedent("""
+            def check_func(*args):
+                msg = '<p>Test message 1<br /></p><p>Test message 2</p>'
+                return {'overall_message': msg,
+                        'input_list': [ {'ok': True, 'msg': '' } ] }
+        """)
+
+        # Generate some XML for a CustomResponse
+        kwargs = {'script':script, 'cfn': 'check_func'}
+        xml_str = CustomResponseXMLFactory().build_xml(**kwargs)
+
+        # Create the problem and render the html
+        problem = LoncapaProblem(xml_str, '1', system=test_system)
+        
+        # Grade the problem
+        correctmap = problem.grade_answers({'1_2_1': 'test'})
+
+        # Render the html
+        rendered_html = etree.XML(problem.get_html())
+
+
+        # Expect that there is a <div> within the response <div>
+        # with css class response_message
+        msg_div_element = rendered_html.find(".//div[@class='response_message']")
+        self.assertEqual(msg_div_element.tag, "div")
+        self.assertEqual(msg_div_element.get('class'), "response_message")
+
+        # Expect that the <div> contains our message (as part of the XML tree)
+        msg_p_elements = msg_div_element.findall('p')
+        self.assertEqual(msg_p_elements[0].tag, "p")
+        self.assertEqual(msg_p_elements[0].text, "Test message 1")
+
+        self.assertEqual(msg_p_elements[1].tag, "p")
+        self.assertEqual(msg_p_elements[1].text, "Test message 2")
+
+
+    def test_substitute_python_vars(self):
+        # Generate some XML with Python variables defined in a script
+        # and used later as attributes
+        xml_str = textwrap.dedent("""
+            <problem>
+                <script>test="TEST"</script>
+                <span attr="$test"></span>
+            </problem>
+        """)
+
+        # Create the problem and render the HTML
+        problem = LoncapaProblem(xml_str, '1', system=test_system)
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the variable $test has been replaced with its value
+        span_element = rendered_html.find('span')
+        self.assertEqual(span_element.get('attr'), "TEST")
+
+    def _create_test_file(self, path, content_str):
+        test_fp = test_system.filestore.open(path, "w")
+        test_fp.write(content_str)
+        test_fp.close()
+
+        self.addCleanup(lambda: os.remove(test_fp.name))

common/lib/capa/capa/tests/test_responsetypes.py

@@ -8,6 +8,7 @@ import json
 from nose.plugins.skip import SkipTest
 import os
 import unittest
+import textwrap
 
 from . import test_system
 
@@ -663,30 +664,43 @@ class CustomResponseTest(ResponseTest):
 
         # Inline code can update the global messages list
         # to pass messages to the CorrectMap for a particular input
-        inline_script = """messages[0] = "Test Message" """
+        # The code can also set the global overall_message (str)
+        # to pass a message that applies to the whole response
+        inline_script = textwrap.dedent("""
+        messages[0] = "Test Message" 
+        overall_message = "Overall message"
+        """)
         problem = self.build_problem(answer=inline_script)
 
         input_dict = {'1_2_1': '0'}
-        msg = problem.grade_answers(input_dict).get_msg('1_2_1')
-        self.assertEqual(msg, "Test Message")
+        correctmap = problem.grade_answers(input_dict)
 
-    def test_function_code(self):
+        # Check that the message for the particular input was received
+        input_msg = correctmap.get_msg('1_2_1')
+        self.assertEqual(input_msg, "Test Message")
 
-        # For function code, we pass in three arguments:
+        # Check that the overall message (for the whole response) was received
+        overall_msg = correctmap.get_overall_message()
+        self.assertEqual(overall_msg, "Overall message")
+
+
+    def test_function_code_single_input(self):
+
+        # For function code, we pass in these arguments:
         # 
         #   'expect' is the expect attribute of the <customresponse>
         #
         #   'answer_given' is the answer the student gave (if there is just one input)
         #       or an ordered list of answers (if there are multiple inputs)
         #   
-        #   'student_answers' is a dictionary of answers by input ID
-        #
         #
         # The function should return a dict of the form 
         # { 'ok': BOOL, 'msg': STRING }
         #
-        script = """def check_func(expect, answer_given, student_answers):
-    return {'ok': answer_given == expect, 'msg': 'Message text'}"""
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return {'ok': answer_given == expect, 'msg': 'Message text'}
+        """)
 
         problem = self.build_problem(script=script, cfn="check_func", expect="42")
 
@@ -698,7 +712,7 @@ class CustomResponseTest(ResponseTest):
         msg = correct_map.get_msg('1_2_1')
 
         self.assertEqual(correctness, 'correct')
-        self.assertEqual(msg, "Message text\n")
+        self.assertEqual(msg, "Message text")
 
         # Incorrect answer
         input_dict = {'1_2_1': '0'}
@@ -708,19 +722,108 @@ class CustomResponseTest(ResponseTest):
         msg = correct_map.get_msg('1_2_1')
 
         self.assertEqual(correctness, 'incorrect')
-        self.assertEqual(msg, "Message text\n")
+        self.assertEqual(msg, "Message text")
+
+    def test_function_code_multiple_input_no_msg(self):
+
+        # Check functions also have the option of returning
+        # a single boolean value 
+        # If true, mark all the inputs correct
+        # If false, mark all the inputs incorrect
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return (answer_given[0] == expect and
+                        answer_given[1] == expect)
+        """)
+
+        problem = self.build_problem(script=script, cfn="check_func", 
+                                    expect="42", num_inputs=2)
+
+        # Correct answer -- expect both inputs marked correct
+        input_dict = {'1_2_1': '42', '1_2_2': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        self.assertEqual(correctness, 'correct')
+
+        correctness = correct_map.get_correctness('1_2_2')
+        self.assertEqual(correctness, 'correct')
+
+        # One answer incorrect -- expect both inputs marked incorrect
+        input_dict = {'1_2_1': '0', '1_2_2': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        self.assertEqual(correctness, 'incorrect')
+
+        correctness = correct_map.get_correctness('1_2_2')
+        self.assertEqual(correctness, 'incorrect')
+
+
+    def test_function_code_multiple_inputs(self):
+
+        # If the <customresponse> has multiple inputs associated with it,
+        # the check function can return a dict of the form:
+        # 
+        # {'overall_message': STRING,
+        #  'input_list': [{'ok': BOOL, 'msg': STRING}, ...] }
+        # 
+        # 'overall_message' is displayed at the end of the response
+        #
+        # 'input_list' contains dictionaries representing the correctness
+        #           and message for each input.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                return {'overall_message': 'Overall message',
+                        'input_list': [
+                            {'ok': check1,  'msg': 'Feedback 1'},
+                            {'ok': check2,  'msg': 'Feedback 2'},
+                            {'ok': check3,  'msg': 'Feedback 3'} ] }
+            """)
+
+        problem = self.build_problem(script=script, 
+                                    cfn="check_func", num_inputs=3)
+
+        # Grade the inputs (one input incorrect)
+        input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3' }
+        correct_map = problem.grade_answers(input_dict)
+
+        # Expect that we receive the overall message (for the whole response)
+        self.assertEqual(correct_map.get_overall_message(), "Overall message")
+
+        # Expect that the inputs were graded individually
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect')
+        self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct')
+        self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct')
 
-    def test_multiple_inputs(self):
+        # Expect that we received messages for each individual input
+        self.assertEqual(correct_map.get_msg('1_2_1'), 'Feedback 1')
+        self.assertEqual(correct_map.get_msg('1_2_2'), 'Feedback 2')
+        self.assertEqual(correct_map.get_msg('1_2_3'), 'Feedback 3')
+
+
+    def test_multiple_inputs_return_one_status(self):
         # When given multiple inputs, the 'answer_given' argument
         # to the check_func() is a list of inputs
+        #
         # The sample script below marks the problem as correct
         # if and only if it receives answer_given=[1,2,3]
         # (or string values ['1','2','3'])
-        script = """def check_func(expect, answer_given, student_answers):
-    check1 = (int(answer_given[0]) == 1)
-    check2 = (int(answer_given[1]) == 2)
-    check3 = (int(answer_given[2]) == 3)
-    return {'ok': (check1 and check2 and check3),  'msg': 'Message text'}"""
+        #
+        # Since we return a dict describing the status of one input,
+        # we expect that the same 'ok' value is applied to each
+        # of the inputs.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                return {'ok': (check1 and check2 and check3),  
+                        'msg': 'Message text'}
+            """)
 
         problem = self.build_problem(script=script, 
                                     cfn="check_func", num_inputs=3)
@@ -743,6 +846,37 @@ class CustomResponseTest(ResponseTest):
         self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct')
         self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct')
 
+        # Message is interpreted as an "overall message"
+        self.assertEqual(correct_map.get_overall_message(), 'Message text')
+
+    def test_script_exception(self):
+
+        # Construct a script that will raise an exception
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                raise Exception("Test")
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func")
+
+        # Expect that an exception gets raised when we check the answer
+        with self.assertRaises(Exception):
+            problem.grade_answers({'1_2_1': '42'})
+    
+    def test_invalid_dict_exception(self):
+
+        # Construct a script that passes back an invalid dict format
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return {'invalid': 'test'}
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func")
+
+        # Expect that an exception gets raised when we check the answer
+        with self.assertRaises(Exception):
+            problem.grade_answers({'1_2_1': '42'})
+
 
 class SchematicResponseTest(ResponseTest):
     from response_xml_factory import SchematicResponseXMLFactory

doc/public/course_data_formats/custom_response.rst

+####################################
+CustomResponse XML and Python Script
+####################################
+
+This document explains how to write a CustomResponse problem.  CustomResponse
+problems execute Python script to check student answers and provide hints.
+
+There are two general ways to create a CustomResponse problem:
+
+
+*****************
+Answer tag format
+*****************
+One format puts the Python code in an ``<answer>`` tag:
+
+.. code-block:: xml
+
+    <problem>
+        <p>What is the sum of 2 and 3?</p>
+
+        <customresponse expect="5">
+        <textline math="1" />
+        </customresponse>
+
+        <answer>
+    # Python script goes here
+        </answer>
+    </problem>
+
+
+The Python script interacts with these variables in the global context:
+    * ``answers``: An ordered list of answers the student provided.
+      For example, if the student answered ``6``, then ``answers[0]`` would
+      equal ``6``.
+    * ``expect``: The value of the ``expect`` attribute of ``<customresponse>``
+      (if provided).
+    * ``correct``: An ordered list of strings indicating whether the
+      student answered the question correctly.  Valid values are
+      ``"correct"``, ``"incorrect"``, and ``"unknown"``.  You can set these
+      values in the script.
+    * ``messages``: An ordered list of message strings that will be displayed
+      beneath each input.  You can use this to provide hints to users.
+      For example ``messages[0] = "The capital of California is Sacramento"``
+      would display that message beneath the first input of the response.
+    * ``overall_message``: A string that will be displayed beneath the
+      entire problem.  You can use this to provide a hint that applies
+      to the entire problem rather than a particular input.
+
+Example of a checking script:
+
+.. code-block:: python
+
+    if answers[0] == expect:
+        correct[0] = 'correct'
+        overall_message = 'Good job!'
+    else:
+        correct[0] = 'incorrect'
+        messages[0] = 'This answer is incorrect'
+        overall_message = 'Please try again'
+
+**Important**: Python is picky about indentation.  Within the ``<answer>`` tag,
+you must begin your script with no indentation.
+
+*****************
+Script tag format
+*****************
+The other way to create a CustomResponse is to put a "checking function"
+in a ``<script>`` tag, then use the ``cfn`` attribute of the
+``<customresponse>`` tag:
+
+.. code-block:: xml
+
+    <problem>
+        <p>What is the sum of 2 and 3?</p>
+
+        <customresponse cfn="check_func" expect="5">
+        <textline math="1" />
+        </customresponse>
+
+        <script type="loncapa/python">
+    def check_func(expect, ans):
+        # Python script goes here
+        </script>
+    </problem>
+
+
+**Important**: Python is picky about indentation.  Within the ``<script>`` tag,
+the ``def check_func(expect, ans):`` line must have no indentation.
+
+The check function accepts two arguments:
+    * ``expect`` is the value of the ``expect`` attribute of ``<customresponse>``
+      (if provided)
+    * ``answer`` is either:
+
+        * The value of the answer the student provided, if there is only one input.
+        * An ordered list of answers the student provided, if there
+          are multiple inputs.
+
+There are several ways that the check function can indicate whether the student
+succeeded.  The check function can return any of the following:
+
+    * ``True``: Indicates that the student answered correctly for all inputs.
+    * ``False``: Indicates that the student answered incorrectly.
+      All inputs will be marked incorrect.
+    * A dictionary of the form: ``{ 'ok': True, 'msg': 'Message' }``
+      If the dictionary's value for ``ok`` is set to ``True``, all inputs are
+      marked correct; if it is set to ``False``, all inputs are marked incorrect.
+      The ``msg`` is displayed beneath all inputs, and it may contain
+      XHTML markup.
+    * A dictionary of the form 
+
+.. code-block:: xml
+      
+    
+    { 'overall_message': 'Overall message',
+        'input_list': [
+            { 'ok': True, 'msg': 'Feedback for input 1'},
+            { 'ok': False, 'msg': 'Feedback for input 2'},
+            ... ] }
+
+The last form is useful for responses that contain multiple inputs.
+It allows you to provide feedback for each input individually,
+as well as a message that applies to the entire response.
+
+Example of a checking function:
+
+.. code-block:: python
+
+    def check_func(expect, answer_given):
+        check1 = (int(answer_given[0]) == 1)
+        check2 = (int(answer_given[1]) == 2)
+        check3 = (int(answer_given[2]) == 3)
+        return {'overall_message': 'Overall message',
+                    'input_list': [
+                        { 'ok': check1, 'msg': 'Feedback 1'},
+                        { 'ok': check2, 'msg': 'Feedback 2'},
+                        { 'ok': check3, 'msg': 'Feedback 3'} ] }
+
+The function checks that the user entered ``1`` for the first input, 
+``2`` for the  second input, and ``3`` for the third input.
+It provides feedback messages for each individual input, as well
+as a message displayed beneath the entire problem.

doc/public/index.rst

@@ -24,6 +24,7 @@ Specific Problem Types
 
    course_data_formats/drag_and_drop/drag_and_drop_input.rst
    course_data_formats/graphical_slider_tool/graphical_slider_tool.rst
+   course_data_formats/custom_response.rst
 
 
 Internal Data Formats