Merge pull request #5048 from Stanford-Online/jbau/edx/custom-response-fractional-grades

capa custom response support for decimal grades

common/lib/capa/capa/responsetypes.py

@@ -1598,11 +1598,17 @@ class CustomResponse(LoncapaResponse):
         correct = self.context['correct']
         messages = self.context['messages']
         overall_message = self.clean_message_html(self.context['overall_message'])
+        grade_decimals = self.context.get('grade_decimals')
+
         correct_map = CorrectMap()
         correct_map.set_overall_message(overall_message)
 
         for k in range(len(idset)):
-            npoints = self.maxpoints[idset[k]] if correct[k] == 'correct' else 0
+            max_points = self.maxpoints[idset[k]]
+            if grade_decimals:
+                npoints = max_points * grade_decimals[k]
+            else:
+                npoints = max_points if correct[k] == 'correct' else 0
             correct_map.set(idset[k], correct[k], msg=messages[k],
                             npoints=npoints)
         return correct_map
@@ -1643,7 +1649,9 @@ class CustomResponse(LoncapaResponse):
             )
             if isinstance(ret, dict):
                 # One kind of dictionary the check function can return has the
-                # form {'ok': BOOLEAN, 'msg': STRING}
+                # form {'ok': BOOLEAN, 'msg': STRING, 'grade_decimal' (optional): FLOAT (between 0.0 and 1.0)}
+                # 'ok' will control the checkmark, while grade_decimal, if present, will scale
+                # the score the student receives on the response.
                 # If there are multiple inputs, they all get marked
                 # to the same correct/incorrect value
                 if 'ok' in ret:
@@ -1658,28 +1666,49 @@ class CustomResponse(LoncapaResponse):
                     else:
                         self.context['messages'][0] = msg
 
+                    if 'grade_decimal' in ret:
+                        decimal = ret['grade_decimal']
+                    else:
+                        decimal = 1.0 if ret['ok'] else 0.0
+                    grade_decimals = [decimal] * len(idset)
+                    self.context['grade_decimals'] = grade_decimals
+
                 # Another kind of dictionary the check function can return has
                 # the form:
-                # {'overall_message': STRING,
-                #  'input_list': [{ 'ok': BOOLEAN, 'msg': STRING }, ...] }
+                # { 'overall_message': STRING,
+                #   'input_list': [
+                #     { 'ok': BOOLEAN, 'msg': STRING, 'grade_decimal' (optional): FLOAT (between 0.0 and 1.0)},
+                #   ...
+                #   ]
+                # }
+                # 'ok' will control the checkmark, while grade_decimal, if present, will scale
+                # the score the student receives on the response.
                 #
                 # This allows the function to return an 'overall message'
                 # that applies to the entire problem, as well as correct/incorrect
-                # status and messages for individual inputs
+                # status, scaled grades, and messages for individual inputs
                 elif 'input_list' in ret:
                     overall_message = ret.get('overall_message', '')
                     input_list = ret['input_list']
 
                     correct = []
                     messages = []
+                    grade_decimals = []
                     for input_dict in input_list:
                         correct.append('correct'
                                        if input_dict['ok'] else 'incorrect')
                         msg = (self.clean_message_html(input_dict['msg'])
                                if 'msg' in input_dict else None)
                         messages.append(msg)
+                        if 'grade_decimal' in input_dict:
+                            decimal = input_dict['grade_decimal']
+                        else:
+                            decimal = 1.0 if input_dict['ok'] else 0.0
+                        grade_decimals.append(decimal)
+
                     self.context['messages'] = messages
                     self.context['overall_message'] = overall_message
+                    self.context['grade_decimals'] = grade_decimals
 
                 # Otherwise, we do not recognize the dictionary
                 # Raise an exception

common/lib/capa/capa/tests/test_correctmap.py

@@ -85,7 +85,7 @@ class CorrectMapTest(unittest.TestCase):
         self.cmap.set(
             answer_id='1_2_1',
             correctness='correct',
-            npoints=5
+            npoints=5.3
         )
 
         self.cmap.set(
@@ -116,7 +116,7 @@ class CorrectMapTest(unittest.TestCase):
         # If points assigned --> npoints
         # If no points assigned and correct --> 1 point
         # If no points assigned and incorrect --> 0 points
-        self.assertEqual(self.cmap.get_npoints('1_2_1'), 5)
+        self.assertEqual(self.cmap.get_npoints('1_2_1'), 5.3)
         self.assertEqual(self.cmap.get_npoints('2_2_1'), 1)
         self.assertEqual(self.cmap.get_npoints('3_2_1'), 5)
         self.assertEqual(self.cmap.get_npoints('4_2_1'), 0)

common/lib/capa/capa/tests/test_responsetypes.py

@@ -1399,7 +1399,7 @@ class CustomResponseTest(ResponseTest):
         #       or an ordered list of answers (if there are multiple inputs)
         #
         # The function should return a dict of the form
-        # { 'ok': BOOL, 'msg': STRING }
+        # { 'ok': BOOL, 'msg': STRING } (no 'grade_decimal' key to test that it's optional)
         #
         script = textwrap.dedent("""
             def check_func(expect, answer_given):
@@ -1414,9 +1414,11 @@ class CustomResponseTest(ResponseTest):
 
         correctness = correct_map.get_correctness('1_2_1')
         msg = correct_map.get_msg('1_2_1')
+        npoints = correct_map.get_npoints('1_2_1')
 
         self.assertEqual(correctness, 'correct')
         self.assertEqual(msg, "Message text")
+        self.assertEqual(npoints, 1)
 
         # Incorrect answer
         input_dict = {'1_2_1': '0'}
@@ -1424,9 +1426,45 @@ class CustomResponseTest(ResponseTest):
 
         correctness = correct_map.get_correctness('1_2_1')
         msg = correct_map.get_msg('1_2_1')
+        npoints = correct_map.get_npoints('1_2_1')
 
         self.assertEqual(correctness, 'incorrect')
         self.assertEqual(msg, "Message text")
+        self.assertEqual(npoints, 0)
+
+    def test_function_code_single_input_decimal_score(self):
+        # For function code, we pass in these arguments:
+        #
+        #   'expect' is the expect attribute of the <customresponse>
+        #
+        #   'answer_given' is the answer the student gave (if there is just one input)
+        #       or an ordered list of answers (if there are multiple inputs)
+        #
+        # The function should return a dict of the form
+        # { 'ok': BOOL, 'msg': STRING, 'grade_decimal': FLOAT }
+        #
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return {
+                    'ok': answer_given == expect,
+                    'msg': 'Message text',
+                    'grade_decimal': 0.9 if answer_given == expect else 0.1,
+                }
+        """)
+
+        problem = self.build_problem(script=script, cfn="check_func", expect="42")
+
+        # Correct answer
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEqual(correct_map.get_npoints('1_2_1'), 0.9)
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct')
+
+        # Incorrect answer
+        input_dict = {'1_2_1': '43'}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEqual(correct_map.get_npoints('1_2_1'), 0.1)
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect')
 
     def test_function_code_multiple_input_no_msg(self):
 
@@ -1469,7 +1507,7 @@ class CustomResponseTest(ResponseTest):
         # the check function can return a dict of the form:
         #
         # {'overall_message': STRING,
-        #  'input_list': [{'ok': BOOL, 'msg': STRING}, ...] }
+        #  'input_list': [{'ok': BOOL, 'msg': STRING}, ...] } (no grade_decimal to test it's optional)
         #
         # 'overall_message' is displayed at the end of the response
         #
@@ -1502,11 +1540,59 @@ class CustomResponseTest(ResponseTest):
         self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct')
         self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct')
 
+        # Expect that the inputs were given correct npoints
+        self.assertEqual(correct_map.get_npoints('1_2_1'), 0)
+        self.assertEqual(correct_map.get_npoints('1_2_2'), 1)
+        self.assertEqual(correct_map.get_npoints('1_2_3'), 1)
+
         # Expect that we received messages for each individual input
         self.assertEqual(correct_map.get_msg('1_2_1'), 'Feedback 1')
         self.assertEqual(correct_map.get_msg('1_2_2'), 'Feedback 2')
         self.assertEqual(correct_map.get_msg('1_2_3'), 'Feedback 3')
 
+    def test_function_code_multiple_inputs_decimal_score(self):
+
+        # If the <customresponse> has multiple inputs associated with it,
+        # the check function can return a dict of the form:
+        #
+        # {'overall_message': STRING,
+        #  'input_list': [{'ok': BOOL, 'msg': STRING, 'grade_decimal': FLOAT}, ...] }
+        #        #
+        # 'input_list' contains dictionaries representing the correctness
+        #           and message for each input.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                score1 = 0.9 if check1 else 0.1
+                score2 = 0.9 if check2 else 0.1
+                score3 = 0.9 if check3 else 0.1
+                return {
+                    'input_list': [
+                        {'ok': check1, 'grade_decimal': score1, 'msg': 'Feedback 1'},
+                        {'ok': check2, 'grade_decimal': score2, 'msg': 'Feedback 2'},
+                        {'ok': check3, 'grade_decimal': score3, 'msg': 'Feedback 3'},
+                    ]
+                }
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func", num_inputs=3)
+
+        # Grade the inputs (one input incorrect)
+        input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3'}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Expect that the inputs were graded individually
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect')
+        self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct')
+        self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct')
+
+        # Expect that the inputs were given correct npoints
+        self.assertEqual(correct_map.get_npoints('1_2_1'), 0.1)
+        self.assertEqual(correct_map.get_npoints('1_2_2'), 0.9)
+        self.assertEqual(correct_map.get_npoints('1_2_3'), 0.9)
+
     def test_function_code_with_extra_args(self):
         script = textwrap.dedent("""\
                     def check_func(expect, answer_given, options, dynamath):

common/lib/xmodule/xmodule/tests/test_capa_module.py

@@ -83,6 +83,7 @@ class CapaFactory(object):
                problem_state=None,
                correct=False,
                xml=None,
+               override_get_score=True,
                **kwargs
                ):
         """
@@ -130,11 +131,12 @@ class CapaFactory(object):
             ScopeIds(None, None, location, location),
         )
 
-        if correct:
-            # TODO: probably better to actually set the internal state properly, but...
-            module.get_score = lambda: {'score': 1, 'total': 1}
-        else:
-            module.get_score = lambda: {'score': 0, 'total': 1}
+        if override_get_score:
+            if correct:
+                # TODO: probably better to actually set the internal state properly, but...
+                module.get_score = lambda: {'score': 1, 'total': 1}
+            else:
+                module.get_score = lambda: {'score': 0, 'total': 1}
 
         return module
 
@@ -211,6 +213,28 @@ class CapaModuleTest(unittest.TestCase):
         other_module = CapaFactory.create(correct=True)
         self.assertEqual(other_module.get_score()['score'], 1)
 
+    def test_get_score(self):
+        """
+        Do 1 test where the internals of get_score are properly set
+
+        @jbau Note: this obviously depends on a particular implementation of get_score, but I think this is actually
+        useful as unit-code coverage for this current implementation.  I don't see a layer where LoncapaProblem
+        is tested directly
+        """
+        from capa.correctmap import CorrectMap
+        student_answers = {'1_2_1': 'abcd'}
+        correct_map = CorrectMap(answer_id='1_2_1', correctness="correct", npoints=0.9)
+        module = CapaFactory.create(correct=True, override_get_score=False)
+        module.lcp.correct_map = correct_map
+        module.lcp.student_answers = student_answers
+        self.assertEqual(module.get_score()['score'], 0.9)
+
+        other_correct_map = CorrectMap(answer_id='1_2_1', correctness="incorrect", npoints=0.1)
+        other_module = CapaFactory.create(correct=False, override_get_score=False)
+        other_module.lcp.correct_map = other_correct_map
+        other_module.lcp.student_answers = student_answers
+        self.assertEqual(other_module.get_score()['score'], 0.1)
+
     def test_showanswer_default(self):
         """
         Make sure the show answer logic does the right thing.