Skip to content

E
edx-platform
Commit 9f6119a1 by Will Daly
Options

* Fixed a bug in CorrectMap in which overall_message was not copied 

  in the update() method
* Implemented CustomResponse behavior for returning overall_message,
  with unit tests
* Added additional CustomResponse unit tests for exception handling
parent 3dd49881
Showing with 207 additions and 31 deletions
common/lib/capa/capa/correctmap.py
......@@ -159,6 +159,7 @@ class CorrectMap(object):
if not isinstance(other_cmap, CorrectMap):
raise Exception('CorrectMap.update called with invalid argument %s' % other_cmap)
self.cmap.update(other_cmap.get_dict())
self.set_overall_message(other_cmap.get_overall_message())
def set_overall_message(self, message_str):
......
common/lib/capa/capa/responsetypes.py
......@@ -965,6 +965,7 @@ def sympy_check2():
# not expecting 'unknown's
correct = ['unknown'] * len(idset)
messages = [''] * len(idset)
overall_message = ""
# put these in the context of the check function evaluator
# note that this doesn't help the "cfn" version - only the exec version
......@@ -996,6 +997,10 @@ def sympy_check2():
# the list of messages to be filled in by the check function
'messages': messages,
# a message that applies to the entire response
# instead of a particular input
'overall_message': overall_message,
# any options to be passed to the cfn
'options': self.xml.get('options'),
'testdat': 'hello world',
......@@ -1010,6 +1015,7 @@ def sympy_check2():
exec self.code in self.context['global_context'], self.context
correct = self.context['correct']
messages = self.context['messages']
overall_message = self.context['overall_message']
except Exception as err:
print "oops in customresponse (code) error %s" % err
print "context = ", self.context
......@@ -1044,28 +1050,66 @@ def sympy_check2():
log.error(traceback.format_exc())
raise Exception("oops in customresponse (cfn) error %s" % err)
log.debug("[courseware.capa.responsetypes.customresponse.get_score] ret = %s" % ret)
if type(ret) == dict:
correct = ['correct'] * len(idset) if ret['ok'] else ['incorrect'] * len(idset)
msg = ret['msg']
if 1:
# try to clean up message html
msg = '<html>' + msg + '</html>'
msg = msg.replace('&#60;', '&lt;')
#msg = msg.replace('&lt;','<')
msg = etree.tostring(fromstring_bs(msg, convertEntities=None),
pretty_print=True)
#msg = etree.tostring(fromstring_bs(msg),pretty_print=True)
msg = msg.replace('&#13;', '')
#msg = re.sub('<html>(.*)</html>','\\1',msg,flags=re.M|re.DOTALL) # python 2.7
msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
messages[0] = msg
# One kind of dictionary the check function can return has the
# form {'ok': BOOLEAN, 'msg': STRING}
# If there are multiple inputs, they all get marked
# to the same correct/incorrect value
# and the first input stores the message
if 'ok' in ret:
correct = ['correct'] * len(idset) if ret['ok'] else ['incorrect'] * len(idset)
msg = ret['msg']
if 1:
# try to clean up message html
msg = '<html>' + msg + '</html>'
msg = msg.replace('&#60;', '&lt;')
#msg = msg.replace('&lt;','<')
msg = etree.tostring(fromstring_bs(msg, convertEntities=None),
pretty_print=True)
#msg = etree.tostring(fromstring_bs(msg),pretty_print=True)
msg = msg.replace('&#13;', '')
#msg = re.sub('<html>(.*)</html>','\\1',msg,flags=re.M|re.DOTALL) # python 2.7
msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
messages[0] = msg
# Another kind of dictionary the check function can return has
# the form:
# {'overall_message': STRING,
# 'input_list': [{ 'ok': BOOLEAN, 'msg': STRING }, ...] }
#
# This allows the function to return an 'overall message'
# that applies to the entire problem, as well as correct/incorrect
# status and messages for individual inputs
elif 'input_list' in ret:
overall_message = ret.get('overall_message', '')
input_list = ret['input_list']
correct = []
messages = []
for input_dict in input_list:
correct.append('correct' if input_dict['ok'] else 'incorrect')
messages.append(input_dict['msg'] if 'msg' in input_dict else None)
# Otherwise, we do not recognize the dictionary
# Raise an exception
else:
log.error(traceback.format_exc())
raise Exception("CustomResponse: check function returned an invalid dict")
# The check function can return a boolean value,
# indicating whether all inputs should be marked
# correct or incorrect
else:
correct = ['correct'] * len(idset) if ret else ['incorrect'] * len(idset)
# build map giving "correct"ness of the answer(s)
correct_map = CorrectMap()
correct_map.set_overall_message(overall_message)
for k in range(len(idset)):
npoints = self.maxpoints[idset[k]] if correct[k] == 'correct' else 0
correct_map.set(idset[k], correct[k], msg=messages[k],
......
common/lib/capa/capa/tests/test_responsetypes.py
......@@ -8,6 +8,7 @@ import json
from nose.plugins.skip import SkipTest
import os
import unittest
import textwrap
from . import test_system
......@@ -663,30 +664,43 @@ class CustomResponseTest(ResponseTest):
# Inline code can update the global messages list
# to pass messages to the CorrectMap for a particular input
inline_script = """messages[0] = "Test Message" """
# The code can also set the global overall_message (str)
# to pass a message that applies to the whole response
inline_script = textwrap.dedent("""
messages[0] = "Test Message"
overall_message = "Overall message"
""")
problem = self.build_problem(answer=inline_script)
input_dict = {'1_2_1': '0'}
msg = problem.grade_answers(input_dict).get_msg('1_2_1')
self.assertEqual(msg, "Test Message")
correctmap = problem.grade_answers(input_dict)
def test_function_code(self):
# Check that the message for the particular input was received
input_msg = correctmap.get_msg('1_2_1')
self.assertEqual(input_msg, "Test Message")
# For function code, we pass in three arguments:
# Check that the overall message (for the whole response) was received
overall_msg = correctmap.get_overall_message()
self.assertEqual(overall_msg, "Overall message")
def test_function_code_single_input(self):
# For function code, we pass in these arguments:
#
# 'expect' is the expect attribute of the <customresponse>
#
# 'answer_given' is the answer the student gave (if there is just one input)
# or an ordered list of answers (if there are multiple inputs)
#
# 'student_answers' is a dictionary of answers by input ID
#
#
# The function should return a dict of the form
# { 'ok': BOOL, 'msg': STRING }
#
script = """def check_func(expect, answer_given, student_answers):
return {'ok': answer_given == expect, 'msg': 'Message text'}"""
script = textwrap.dedent("""
def check_func(expect, answer_given):
return {'ok': answer_given == expect, 'msg': 'Message text'}
""")
problem = self.build_problem(script=script, cfn="check_func", expect="42")
......@@ -710,17 +724,134 @@ class CustomResponseTest(ResponseTest):
self.assertEqual(correctness, 'incorrect')
self.assertEqual(msg, "Message text\n")
def test_multiple_inputs(self):
def test_function_code_multiple_input_no_msg(self):
# Check functions also have the option of returning
# a single boolean value
# If true, mark all the inputs correct
# If false, mark all the inputs incorrect
script = textwrap.dedent("""
def check_func(expect, answer_given):
return (answer_given[0] == expect and
answer_given[1] == expect)
""")
problem = self.build_problem(script=script, cfn="check_func",
expect="42", num_inputs=2)
# Correct answer -- expect both inputs marked correct
input_dict = {'1_2_1': '42', '1_2_2': '42'}
correct_map = problem.grade_answers(input_dict)
correctness = correct_map.get_correctness('1_2_1')
self.assertEqual(correctness, 'correct')
correctness = correct_map.get_correctness('1_2_2')
self.assertEqual(correctness, 'correct')
# One answer incorrect -- expect both inputs marked incorrect
input_dict = {'1_2_1': '0', '1_2_2': '42'}
correct_map = problem.grade_answers(input_dict)
correctness = correct_map.get_correctness('1_2_1')
self.assertEqual(correctness, 'incorrect')
correctness = correct_map.get_correctness('1_2_2')
self.assertEqual(correctness, 'incorrect')
def test_script_exception(self):
# Construct a script that will raise an exception
script = textwrap.dedent("""
def check_func(expect, answer_given):
raise Exception("Test")
""")
problem = self.build_problem(script=script, cfn="check_func")
# Expect that an exception gets raised when we check the answer
with self.assertRaises(Exception):
problem.grade_answers({'1_2_1': '42'})
def test_invalid_dict_exception(self):
# Construct a script that passes back an invalid dict format
script = textwrap.dedent("""
def check_func(expect, answer_given):
return {'invalid': 'test'}
""")
problem = self.build_problem(script=script, cfn="check_func")
# Expect that an exception gets raised when we check the answer
with self.assertRaises(Exception):
problem.grade_answers({'1_2_1': '42'})
def test_function_code_multiple_inputs(self):
# If the <customresponse> has multiple inputs associated with it,
# the check function can return a dict of the form:
#
# {'overall_message': STRING,
# 'input_list': [{'ok': BOOL, 'msg': STRING}, ...] }
#
# 'overall_message' is displayed at the end of the response
#
# 'input_list' contains dictionaries representing the correctness
# and message for each input.
script = textwrap.dedent("""
def check_func(expect, answer_given):
check1 = (int(answer_given[0]) == 1)
check2 = (int(answer_given[1]) == 2)
check3 = (int(answer_given[2]) == 3)
return {'overall_message': 'Overall message',
'input_list': [
{'ok': check1, 'msg': 'Feedback 1'},
{'ok': check2, 'msg': 'Feedback 2'},
{'ok': check3, 'msg': 'Feedback 3'} ] }
""")
problem = self.build_problem(script=script,
cfn="check_func", num_inputs=3)
# Grade the inputs (one input incorrect)
input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3' }
correct_map = problem.grade_answers(input_dict)
# Expect that we receive the overall message (for the whole response)
self.assertEqual(correct_map.get_overall_message(), "Overall message")
# Expect that the inputs were graded individually
self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect')
self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct')
self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct')
# Expect that we received messages for each individual input
self.assertEqual(correct_map.get_msg('1_2_1'), 'Feedback 1')
self.assertEqual(correct_map.get_msg('1_2_2'), 'Feedback 2')
self.assertEqual(correct_map.get_msg('1_2_3'), 'Feedback 3')
def test_multiple_inputs_return_one_status(self):
# When given multiple inputs, the 'answer_given' argument
# to the check_func() is a list of inputs
#
# The sample script below marks the problem as correct
# if and only if it receives answer_given=[1,2,3]
# (or string values ['1','2','3'])
script = """def check_func(expect, answer_given, student_answers):
check1 = (int(answer_given[0]) == 1)
check2 = (int(answer_given[1]) == 2)
check3 = (int(answer_given[2]) == 3)
return {'ok': (check1 and check2 and check3), 'msg': 'Message text'}"""
#
# Since we return a dict describing the status of one input,
# we expect that the same 'ok' value is applied to each
# of the inputs.
script = textwrap.dedent("""
def check_func(expect, answer_given):
check1 = (int(answer_given[0]) == 1)
check2 = (int(answer_given[1]) == 2)
check3 = (int(answer_given[2]) == 3)
return {'ok': (check1 and check2 and check3),
'msg': 'Message text'}
""")
problem = self.build_problem(script=script,
cfn="check_func", num_inputs=3)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment