Skip to content

E
edx-platform
Commit aba25c2d by Waheed Ahmed Committed by Adam Palay
Options

In OpenEndedModule latest_score and all_score calculate sum of rubric scores… 

In OpenEndedModule latest_score and all_score calculate sum of rubric scores every time for ML grading.
ORA-72
parent 7e4820af
Showing with 903 additions and 8 deletions
common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py
......@@ -258,8 +258,23 @@ class CombinedOpenEndedV1Module():
if not task_states:
return (0, 0, state_values[OpenEndedChild.INITIAL], idx)
final_child_state = json.loads(task_states[-1])
scores = [attempt.get('score', 0) for attempt in final_child_state.get('child_history', [])]
final_task_xml = self.task_xml[-1]
final_child_state_json = task_states[-1]
final_child_state = json.loads(final_child_state_json)
tag_name = self.get_tag_name(final_task_xml)
children = self.child_modules()
task_descriptor = children['descriptors'][tag_name](self.system)
task_parsed_xml = task_descriptor.definition_from_xml(etree.fromstring(final_task_xml), self.system)
task = children['modules'][tag_name](
self.system,
self.location,
task_parsed_xml,
task_descriptor,
self.static_data,
instance_state=final_child_state_json,
)
scores = task.all_scores()
if scores:
best_score = max(scores)
else:
......
common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py
......@@ -679,7 +679,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
return {
'success': success,
'error': error_message,
'student_response': data['student_answer'].replace("\n","<br/>")
'student_response': data['student_answer'].replace("\n", "<br/>")
}
def update_score(self, data, system):
......@@ -738,6 +738,44 @@ class OpenEndedModule(openendedchild.OpenEndedChild):
html = system.render_template('{0}/open_ended.html'.format(self.TEMPLATE_DIR), context)
return html
def latest_score(self):
"""None if not available"""
if not self.child_history:
return None
return self.score_for_attempt(-1)
def all_scores(self):
"""None if not available"""
if not self.child_history:
return None
return [self.score_for_attempt(index) for index in xrange(0, len(self.child_history))]
def score_for_attempt(self, index):
"""
Return sum of rubric scores for ML grading otherwise return attempt["score"].
"""
attempt = self.child_history[index]
score = attempt.get('score')
post_assessment_data = self._parse_score_msg(attempt.get('post_assessment'), self.system)
grader_types = post_assessment_data.get('grader_types')
# According to _parse_score_msg in ML grading there should be only one grader type.
if len(grader_types) == 1 and grader_types[0] == 'ML':
rubric_scores = post_assessment_data.get("rubric_scores")
# Similarly there should be only one list of rubric scores.
if len(rubric_scores) == 1:
rubric_scores_sum = sum(rubric_scores[0])
log.debug("""Score normalized for location={loc}, old_score={old_score},
new_score={new_score}, rubric_score={rubric_score}""".format(
loc=self.location_string,
old_score=score,
new_score=rubric_scores_sum,
rubric_score=rubric_scores
))
return rubric_scores_sum
return score
class OpenEndedDescriptor():
"""
......
common/lib/xmodule/xmodule/tests/test_combined_open_ended.py
......@@ -27,7 +27,9 @@ from xmodule.progress import Progress
from xmodule.tests.test_util_open_ended import (
DummyModulestore, TEST_STATE_SA_IN,
MOCK_INSTANCE_STATE, TEST_STATE_SA, TEST_STATE_AI, TEST_STATE_AI2, TEST_STATE_AI2_INVALID,
TEST_STATE_SINGLE, TEST_STATE_PE_SINGLE, MockUploadedFile
TEST_STATE_SINGLE, TEST_STATE_PE_SINGLE, MockUploadedFile, INSTANCE_INCONSISTENT_STATE,
INSTANCE_INCONSISTENT_STATE2, INSTANCE_INCONSISTENT_STATE3, INSTANCE_INCONSISTENT_STATE4,
INSTANCE_INCONSISTENT_STATE5
)
from xblock.field_data import DictFieldData
......@@ -358,7 +360,7 @@ class OpenEndedModuleTest(unittest.TestCase):
# Create a module with no state yet. Important that this start off as a blank slate.
test_module = OpenEndedModule(self.test_system, self.location,
self.definition, self.descriptor, self.static_data, self.metadata)
self.definition, self.descriptor, self.static_data, self.metadata)
saved_response = "Saved response."
submitted_response = "Submitted response."
......@@ -369,7 +371,7 @@ class OpenEndedModuleTest(unittest.TestCase):
self.assertEqual(test_module.get_display_answer(), "")
# Now, store an answer in the module.
test_module.handle_ajax("store_answer", {'student_answer' : saved_response}, get_test_system())
test_module.handle_ajax("store_answer", {'student_answer': saved_response}, get_test_system())
# The stored answer should now equal our response.
self.assertEqual(test_module.stored_answer, saved_response)
self.assertEqual(test_module.get_display_answer(), saved_response)
......@@ -387,6 +389,7 @@ class OpenEndedModuleTest(unittest.TestCase):
# Confirm that the answer is stored properly.
self.assertEqual(test_module.latest_answer(), submitted_response)
class CombinedOpenEndedModuleTest(unittest.TestCase):
"""
Unit tests for the combined open ended xmodule
......@@ -610,7 +613,6 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
metadata=self.metadata,
instance_state={'task_states': TEST_STATE_SA_IN})
def test_get_score_realistic(self):
"""
Try to parse the correct score from a json instance state
......@@ -717,6 +719,175 @@ class CombinedOpenEndedModuleTest(unittest.TestCase):
self.ai_state_success(TEST_STATE_PE_SINGLE, iscore=0, tasks=[self.task_xml2])
class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase):
"""
Unit tests for the combined open ended xmodule rubric scores consistency.
"""
# location, definition_template, prompt, rubric, max_score, metadata, oeparam, task_xml1, task_xml2
# All these variables are used to construct the xmodule descriptor.
location = Location(["i4x", "edX", "open_ended", "combinedopenended",
"SampleQuestion"])
definition_template = """
<combinedopenended attempts="10000">
{rubric}
{prompt}
<task>
{task1}
</task>
<task>
{task2}
</task>
</combinedopenended>
"""
prompt = "<prompt>This is a question prompt</prompt>"
rubric = '''<rubric><rubric>
<category>
<description>Response Quality</description>
<option>The response is not a satisfactory answer to the question. It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
<option>Second option</option>
</category>
</rubric></rubric>'''
max_score = 10
metadata = {'attempts': '10', 'max_score': max_score}
oeparam = etree.XML('''
<openendedparam>
<initial_display>Enter essay here.</initial_display>
<answer_display>This is the answer.</answer_display>
<grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
</openendedparam>
''')
task_xml1 = '''
<selfassessment>
<hintprompt>
What hint about this problem would you give to someone?
</hintprompt>
<submitmessage>
Save Succcesful. Thanks for participating!
</submitmessage>
</selfassessment>
'''
task_xml2 = '''
<openended min_score_to_attempt="1" max_score_to_attempt="10">
<openendedparam>
<initial_display>Enter essay here.</initial_display>
<answer_display>This is the answer.</answer_display>
<grader_payload>{"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"}</grader_payload>
</openendedparam>
</openended>'''
static_data = {
'max_attempts': 20,
'prompt': prompt,
'rubric': rubric,
'max_score': max_score,
'display_name': 'Name',
'accept_file_upload': False,
'close_date': "",
's3_interface': test_util_open_ended.S3_INTERFACE,
'open_ended_grading_interface': test_util_open_ended.OPEN_ENDED_GRADING_INTERFACE,
'skip_basic_checks': False,
'graded': True,
}
definition = {'prompt': etree.XML(prompt), 'rubric': etree.XML(rubric), 'task_xml': [task_xml1, task_xml2]}
full_definition = definition_template.format(prompt=prompt, rubric=rubric, task1=task_xml1, task2=task_xml2)
descriptor = Mock(data=full_definition)
test_system = get_test_system()
test_system.open_ended_grading_interface = None
combinedoe_container = CombinedOpenEndedModule(
descriptor=descriptor,
runtime=test_system,
field_data=DictFieldData({
'data': full_definition,
'weight': '1',
}),
scope_ids=ScopeIds(None, None, None, None),
)
def setUp(self):
self.combinedoe = CombinedOpenEndedV1Module(self.test_system,
self.location,
self.definition,
self.descriptor,
static_data=self.static_data,
metadata=self.metadata,
instance_state=json.loads(INSTANCE_INCONSISTENT_STATE))
def test_get_score(self):
"""
If grader type is ML score should be updated from rubric scores. Aggregate rubric scores = sum([3])*5.
"""
score_dict = self.combinedoe.get_score()
self.assertEqual(score_dict['score'], 15.0)
self.assertEqual(score_dict['total'], 5.0)
def test_get_score_with_pe_grader(self):
"""
If grader type is PE score should not be updated from rubric scores. Aggregate rubric scores = sum([3])*5.
"""
combinedoe = CombinedOpenEndedV1Module(self.test_system,
self.location,
self.definition,
self.descriptor,
static_data=self.static_data,
metadata=self.metadata,
instance_state=json.loads(INSTANCE_INCONSISTENT_STATE2))
score_dict = combinedoe.get_score()
self.assertNotEqual(score_dict['score'], 15.0)
def test_get_score_with_different_score_value_in_rubric(self):
"""
If grader type is ML score should be updated from rubric scores. Aggregate rubric scores = sum([5])*5.
"""
combinedoe = CombinedOpenEndedV1Module(self.test_system,
self.location,
self.definition,
self.descriptor,
static_data=self.static_data,
metadata=self.metadata,
instance_state=json.loads(INSTANCE_INCONSISTENT_STATE3))
score_dict = combinedoe.get_score()
self.assertEqual(score_dict['score'], 25.0)
self.assertEqual(score_dict['total'], 5.0)
def test_get_score_with_old_task_states(self):
"""
If grader type is ML and old_task_states are present in instance inconsistent state score should be updated
from rubric scores. Aggregate rubric scores = sum([3])*5.
"""
combinedoe = CombinedOpenEndedV1Module(self.test_system,
self.location,
self.definition,
self.descriptor,
static_data=self.static_data,
metadata=self.metadata,
instance_state=json.loads(INSTANCE_INCONSISTENT_STATE4))
score_dict = combinedoe.get_score()
self.assertEqual(score_dict['score'], 15.0)
self.assertEqual(score_dict['total'], 5.0)
def test_get_score_with_score_missing(self):
"""
If grader type is ML and score field is missing in instance inconsistent state score should be updated from
rubric scores. Aggregate rubric scores = sum([3])*5.
"""
combinedoe = CombinedOpenEndedV1Module(self.test_system,
self.location,
self.definition,
self.descriptor,
static_data=self.static_data,
metadata=self.metadata,
instance_state=json.loads(INSTANCE_INCONSISTENT_STATE5))
score_dict = combinedoe.get_score()
self.assertEqual(score_dict['score'], 15.0)
self.assertEqual(score_dict['total'], 5.0)
class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore):
"""
Test the student flow in the combined open ended xmodule
......@@ -948,6 +1119,7 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore):
reset_data = json.loads(self._handle_ajax("reset", {}))
self.assertEqual(reset_data['success'], False)
class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
"""
Test if student is able to upload images properly.
......@@ -1018,7 +1190,7 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore):
# Simulate a student saving an answer with a link.
response = module.handle_ajax("save_answer", {
"student_answer": "{0} {1}".format(self.answer_text, self.answer_link)
})
})
response = json.loads(response)
......
common/lib/xmodule/xmodule/tests/test_util_open_ended.py
import json
from textwrap import dedent
from xmodule.modulestore import Location
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.tests import DATA_DIR, get_test_system
......@@ -98,12 +100,680 @@ class DummyModulestore(object):
descriptor.xmodule_runtime = self.get_module_system(descriptor)
return descriptor
def serialize_child_history(task_state):
"""
To json serialize feedback and post_assessment in child_history of task state.
"""
child_history = task_state.get("child_history", [])
for i, attempt in enumerate(child_history):
if "post_assessment" in attempt:
if "feedback" in attempt["post_assessment"]:
attempt["post_assessment"]["feedback"] = json.dumps(attempt["post_assessment"].get("feedback"))
task_state["child_history"][i]["post_assessment"] = json.dumps(attempt["post_assessment"])
def serialize_open_ended_instance_state(json_str):
"""
To json serialize task_states and old_task_states in instance state.
"""
json_data = json.loads(json_str)
task_states = json_data.get('task_states', [])
for i, task_state in enumerate(task_states):
serialize_child_history(task_state)
json_data['task_states'][i] = json.dumps(task_state)
old_task_states = json_data.get('old_task_states', [])
for i, old_task in enumerate(old_task_states):
for j, task_state in enumerate(old_task):
old_task[j] = json.dumps(task_state)
json_data['old_task_states'][i] = old_task
return json.dumps(json_data)
# Task state for a module with self assessment then instructor assessment.
TEST_STATE_SA_IN = ["{\"child_created\": false, \"child_attempts\": 2, \"version\": 1, \"child_history\": [{\"answer\": \"However venture pursuit he am mr cordial. Forming musical am hearing studied be luckily. Ourselves for determine attending how led gentleman sincerity. Valley afford uneasy joy she thrown though bed set. In me forming general prudent on country carried. Behaved an or suppose justice. Seemed whence how son rather easily and change missed. Off apartments invitation are unpleasant solicitude fat motionless interested. Hardly suffer wisdom wishes valley as an. As friendship advantages resolution it alteration stimulated he or increasing. \\r<br><br>Now led tedious shy lasting females off. Dashwood marianne in of entrance be on wondered possible building. Wondered sociable he carriage in speedily margaret. Up devonshire of he thoroughly insensible alteration. An mr settling occasion insisted distance ladyship so. Not attention say frankness intention out dashwoods now curiosity. Stronger ecstatic as no judgment daughter speedily thoughts. Worse downs nor might she court did nay forth these. \", \"post_assessment\": \"[3, 3, 2, 2, 2]\", \"score\": 12}, {\"answer\": \"Delightful remarkably mr on announcing themselves entreaties favourable. About to in so terms voice at. Equal an would is found seems of. The particular friendship one sufficient terminated frequently themselves. It more shed went up is roof if loud case. Delay music in lived noise an. Beyond genius really enough passed is up. \\r<br><br>John draw real poor on call my from. May she mrs furnished discourse extremely. Ask doubt noisy shade guest did built her him. Ignorant repeated hastened it do. Consider bachelor he yourself expenses no. Her itself active giving for expect vulgar months. Discovery commanded fat mrs remaining son she principle middleton neglected. Be miss he in post sons held. No tried is defer do money scale rooms. \", \"post_assessment\": \"[3, 3, 2, 2, 2]\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"done\"}", "{\"child_created\": false, \"child_attempts\": 0, \"version\": 1, \"child_history\": [{\"answer\": \"However venture pursuit he am mr cordial. Forming musical am hearing studied be luckily. Ourselves for determine attending how led gentleman sincerity. Valley afford uneasy joy she thrown though bed set. In me forming general prudent on country carried. Behaved an or suppose justice. Seemed whence how son rather easily and change missed. Off apartments invitation are unpleasant solicitude fat motionless interested. Hardly suffer wisdom wishes valley as an. As friendship advantages resolution it alteration stimulated he or increasing. \\r<br><br>Now led tedious shy lasting females off. Dashwood marianne in of entrance be on wondered possible building. Wondered sociable he carriage in speedily margaret. Up devonshire of he thoroughly insensible alteration. An mr settling occasion insisted distance ladyship so. Not attention say frankness intention out dashwoods now curiosity. Stronger ecstatic as no judgment daughter speedily thoughts. Worse downs nor might she court did nay forth these. \", \"post_assessment\": \"{\\\"submission_id\\\": 1460, \\\"score\\\": 12, \\\"feedback\\\": \\\"{\\\\\\\"feedback\\\\\\\": \\\\\\\"\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 5413, \\\"grader_type\\\": \\\"IN\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>\\\\nIdeas\\\\n</description><score>3</score><option points='0'>\\\\nDifficult for the reader to discern the main idea. Too brief or too repetitive to establish or maintain a focus.\\\\n</option><option points='1'>\\\\nAttempts a main idea. Sometimes loses focus or ineffectively displays focus.\\\\n</option><option points='2'>\\\\nPresents a unifying theme or main idea, but may include minor tangents. Stays somewhat focused on topic and task.\\\\n</option><option points='3'>\\\\nPresents a unifying theme or main idea without going off on tangents. Stays completely focused on topic and task.\\\\n</option></category><category><description>\\\\nContent\\\\n</description><score>3</score><option points='0'>\\\\nIncludes little information with few or no details or unrelated details. Unsuccessful in attempts to explore any facets of the topic.\\\\n</option><option points='1'>\\\\nIncludes little information and few or no details. Explores only one or two facets of the topic.\\\\n</option><option points='2'>\\\\nIncludes sufficient information and supporting details. (Details may not be fully developed; ideas may be listed.) Explores some facets of the topic.\\\\n</option><option points='3'>\\\\nIncludes in-depth information and exceptional supporting details that are fully developed. Explores all facets of the topic.\\\\n</option></category><category><description>\\\\nOrganization\\\\n</description><score>2</score><option points='0'>\\\\nIdeas organized illogically, transitions weak, and response difficult to follow.\\\\n</option><option points='1'>\\\\nAttempts to logically organize ideas. Attempts to progress in an order that enhances meaning, and demonstrates use of transitions.\\\\n</option><option points='2'>\\\\nIdeas organized logically. Progresses in an order that enhances meaning. Includes smooth transitions.\\\\n</option></category><category><description>\\\\nStyle\\\\n</description><score>2</score><option points='0'>\\\\nContains limited vocabulary, with many words used incorrectly. Demonstrates problems with sentence patterns.\\\\n</option><option points='1'>\\\\nContains basic vocabulary, with words that are predictable and common. Contains mostly simple sentences (although there may be an attempt at more varied sentence patterns).\\\\n</option><option points='2'>\\\\nIncludes vocabulary to make explanations detailed and precise. Includes varied sentence patterns, including complex sentences.\\\\n</option></category><category><description>\\\\nVoice\\\\n</description><score>2</score><option points='0'>\\\\nDemonstrates language and tone that may be inappropriate to task and reader.\\\\n</option><option points='1'>\\\\nDemonstrates an attempt to adjust language and tone to task and reader.\\\\n</option><option points='2'>\\\\nDemonstrates effective adjustment of language and tone to task and reader.\\\\n</option></category></rubric>\\\"}\", \"score\": 12}, {\"answer\": \"Delightful remarkably mr on announcing themselves entreaties favourable. About to in so terms voice at. Equal an would is found seems of. The particular friendship one sufficient terminated frequently themselves. It more shed went up is roof if loud case. Delay music in lived noise an. Beyond genius really enough passed is up. \\r<br><br>John draw real poor on call my from. May she mrs furnished discourse extremely. Ask doubt noisy shade guest did built her him. Ignorant repeated hastened it do. Consider bachelor he yourself expenses no. Her itself active giving for expect vulgar months. Discovery commanded fat mrs remaining son she principle middleton neglected. Be miss he in post sons held. No tried is defer do money scale rooms. \", \"post_assessment\": \"{\\\"submission_id\\\": 1462, \\\"score\\\": 12, \\\"feedback\\\": \\\"{\\\\\\\"feedback\\\\\\\": \\\\\\\"\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 5418, \\\"grader_type\\\": \\\"IN\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>\\\\nIdeas\\\\n</description><score>3</score><option points='0'>\\\\nDifficult for the reader to discern the main idea. Too brief or too repetitive to establish or maintain a focus.\\\\n</option><option points='1'>\\\\nAttempts a main idea. Sometimes loses focus or ineffectively displays focus.\\\\n</option><option points='2'>\\\\nPresents a unifying theme or main idea, but may include minor tangents. Stays somewhat focused on topic and task.\\\\n</option><option points='3'>\\\\nPresents a unifying theme or main idea without going off on tangents. Stays completely focused on topic and task.\\\\n</option></category><category><description>\\\\nContent\\\\n</description><score>3</score><option points='0'>\\\\nIncludes little information with few or no details or unrelated details. Unsuccessful in attempts to explore any facets of the topic.\\\\n</option><option points='1'>\\\\nIncludes little information and few or no details. Explores only one or two facets of the topic.\\\\n</option><option points='2'>\\\\nIncludes sufficient information and supporting details. (Details may not be fully developed; ideas may be listed.) Explores some facets of the topic.\\\\n</option><option points='3'>\\\\nIncludes in-depth information and exceptional supporting details that are fully developed. Explores all facets of the topic.\\\\n</option></category><category><description>\\\\nOrganization\\\\n</description><score>2</score><option points='0'>\\\\nIdeas organized illogically, transitions weak, and response difficult to follow.\\\\n</option><option points='1'>\\\\nAttempts to logically organize ideas. Attempts to progress in an order that enhances meaning, and demonstrates use of transitions.\\\\n</option><option points='2'>\\\\nIdeas organized logically. Progresses in an order that enhances meaning. Includes smooth transitions.\\\\n</option></category><category><description>\\\\nStyle\\\\n</description><score>2</score><option points='0'>\\\\nContains limited vocabulary, with many words used incorrectly. Demonstrates problems with sentence patterns.\\\\n</option><option points='1'>\\\\nContains basic vocabulary, with words that are predictable and common. Contains mostly simple sentences (although there may be an attempt at more varied sentence patterns).\\\\n</option><option points='2'>\\\\nIncludes vocabulary to make explanations detailed and precise. Includes varied sentence patterns, including complex sentences.\\\\n</option></category><category><description>\\\\nVoice\\\\n</description><score>2</score><option points='0'>\\\\nDemonstrates language and tone that may be inappropriate to task and reader.\\\\n</option><option points='1'>\\\\nDemonstrates an attempt to adjust language and tone to task and reader.\\\\n</option><option points='2'>\\\\nDemonstrates effective adjustment of language and tone to task and reader.\\\\n</option></category></rubric>\\\"}\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"post_assessment\"}"]
# Mock instance state. Should receive a score of 15.
MOCK_INSTANCE_STATE = r"""{"ready_to_reset": false, "skip_spelling_checks": true, "current_task_number": 1, "weight": 5.0, "graceperiod": "1 day 12 hours 59 minutes 59 seconds", "graded": "True", "task_states": ["{\"child_created\": false, \"child_attempts\": 4, \"version\": 1, \"child_history\": [{\"answer\": \"After 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\\r\\nStarting Mass (g)\\tEnding Mass (g)\\tDifference in Mass (g)\\r\\nMarble\\t 9.8\\t 9.4\\t\\u20130.4\\r\\nLimestone\\t10.4\\t 9.1\\t\\u20131.3\\r\\nWood\\t11.2\\t11.2\\t 0.0\\r\\nPlastic\\t 7.2\\t 7.1\\t\\u20130.1\\r\\nAfter reading the\", \"post_assessment\": \"[3]\", \"score\": 3}, {\"answer\": \"To replicate the experiment, the procedure would require more detail. One piece of information that is omitted is the amount of vinegar used in the experiment. It is also important to know what temperature the experiment was kept at during the 24 hours. Finally, the procedure needs to include details about the experiment, for example if the whole sample must be submerged.\", \"post_assessment\": \"[3]\", \"score\": 3}, {\"answer\": \"e the mass of four different samples.\\r\\nPour vinegar in each of four separate, but identical, containers.\\r\\nPlace a sample of one material into one container and label. Repeat with remaining samples, placing a single sample into a single container.\\r\\nAfter 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\", \"post_assessment\": \"[3]\", \"score\": 3}, {\"answer\": \"\", \"post_assessment\": \"[3]\", \"score\": 3}], \"max_score\": 3, \"child_state\": \"done\"}", "{\"child_created\": false, \"child_attempts\": 0, \"version\": 1, \"child_history\": [{\"answer\": \"The students\\u2019 data are recorded in the table below.\\r\\n\\r\\nStarting Mass (g)\\tEnding Mass (g)\\tDifference in Mass (g)\\r\\nMarble\\t 9.8\\t 9.4\\t\\u20130.4\\r\\nLimestone\\t10.4\\t 9.1\\t\\u20131.3\\r\\nWood\\t11.2\\t11.2\\t 0.0\\r\\nPlastic\\t 7.2\\t 7.1\\t\\u20130.1\\r\\nAfter reading the group\\u2019s procedure, describe what additional information you would need in order to replicate the expe\", \"post_assessment\": \"{\\\"submission_id\\\": 3097, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: More grammar errors than average.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"the students data are recorded in the <bg>table below . starting mass</bg> g ending mass g difference in mass g marble . . . limestone . . . wood . . . plastic . . . after reading the groups <bg>procedure , describe what additional</bg> information you would need in order to replicate the <bs>expe</bs>\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3233, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>Response Quality</description><score>0</score><option points='0'>The response is not a satisfactory answer to the question. It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option><option points='1'>The response is a marginal answer to the question. It may contain some elements of a proficient response, but it is inaccurate or incomplete.</option><option points='2'>The response is a proficient answer to the question. It is generally correct, although it may contain minor inaccuracies. There is limited evidence of higher-order thinking.</option><option points='3'>The response is correct, complete, and contains evidence of higher-order thinking.</option></category></rubric>\\\"}\", \"score\": 0}, {\"answer\": \"After 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\\r\\nStarting Mass (g)\\tEnding Mass (g)\\tDifference in Mass (g)\\r\\nMarble\\t 9.8\\t 9.4\\t\\u20130.4\\r\\nLimestone\\t10.4\\t 9.1\\t\\u20131.3\\r\\nWood\\t11.2\\t11.2\\t 0.0\\r\\nPlastic\\t 7.2\\t 7.1\\t\\u20130.1\\r\\nAfter reading the\", \"post_assessment\": \"{\\\"submission_id\\\": 3098, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"after hours , remove the samples from the containers and rinse each sample with distilled water . allow the samples to sit and dry for minutes . determine the mass of each sample . the students data are recorded in the <bg>table below . starting mass</bg> g ending mass g difference in mass g marble . . . limestone . . . wood . . . plastic . . . after reading the\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3235, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>Response Quality</description><score>0</score><option points='0'>The response is not a satisfactory answer to the question. It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option><option points='1'>The response is a marginal answer to the question. It may contain some elements of a proficient response, but it is inaccurate or incomplete.</option><option points='2'>The response is a proficient answer to the question. It is generally correct, although it may contain minor inaccuracies. There is limited evidence of higher-order thinking.</option><option points='3'>The response is correct, complete, and contains evidence of higher-order thinking.</option></category></rubric>\\\"}\", \"score\": 0}, {\"answer\": \"To replicate the experiment, the procedure would require more detail. One piece of information that is omitted is the amount of vinegar used in the experiment. It is also important to know what temperature the experiment was kept at during the 24 hours. Finally, the procedure needs to include details about the experiment, for example if the whole sample must be submerged.\", \"post_assessment\": \"{\\\"submission_id\\\": 3099, \\\"score\\\": 3, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"to replicate the experiment , the procedure would require <bg>more detail . one</bg> piece of information <bg>that is omitted is the</bg> amount of vinegar used in the experiment . it is also important to know what temperature the experiment was kept at during the hours . finally , the procedure needs to include details about the experiment , for example if the whole sample must be submerged .\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3237, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>Response Quality</description><score>3</score><option points='0'>The response is not a satisfactory answer to the question. It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option><option points='1'>The response is a marginal answer to the question. It may contain some elements of a proficient response, but it is inaccurate or incomplete.</option><option points='2'>The response is a proficient answer to the question. It is generally correct, although it may contain minor inaccuracies. There is limited evidence of higher-order thinking.</option><option points='3'>The response is correct, complete, and contains evidence of higher-order thinking.</option></category></rubric>\\\"}\", \"score\": 3}, {\"answer\": \"e the mass of four different samples.\\r\\nPour vinegar in each of four separate, but identical, containers.\\r\\nPlace a sample of one material into one container and label. Repeat with remaining samples, placing a single sample into a single container.\\r\\nAfter 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\", \"post_assessment\": \"{\\\"submission_id\\\": 3100, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"e the mass of four different samples . pour vinegar in <bg>each of four separate</bg> , but identical , containers . place a sample of one material into one container and label . repeat with remaining samples , placing a single sample into a single container . after hours , remove the samples from the containers and rinse each sample with distilled water . allow the samples to sit and dry for minutes . determine the mass of each sample . the students data are recorded in the table below . \\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3239, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>Response Quality</description><score>0</score><option points='0'>The response is not a satisfactory answer to the question. It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option><option points='1'>The response is a marginal answer to the question. It may contain some elements of a proficient response, but it is inaccurate or incomplete.</option><option points='2'>The response is a proficient answer to the question. It is generally correct, although it may contain minor inaccuracies. There is limited evidence of higher-order thinking.</option><option points='3'>The response is correct, complete, and contains evidence of higher-order thinking.</option></category></rubric>\\\"}\", \"score\": 0}, {\"answer\": \"\", \"post_assessment\": \"{\\\"submission_id\\\": 3101, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"invalid essay .\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3241, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>Response Quality</description><score>0</score><option points='0'>The response is not a satisfactory answer to the question. It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option><option points='1'>The response is a marginal answer to the question. It may contain some elements of a proficient response, but it is inaccurate or incomplete.</option><option points='2'>The response is a proficient answer to the question. It is generally correct, although it may contain minor inaccuracies. There is limited evidence of higher-order thinking.</option><option points='3'>The response is correct, complete, and contains evidence of higher-order thinking.</option></category></rubric>\\\"}\", \"score\": 0}], \"max_score\": 3, \"child_state\": \"done\"}"], "attempts": "10000", "student_attempts": 0, "due": null, "state": "done", "accept_file_upload": false, "display_name": "Science Question -- Machine Assessed"}"""
# Instance state. To test the rubric scores are consistent. Should receive a score of 15.
INSTANCE_INCONSISTENT_STATE = serialize_open_ended_instance_state("""
{ "accept_file_upload" : false,
"attempts" : "10000",
"current_task_number" : 1,
"display_name" : "Science Question -- Machine Assessed",
"due" : null,
"graceperiod" : "1 day 12 hours 59 minutes 59 seconds",
"graded" : "True",
"ready_to_reset" : false,
"skip_spelling_checks" : true,
"state" : "done",
"student_attempts" : 0,
"task_states" : [ { "child_attempts" : 4,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : [ 3 ],
"score" : 1
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : [ 3 ],
"score" : 1
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : [ 3 ],
"score" : 1
},
{ "answer" : "",
"post_assessment" : [ 3 ],
"score" : 1
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
},
{ "child_attempts" : 0,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3233,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3097,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3235,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3098,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3237,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>3</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 2,
"submission_id" : 3099,
"success" : true
},
"score" : 2
},
{ "answer" : "Student answer 4th attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3239,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3100,
"success" : true
},
"score" : 0
},
{ "answer" : "",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "invalid essay .",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3241,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3101,
"success" : true
},
"score" : 0
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
}
],
"weight" : 5.0
}
""")
# Instance state. Should receive a score of 10 if grader type is PE.
INSTANCE_INCONSISTENT_STATE2 = serialize_open_ended_instance_state("""
{ "accept_file_upload" : false,
"attempts" : "10000",
"current_task_number" : 1,
"display_name" : "Science Question -- Machine Assessed",
"due" : null,
"graceperiod" : "1 day 12 hours 59 minutes 59 seconds",
"graded" : "True",
"ready_to_reset" : false,
"skip_spelling_checks" : true,
"state" : "done",
"student_attempts" : 0,
"task_states" : [ { "child_attempts" : 4,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "",
"post_assessment" : [3],
"score" : 1
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
},
{ "child_attempts" : 0,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3233,
"grader_type" : "PE",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3097,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3235,
"grader_type" : "PE",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3098,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3237,
"grader_type" : "PE",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>5</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 2,
"submission_id" : 3099,
"success" : true
},
"score" : 2
},
{ "answer" : "Student answer 4th attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3239,
"grader_type" : "PE",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3100,
"success" : true
},
"score" : 0
},
{ "answer" : "",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "invalid essay .",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3241,
"grader_type" : "PE",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3101,
"success" : true
},
"score" : 0
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
}
],
"weight" : 5.0
}
""")
# Instance state. To test score if sum of rubric score is different from score value. Should receive score of 25.
INSTANCE_INCONSISTENT_STATE3 = serialize_open_ended_instance_state("""
{ "accept_file_upload" : false,
"attempts" : "10000",
"current_task_number" : 1,
"display_name" : "Science Question -- Machine Assessed",
"due" : null,
"graceperiod" : "1 day 12 hours 59 minutes 59 seconds",
"graded" : "True",
"ready_to_reset" : false,
"skip_spelling_checks" : true,
"state" : "done",
"student_attempts" : 0,
"task_states" : [ { "child_attempts" : 4,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "",
"post_assessment" : [3],
"score" : 1
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
},
{ "child_attempts" : 0,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3233,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>2</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3097,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3235,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3098,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3237,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>5</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 2,
"submission_id" : 3099,
"success" : true
},
"score" : 2
},
{ "answer" : "Student answer 4th attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3239,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3100,
"success" : true
},
"score" : 0
},
{ "answer" : "",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "invalid essay .",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3241,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3101,
"success" : true
},
"score" : 0
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
}
],
"weight" : 5.0
}
""")
# Instance state. To test score if old task states are available. Should receive a score of 15.
INSTANCE_INCONSISTENT_STATE4 = serialize_open_ended_instance_state("""
{ "accept_file_upload" : false,
"attempts" : "10000",
"current_task_number" : 0,
"display_name" : "Science Question -- Machine Assessed",
"due" : null,
"graceperiod" : "1 day 12 hours 59 minutes 59 seconds",
"graded" : "True",
"old_task_states" : [ [ { "child_attempts" : 4,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : "[3]",
"score" : 1
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : "[3]",
"score" : 1
},
{ "answer" : "Student answer 3rd attempt.",
"post_assesssment" : "[3]",
"score" : 1
},
{ "answer" : "",
"post_assessment" : "[3]",
"score" : 1
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
} ] ],
"ready_to_reset" : false,
"skip_spelling_checks" : true,
"state" : "assessing",
"student_attempts" : 0,
"task_states" : [ { "child_attempts" : 4,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "",
"post_assessment" : [3],
"score" : 1
}
],
"child_state" : "done",
"max_score" : 3,
"stored_answer" : null,
"version" : 1
},
{ "child_attempts" : 0,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3233,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3097,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3235,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3098,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3237,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>3</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 2,
"submission_id" : 3099,
"success" : true
},
"score" : 2
},
{ "answer" : "Student answer 4th attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3239,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3100,
"success" : true
},
"score" : 0
},
{ "answer" : "",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "invalid essay .",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3241,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3101,
"success" : true
},
"score" : 0
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
}
],
"weight" : 5.0
}
""")
# Instance state. To test score if rubric scores are available but score is missing. Should receive a score of 15.
INSTANCE_INCONSISTENT_STATE5 = serialize_open_ended_instance_state("""
{ "accept_file_upload" : false,
"attempts" : "10000",
"current_task_number" : 1,
"display_name" : "Science Question -- Machine Assessed",
"due" : null,
"graceperiod" : "1 day 12 hours 59 minutes 59 seconds",
"graded" : "True",
"ready_to_reset" : false,
"skip_spelling_checks" : true,
"state" : "done",
"student_attempts" : 0,
"task_states" : [ { "child_attempts" : 4,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : [3],
"score" : 1
},
{ "answer" : "",
"post_assessment" : [3],
"score" : 1
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
},
{ "child_attempts" : 0,
"child_created" : false,
"child_history" : [ { "answer" : "Student answer 1st attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3233,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3097,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 2nd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3235,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3098,
"success" : true
},
"score" : 0
},
{ "answer" : "Student answer 3rd attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3237,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>3</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 2,
"submission_id" : 3099,
"success" : true
}
},
{ "answer" : "Student answer 4th attempt.",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "valid essay",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3239,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3100,
"success" : true
},
"score" : 0
},
{ "answer" : "",
"post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.",
"markup-text" : "invalid essay .",
"spelling" : "Spelling: Ok."
},
"grader_id" : 3241,
"grader_type" : "ML",
"rubric_scores_complete" : true,
"rubric_xml" : "<rubric><category><description>Response Quality</description><score>0</score><option points='0'>Category one description.</option><option points='1'>Category two description.</option><option points='2'>Category three description.</option><option points='3'>Category four description.</option></category></rubric>",
"score" : 0,
"submission_id" : 3101,
"success" : true
},
"score" : 0
}
],
"child_state" : "done",
"max_score" : 3,
"version" : 1
}
],
"weight" : 5.0
}
""")
# Task state with self assessment only.
TEST_STATE_SA = ["{\"child_created\": false, \"child_attempts\": 1, \"version\": 1, \"child_history\": [{\"answer\": \"Censorship in the Libraries\\r<br>'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author\\r<br><br>Write a persuasive essay to a newspaper reflecting your vies on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.\", \"post_assessment\": \"[3, 3, 2, 2, 2]\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"done\"}", "{\"child_created\": false, \"child_attempts\": 0, \"version\": 1, \"child_history\": [{\"answer\": \"Censorship in the Libraries\\r<br>'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author\\r<br><br>Write a persuasive essay to a newspaper reflecting your vies on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.\", \"post_assessment\": \"{\\\"submission_id\\\": 1461, \\\"score\\\": 12, \\\"feedback\\\": \\\"{\\\\\\\"feedback\\\\\\\": \\\\\\\"\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 5414, \\\"grader_type\\\": \\\"IN\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"<rubric><category><description>\\\\nIdeas\\\\n</description><score>3</score><option points='0'>\\\\nDifficult for the reader to discern the main idea. Too brief or too repetitive to establish or maintain a focus.\\\\n</option><option points='1'>\\\\nAttempts a main idea. Sometimes loses focus or ineffectively displays focus.\\\\n</option><option points='2'>\\\\nPresents a unifying theme or main idea, but may include minor tangents. Stays somewhat focused on topic and task.\\\\n</option><option points='3'>\\\\nPresents a unifying theme or main idea without going off on tangents. Stays completely focused on topic and task.\\\\n</option></category><category><description>\\\\nContent\\\\n</description><score>3</score><option points='0'>\\\\nIncludes little information with few or no details or unrelated details. Unsuccessful in attempts to explore any facets of the topic.\\\\n</option><option points='1'>\\\\nIncludes little information and few or no details. Explores only one or two facets of the topic.\\\\n</option><option points='2'>\\\\nIncludes sufficient information and supporting details. (Details may not be fully developed; ideas may be listed.) Explores some facets of the topic.\\\\n</option><option points='3'>\\\\nIncludes in-depth information and exceptional supporting details that are fully developed. Explores all facets of the topic.\\\\n</option></category><category><description>\\\\nOrganization\\\\n</description><score>2</score><option points='0'>\\\\nIdeas organized illogically, transitions weak, and response difficult to follow.\\\\n</option><option points='1'>\\\\nAttempts to logically organize ideas. Attempts to progress in an order that enhances meaning, and demonstrates use of transitions.\\\\n</option><option points='2'>\\\\nIdeas organized logically. Progresses in an order that enhances meaning. Includes smooth transitions.\\\\n</option></category><category><description>\\\\nStyle\\\\n</description><score>2</score><option points='0'>\\\\nContains limited vocabulary, with many words used incorrectly. Demonstrates problems with sentence patterns.\\\\n</option><option points='1'>\\\\nContains basic vocabulary, with words that are predictable and common. Contains mostly simple sentences (although there may be an attempt at more varied sentence patterns).\\\\n</option><option points='2'>\\\\nIncludes vocabulary to make explanations detailed and precise. Includes varied sentence patterns, including complex sentences.\\\\n</option></category><category><description>\\\\nVoice\\\\n</description><score>2</score><option points='0'>\\\\nDemonstrates language and tone that may be inappropriate to task and reader.\\\\n</option><option points='1'>\\\\nDemonstrates an attempt to adjust language and tone to task and reader.\\\\n</option><option points='2'>\\\\nDemonstrates effective adjustment of language and tone to task and reader.\\\\n</option></category></rubric>\\\"}\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"post_assessment\"}"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment