Commit 1dbf9d97 by Stephen Sanchez

Adding Workflow API support for AI grading

parent 22732a0f
......@@ -24,6 +24,62 @@ from openassessment.assessment.worker import grading as grading_tasks
logger = logging.getLogger(__name__)
def submitter_is_finished(submission_uuid, requirements):
"""
Determine if the submitter has finished their requirements for Example
Based Assessment. Always returns True.
Args:
submission_uuid (str): Not used.
requirements (dict): Not used.
Returns:
True
"""
return True
def assessment_is_finished(submission_uuid, requirements):
"""
Determine if the assessment of the given submission is completed. This
checks to see if the AI has completed the assessment.
Args:
submission_uuid (str): The UUID of the submission being graded.
requirements (dict): Not used.
Returns:
True if the assessment has been completed for this submission.
"""
return bool(get_latest_assessment(submission_uuid))
def get_score(submission_uuid, requirements):
"""
Generate a score based on a completed assessment for the given submission.
If no assessment has been completed for this submission, this will return
None.
Args:
submission_uuid (str): The UUID for the submission to get a score for.
requirements (dict): Not used.
Returns:
A dictionary with the points earned and points possible.
"""
assessment = get_latest_assessment(submission_uuid)
if not assessment:
return None
return {
"points_earned": assessment["points_earned"],
"points_possible": assessment["points_possible"]
}
def submit(submission_uuid, rubric, algorithm_id):
"""
Submit a response for AI assessment.
......
......@@ -62,6 +62,22 @@ AI_ALGORITHMS = {
}
def train_classifiers(rubric_dict, classifier_score_overrides):
"""
Simple utility function to train classifiers.
Args:
rubric_dict (dict): The rubric to train the classifiers on.
classifier_score_overrides (dict): A dictionary of classifier overrides
to set the scores for the given submission.
"""
rubric = rubric_from_dict(rubric_dict)
AIClassifierSet.create_classifier_set(
classifier_score_overrides, rubric, ALGORITHM_ID
)
class AITrainingTest(CacheResetTest):
"""
Tests for AI training tasks.
......@@ -161,15 +177,7 @@ class AIGradingTest(CacheResetTest):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
# Create the classifier set for our fake AI algorithm
# To isolate these tests from the tests for the training
# task, we use the database models directly.
# We also use a stub AI algorithm that simply returns
# whatever scores we specify in the classifier data.
rubric = rubric_from_dict(RUBRIC)
AIClassifierSet.create_classifier_set(
self.CLASSIFIER_SCORE_OVERRIDES, rubric, ALGORITHM_ID
)
train_classifiers(RUBRIC, self.CLASSIFIER_SCORE_OVERRIDES)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_grade_essay(self):
......@@ -185,6 +193,11 @@ class AIGradingTest(CacheResetTest):
expected_score = self.CLASSIFIER_SCORE_OVERRIDES[criterion_name]['score_override']
self.assertEqual(part['option']['points'], expected_score)
score = ai_api.get_score(self.submission_uuid, {})
self.assertEquals(score["points_possible"], 4)
self.assertEquals(score["points_earned"], 3)
@mock.patch('openassessment.assessment.api.ai.grading_tasks.grade_essay')
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_submit_no_classifiers_available(self, mock_task):
......@@ -237,3 +250,17 @@ class AIGradingTest(CacheResetTest):
mock_call.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AIGradingInternalError):
ai_api.get_latest_assessment(self.submission_uuid)
class AIUntrainedGradingTest:
"""
Tests that do not run the setup to train classifiers.
"""
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_no_score(self):
# Test that no score has been created, and get_score returns None.
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
score = ai_api.get_score(self.submission_uuid, {})
self.assertIsNone(score)
\ No newline at end of file
......@@ -131,7 +131,7 @@ class AIAlgorithm(object):
try:
algorithm_cls = getattr(importlib.import_module(module_path), name)
return algorithm_cls()
except (ImportError, AttributeError):
except (ImportError, ValueError, AttributeError):
raise AlgorithmLoadError(algorithm_id, cls_path)
......
......@@ -116,6 +116,30 @@
{% endif %}
{% endfor %}
{% for part in example_based_assessment.parts %}
{% if part.option.criterion.name == criterion.name %}
<li class="answer example-based-assessment"
id="question--{{ criterion_num }}__answer--example-based">
<h5 class="answer__title">
<span class="answer__source">
<span class="answer__source__value">{% trans "Example Based Assessment" %}</span>
</span>
<span class="answer__value">
<span class="answer__value__label sr">{% trans "Example Based Assessment" %}: </span>
<span class="answer__value__value">
{{ part.option.name }}
<span class="ui-hint hint--top" data-hint="{{ part.option.explanation }}">
<i class="ico icon-info-sign"
title="{% blocktrans with name=part.option.name %}More information about {{ name }}{% endblocktrans %}"></i>
</span>
</span>
</span>
</h5>
</li>
{% endif %}
{% endfor %}
{% if criterion.feedback %}
<li class="answer--feedback ui-toggle-visibility is--collapsed">
<h5 class="answer--feedback__title ui-toggle-visibility__control">
......
......@@ -6,7 +6,7 @@
<span class="wrapper--copy">
<span class="step__label">{% trans "Your Grade" %}:</span>
<span class="grade__value">
<span class="grade__value__title">{% trans "Waiting for Peer Assessment" %}</span>
<span class="grade__value__title">{% trans "Waiting for Assessments" %}</span>
</span>
</span>
</h2>
......@@ -16,7 +16,13 @@
<div class="wrapper--step__content">
<div class="step__content">
<div class="grade__value__description">
<p>{% trans "Your response is still undergoing peer assessment. After your peers have assessed your response, you'll see their comments and receive your final grade." %}</p>
{% if waiting == 'peer' %}
<p>{% trans "Your response is still undergoing peer assessment. After your peers have assessed your response, you'll see their comments and receive your final grade." %}</p>
{% elif waiting == 'example-based' %}
<p>{% trans "Your response is still undergoing example based assessment. After your response has been assessed, you'll see the comments and receive your final grade." %}</p>
{% elif waiting == 'all' %}
<p>{% trans "Your response is still undergoing peer assessment and example based assessment. After your example based assessment has been generated and peers have assessed your response, you'll see their comments and receive your final grade." %}</p>
{% endif %}
</div>
</div>
</div>
......
......@@ -4,8 +4,12 @@
<h3 class="message__title">{% trans "You Have Completed This Assignment" %} </h3>
<div class="message__content">
<p>
{% if waiting %}
{% trans "Your grade will be available when your peers have completed their assessments of your response." %}
{% if waiting == 'peer' %}
<p>{% trans "Your grade will be available when your peers have completed their assessments of your response." %}</p>
{% elif waiting == 'example-based' %}
<p>{% trans "Your grade will be available when the example based assessment of your response has been generated." %}</p>
{% elif waiting == 'all' %}
<p>{% trans "Your grade will be available when your peers have completed their assessments of your response, and an example based assessment of your response has been generated." %}</p>
{% else %}
{% blocktrans %}
Review <a data-behavior="ui-scroll" href="#openassessment__grade"> your grade and your assessment details</a>.
......
......@@ -8,7 +8,8 @@ import logging
from django.db import DatabaseError
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.errors import PeerAssessmentError
from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.errors import PeerAssessmentError, AIError
from submissions import api as sub_api
from .models import AssessmentWorkflow, AssessmentWorkflowStep
from .serializers import AssessmentWorkflowSerializer
......@@ -59,7 +60,7 @@ class AssessmentWorkflowNotFoundError(AssessmentWorkflowError):
pass
def create_workflow(submission_uuid, steps):
def create_workflow(submission_uuid, steps, rubric=None, algorithm_id=None):
"""Begins a new assessment workflow.
Create a new workflow that other assessments will record themselves against.
......@@ -67,6 +68,11 @@ def create_workflow(submission_uuid, steps):
Args:
submission_uuid (str): The UUID for the submission that all our
assessments will be evaluating.
rubric (dict): The rubric that will be used for grading in this workflow.
The rubric is only used when Example Based Assessment is configured.
algorithm_id (str): The version of the AI that will be used for evaluating
submissions, if Example Based Assessments are configured for this
location.
steps (list): List of steps that are part of the workflow, in the order
that the user must complete them. Example: `["peer", "self"]`
......@@ -128,17 +134,34 @@ def create_workflow(submission_uuid, steps):
# We're not using a serializer to deserialize this because the only variable
# we're getting from the outside is the submission_uuid, which is already
# validated by this point.
status = AssessmentWorkflow.STATUS.peer
if steps[0] == "peer":
status = AssessmentWorkflow.STATUS.waiting
step = steps[0]
# AI will not set the Workflow Status, since it will immediately replace the
# status with the next step, or "waiting".
if step == "ai":
step = steps[1] if len(steps) > 1 else None
if not rubric or not algorithm_id:
err_msg = u"Rubric and Algorithm ID must be configured for Example Based Assessment."
raise AssessmentWorkflowInternalError(err_msg)
try:
ai_api.submit(submission_uuid, rubric, algorithm_id)
except AIError as err:
err_msg = u"Could not submit submission for Example Based Grading: {}".format(err)
logger.exception(err_msg)
raise AssessmentWorkflowInternalError(err_msg)
if step == "peer":
status = AssessmentWorkflow.STATUS.peer
try:
peer_api.create_peer_workflow(submission_uuid)
except PeerAssessmentError as err:
err_msg = u"Could not create assessment workflow: {}".format(err)
logger.exception(err_msg)
raise AssessmentWorkflowInternalError(err_msg)
elif steps[0] == "self":
elif step == "self":
status = AssessmentWorkflow.STATUS.self
elif steps[0] == "training":
elif step == "training":
status = AssessmentWorkflow.STATUS.training
try:
......
......@@ -50,7 +50,8 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
STEPS = [
"peer", # User needs to assess peer submissions
"self", # User needs to assess themselves
"training", # User needs to practice grading using example essays
"training", # User needs to practice grading using example essays
"ai", # User submission will be graded by AI.
]
STATUSES = [
......@@ -95,7 +96,11 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
"complete": step.api().submitter_is_finished(
self.submission_uuid,
assessment_requirements.get(step.name, {})
)
),
"graded": step.api().assessment_is_finished(
self.submission_uuid,
assessment_requirements.get(step.name, {})
),
}
return status_dict
......@@ -130,6 +135,7 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
specific requirements in this dict.
"""
from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
......@@ -174,6 +180,8 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
)
elif self.STATUS.self in step_names:
score = self_api.get_score(self.submission_uuid, {})
elif self.STATUS.ai in step_names:
score = ai_api.get_score(self.submission_uuid, {})
if score:
self.set_score(score)
......@@ -266,6 +274,7 @@ class AssessmentWorkflowStep(models.Model):
Returns an API associated with this workflow step. If no API is
associated with this workflow step, None is returned.
"""
from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
from openassessment.assessment.api import student_training
......@@ -276,6 +285,9 @@ class AssessmentWorkflowStep(models.Model):
api = peer_api
elif self.name == AssessmentWorkflow.STATUS.training:
api = student_training
elif self.name == AssessmentWorkflow.STATUS.ai:
api = ai_api
return api
def update(self, submission_uuid, assessment_requirements):
......
......@@ -57,6 +57,66 @@
},
"self": {}
}
},
"ai": {
"steps": ["ai"],
"requirements": {
"ai": {}
}
},
"ai_peer": {
"steps": ["ai", "peer"],
"requirements": {
"ai": {},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
}
}
},
"ai_training_peer": {
"steps": ["ai", "training", "peer"],
"requirements": {
"ai": {},
"training": {
"num_required": 2
},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
}
}
},
"ai_self": {
"steps": ["ai", "self"],
"requirements": {
"ai": {},
"self": {}
}
},
"ai_peer_self": {
"steps": ["ai", "peer", "self"],
"requirements": {
"ai": {},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
},
"self": {}
}
},
"ai_training_peer_self": {
"steps": ["ai", "training", "peer", "self"],
"requirements": {
"ai": {},
"training": {
"num_required": 2
},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
},
"self": {}
}
}
}
\ No newline at end of file
......@@ -9,9 +9,26 @@ from openassessment.test_utils import CacheResetTest
from openassessment.workflow.models import AssessmentWorkflow
from submissions.models import Submission
import openassessment.workflow.api as workflow_api
from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.errors import AIError
from openassessment.assessment.models import StudentTrainingWorkflow
import submissions.api as sub_api
RUBRIC_DICT = {
"criteria": [
{
"name": "secret",
"prompt": "Did the writer keep it secret?",
"options": [
{"name": "no", "points": "0", "explanation": ""},
{"name": "yes", "points": "1", "explanation": ""},
]
},
]
}
ALGORITHM_ID = "Ease"
ITEM_1 = {
"student_id": "Optimus Prime 001",
"item_id": "Matrix of Leadership",
......@@ -25,8 +42,10 @@ class TestAssessmentWorkflowApi(CacheResetTest):
@ddt.file_data('data/assessments.json')
def test_create_workflow(self, data):
first_step = data["steps"][0] if data["steps"] else "peer"
if "ai" in data["steps"]:
first_step = data["steps"][1] if len(data["steps"]) > 1 else "waiting"
submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
workflow = workflow_api.create_workflow(submission["uuid"], data["steps"])
workflow = workflow_api.create_workflow(submission["uuid"], data["steps"], RUBRIC_DICT, ALGORITHM_ID)
workflow_keys = set(workflow.keys())
self.assertEqual(
......@@ -55,7 +74,7 @@ class TestAssessmentWorkflowApi(CacheResetTest):
def test_update_peer_workflow(self):
submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"])
workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"], RUBRIC_DICT, ALGORITHM_ID)
StudentTrainingWorkflow.get_or_create_workflow(submission_uuid=submission["uuid"])
requirements = {
"training": {
......@@ -104,6 +123,32 @@ class TestAssessmentWorkflowApi(CacheResetTest):
with self.assertRaises(workflow_api.AssessmentWorkflowRequestError):
workflow = workflow_api.create_workflow(123, data["steps"])
@patch.object(ai_api, 'assessment_is_finished')
@patch.object(ai_api, 'get_score')
def test_ai_score_set(self, mock_score, mock_is_finished):
submission = sub_api.create_submission(ITEM_1, "Ultra Magnus fumble")
workflow_api.create_workflow(submission["uuid"], ["ai"], RUBRIC_DICT, ALGORITHM_ID)
mock_is_finished.return_value = True
score = {"points_earned": 7, "points_possible": 10}
mock_score.return_value = score
workflow = workflow_api.get_workflow_for_submission(submission["uuid"], {})
self.assertEquals(workflow["score"]["points_earned"], score["points_earned"])
self.assertEquals(workflow["score"]["points_possible"], score["points_possible"])
@ddt.data((RUBRIC_DICT, None), (None, ALGORITHM_ID))
@ddt.unpack
@raises(workflow_api.AssessmentWorkflowInternalError)
def test_create_ai_workflow_no_rubric(self, rubric, algorithm_id):
submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
workflow_api.create_workflow(submission["uuid"], ["ai"], rubric, algorithm_id)
@patch.object(ai_api, 'submit')
@raises(workflow_api.AssessmentWorkflowInternalError)
def test_ai_submit_failures(self, mock_submit):
mock_submit.side_effect = AIError("Kaboom!")
submission = sub_api.create_submission(ITEM_1, "Ultra Magnus fumble")
workflow_api.create_workflow(submission["uuid"], ["ai"], RUBRIC_DICT, ALGORITHM_ID)
@patch.object(Submission.objects, 'get')
@ddt.file_data('data/assessments.json')
@raises(workflow_api.AssessmentWorkflowInternalError)
......
......@@ -5,6 +5,7 @@ from nose.tools import raises
from openassessment.workflow.models import emit_event
from openassessment.workflow.test.events import fake_event_logger
class TestEmitEvent(TestCase):
def test_emit_wired_correctly(self):
......
......@@ -51,7 +51,7 @@ class GradeMixin(object):
if status == "done":
path, context = self.render_grade_complete(workflow)
elif status == "waiting":
path = 'openassessmentblock/grade/oa_grade_waiting.html'
path, context = self.render_grade_waiting(workflow)
elif status is None:
path = 'openassessmentblock/grade/oa_grade_not_started.html'
else: # status is 'self' or 'peer', which implies that the workflow is incomplete
......@@ -61,6 +61,22 @@ class GradeMixin(object):
else:
return self.render_assessment(path, context)
def render_grade_waiting(self, workflow):
"""
Render the grade waiting state.
Args:
workflow (dict): The serialized Workflow model.
Returns:
tuple of context (dict) and template_path (string)
"""
context = {
"waiting": self.get_waiting_details(workflow["status_details"])
}
return 'openassessmentblock/grade/oa_grade_waiting.html', context
def render_grade_complete(self, workflow):
"""
Render the grade complete state.
......
......@@ -38,6 +38,8 @@ class MessageMixin(object):
# Finds the cannonical status of the workflow and the is_closed status of the problem
status = workflow.get('status')
status_details = workflow.get('status_details', {})
is_closed = deadline_info.get('general').get('is_closed')
# Finds the status_information which describes the closed status of the current step (defaults to submission)
......@@ -53,7 +55,7 @@ class MessageMixin(object):
# Render the instruction message based on the status of the workflow
# and the closed status.
if status == "done" or status == "waiting":
path, context = self.render_message_complete(status)
path, context = self.render_message_complete(status_details)
elif is_closed or status_is_closed:
path, context = self.render_message_closed(status_info)
elif status == "self":
......@@ -66,7 +68,7 @@ class MessageMixin(object):
path, context = self.render_message_open(deadline_info)
return self.render_assessment(path, context)
def render_message_complete(self, status):
def render_message_complete(self, status_details):
"""
Renders the "Complete" message state (Either Waiting or Done)
......@@ -76,10 +78,10 @@ class MessageMixin(object):
Returns:
The path (String) and context (dict) to render the "Complete" message template
"""
context = {
"waiting": (status == "waiting")
"waiting": self.get_waiting_details(status_details),
}
return 'openassessmentblock/message/oa_message_complete.html', context
def render_message_training(self, deadline_info):
......
......@@ -292,8 +292,9 @@ class OpenAssessmentBlock(
"""
ui_models = [UI_MODELS["submission"]]
for assessment in self.valid_assessments:
ui_model = UI_MODELS[assessment["name"]]
ui_models.append(dict(assessment, **ui_model))
ui_model = UI_MODELS.get(assessment["name"])
if ui_model:
ui_models.append(dict(assessment, **ui_model))
ui_models.append(UI_MODELS["grade"])
return ui_models
......@@ -311,6 +312,10 @@ class OpenAssessmentBlock(
load('static/xml/unicode.xml')
),
(
"OpenAssessmentBlock Example Based Rubric",
load('static/xml/example_based_example.xml')
),
(
"OpenAssessmentBlock Poverty Rubric",
load('static/xml/poverty_rubric_example.xml')
),
......@@ -480,6 +485,51 @@ class OpenAssessmentBlock(
else:
return False, None, open_range[0], open_range[1]
def get_waiting_details(self, status_details):
"""
Returns the specific waiting status based on the given status_details.
This status can currently be peer, example-based, or both. This is
determined by checking that status details to see if all assessment
modules have been graded.
Args:
status_details (dict): A dictionary containing the details of each
assessment module status. This will contain keys such as
"peer" and "ai", referring to dictionaries, which in turn will
have the key "graded". If this key has a value set, these
assessment modules have been graded.
Returns:
A string of "peer", "exampled-based", or "all" to indicate which
assessment modules in the workflow are waiting on assessments.
Returns None if no module is waiting on an assessment.
Examples:
>>> now = dt.datetime.utcnow().replace(tzinfo=pytz.utc)
>>> status_details = {
>>> 'peer': {
>>> 'completed': None,
>>> 'graded': now
>>> },
>>> 'ai': {
>>> 'completed': now,
>>> 'graded': None
>>> }
>>> }
>>> self.get_waiting_details(status_details)
"peer"
"""
waiting = None
peer_waiting = "peer" in status_details and not status_details["peer"]["graded"]
ai_waiting = "ai" in status_details and not status_details["ai"]["graded"]
if peer_waiting and ai_waiting:
waiting = "all"
elif peer_waiting:
waiting = "peer"
elif ai_waiting:
waiting = "example-based"
return waiting
def is_released(self, step=None):
"""
Check if a question has been released.
......
<openassessment>
<title>Example Based Example</title>
<assessments>
<assessment name="example-based-assessment" algorithm_id="fake">
<example>
<answer>Born in northern New South Wales, Dowling entered the Royal Australian Naval College in 1915. After graduating in 1919 he went to sea aboard various Royal Navy and RAN vessels, and later specialised in gunnery. In 1937, he was given command of the sloop HMAS Swan. Following the outbreak of World War II, he saw action in the Mediterranean theatre as executive officer of the Royal Navy cruiser HMS Naiad, and survived her sinking by a German U-boat in March 1942. Returning to Australia, he served as Director of Plans and later Deputy Chief of Naval Staff before taking command of the light cruiser HMAS Hobart in November 1944. His achievements in the South West Pacific earned him the Distinguished Service Order.
Dowling took command of the RAN's first aircraft carrier, HMAS Sydney, in 1948. He became Chief of Naval Personnel in 1950, and Flag Officer Commanding HM Australian Fleet in 1953. Soon after taking up the position of CNS in February 1955, he was promoted to vice admiral and appointed a Companion of the Order of the Bath. As CNS he had to deal with shortages of money, manpower and equipment, and with the increasing role of the United States in Australia's defence planning, at the expense of traditional ties with Britain. Knighted in 1957, Dowling was Chairman of COSC from March 1959 until May 1961, when he retired from the military. In 1963 he was appointed a Knight Commander of the Royal Victorian Order and became Australian Secretary to HM Queen Elizabeth II, serving until his death in 1969.
</answer>
<select criterion="Ideas" option="Bad" />
<select criterion="Content" option="Bad" />
</example>
<example>
<answer>Roy Russell Dowling was born on 28 May 1901 in Condong, a township on the Tweed River in northern New South Wales. His parents were sugar cane inspector Russell Dowling and his wife Lily. The youth entered the Royal Australian Naval College (RANC) at Jervis Bay, Federal Capital Territory, in 1915. An underachiever academically, he excelled at sports, and became chief cadet captain before graduating in 1918 with the King's Medal, awarded for "gentlemanly bearing, character, good influence among his fellows and officer-like qualities".[1][2] The following year he was posted to Britain as a midshipman, undergoing training with the Royal Navy and seeing service on HMS Ramillies and HMS Venturous.[3] By January 1923 he was back in Australia, serving aboard the cruiser HMAS Adelaide. He was promoted to lieutenant in March.[4] In April 1924, Adelaide joined the Royal Navy's Special Service Squadron on its worldwide cruise, taking in New Zealand, Canada, the United States, Panama, and the West Indies, before docking in September at Portsmouth, England. There Dowling left the ship for his next appointment, training as a gunnery officer and serving in that capacity at HMS Excellent.
</answer>
<select criterion="Ideas" option="Good" />
<select criterion="Content" option="Bad" />
</example>
<example>
<answer>After his return to Australia in December 1926, Dowling spent eighteen months on HMAS Platypus and HMAS Anzac, where he continued to specialise in gunnery. In July 1928, he took on an instructional role at the gunnery school in Flinders Naval Depot on Western Port Bay, Victoria. He married Jessie Blanch in Melbourne on 8 May 1930; the couple had two sons and three daughters.[1][6] Jessie accompanied him on his next posting to Britain commencing in January 1931.</answer>
<select criterion="Ideas" option="Bad" />
<select criterion="Content" option="Good" />
</example>
<example>
<answer>He was promoted to lieutenant commander on 15 March, and was appointed gunnery officer on the light cruiser HMS Colombo in May. Dowling returned to Australia in January 1933, and was appointed squadron gunnery officer aboard the heavy cruiser HMAS Canberra that April.[1][4] The ship operated mainly within Australian waters over the next two years.[7] In July 1935, Dowling took charge of the gunnery school at Flinders Naval Depot. He was promoted to commander on 31 December 1936.[1][4] The following month, he assumed command of the newly commissioned Grimsby-class sloop HMAS Swan, carrying out duties in the South West Pacific.[8] Completing his tenure on Swan in January 1939, he was briefly assigned to the Navy Office, Melbourne, before returning to Britain in March for duty at HMS Pembroke, where he awaited posting aboard the yet-to-be-commissioned anti-aircraft cruiser, HMS Naiad.</answer>
<select criterion="Ideas" option="Good" />
<select criterion="Content" option="Good" />
</example>
</assessment>
</assessments>
<rubric>
<prompt>
Censorship in the Libraries
'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author
Write a persuasive essay to a newspaper reflecting your views on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.
Read for conciseness, clarity of thought, and form.
</prompt>
<criterion feedback="optional">
<name>Ideas</name>
<prompt>Determine if there is a unifying theme or main idea.</prompt>
<option points="0">
<name>Bad</name>
<explanation>Difficult for the reader to discern the main idea. Too brief or too repetitive to establish or maintain a focus.</explanation>
</option>
<option points="3">
<name>Good</name>
<explanation>Presents a unifying theme or main idea, but may include minor tangents. Stays somewhat focused on topic and task.</explanation>
</option>
</criterion>
<criterion>
<name>Content</name>
<prompt>Assess the content of the submission</prompt>
<option points="0">
<name>Bad</name>
<explanation>Includes little information with few or no details or unrelated details. Unsuccessful in attempts to explore any facets of the topic.</explanation>
</option>
<option points="1">
<name>Good</name>
<explanation>Includes little information and few or no details. Explores only one or two facets of the topic.</explanation>
</option>
</criterion>
<feedbackprompt>
(Optional) What aspects of this response stood out to you? What did it do well? How could it improve?
</feedbackprompt>
</rubric>
</openassessment>
\ No newline at end of file
......@@ -37,8 +37,29 @@
</criterion>
</rubric>
<assessments>
<assessment name="example-based-assessment" algorithm_id="fake">
<example>
<answer>Example Answer One</answer>
<select criterion="Ideas" option="Poor" />
<select criterion="Content" option="Poor" />
</example>
<example>
<answer>Example Answer Two</answer>
<select criterion="Ideas" option="Fair" />
<select criterion="Content" option="Fair" />
</example>
<example>
<answer>Example Answer Three</answer>
<select criterion="Ideas" option="Fair" />
<select criterion="Content" option="Good" />
</example>
<example>
<answer>Example Answer Four</answer>
<select criterion="Ideas" option="Poor" />
<select criterion="Content" option="Good" />
</example>
</assessment>
<assessment name="peer-assessment" must_grade="5" must_be_graded_by="3" />
<assessment name="self-assessment" />
<!-- TODO: for now we're inserting the example-based assessment programmatically, until the XML format changes land -->
</assessments>
</openassessment>
......@@ -40,6 +40,28 @@
</criterion>
</rubric>
<assessments>
<assessment name="example-based-assessment" algorithm_id="fake">
<example>
<answer>Example Answer One</answer>
<select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="Ġööḋ" />
<select criterion="Form" option="Poor" />
</example>
<example>
<answer>Example Answer Two</answer>
<select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="ﻉซƈﻉɭɭﻉกՇ" />
<select criterion="Form" option="Fair" />
</example>
<example>
<answer>Example Answer Three</answer>
<select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="Ġööḋ" />
<select criterion="Form" option="Good" />
</example>
<example>
<answer>Example Answer Four</answer>
<select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="ﻉซƈﻉɭɭﻉกՇ" />
<select criterion="Form" option="Good" />
</example>
</assessment>
<assessment name="peer-assessment" must_grade="2" must_be_graded_by="2" />
<assessment name="self-assessment" />
</assessments>
......
{
"waiting_for_peer": {
"waiting_for_peer": true,
"waiting_for_ai": false,
"expected_response": "peer assessment"
},
"waiting_for_ai": {
"waiting_for_peer": false,
"waiting_for_ai": true,
"expected_response": "example based assessment"
},
"waiting_for_both": {
"waiting_for_peer": true,
"waiting_for_ai": true,
"expected_response": "peer assessment and example based assessment"
},
"not_waiting": {
"waiting_for_peer": false,
"waiting_for_ai": false,
"expected_response": "your grade:"
}
}
\ No newline at end of file
......@@ -3,14 +3,25 @@
Tests for grade handlers in Open Assessment XBlock.
"""
import copy
import ddt
import json
from django.test.utils import override_settings
from submissions import api as sub_api
from openassessment.workflow import api as workflow_api
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
from .base import XBlockHandlerTestCase, scenario
# Test dependency on Stub AI Algorithm configuration
from openassessment.assessment.test.test_ai import (
ALGORITHM_ID, AI_ALGORITHMS, train_classifiers
)
CLASSIFIER_SCORE_OVERRIDES = {
u"𝓒𝓸𝓷𝓬𝓲𝓼𝓮": {'score_override': 1},
u"Form": {'score_override': 2}
}
@ddt.ddt
class TestGrade(XBlockHandlerTestCase):
"""
View-level tests for the XBlock grade handlers.
......@@ -40,8 +51,11 @@ class TestGrade(XBlockHandlerTestCase):
STEPS = ['peer', 'self']
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/grade_scenario.xml', user_id='Greggs')
def test_render_grade(self, xblock):
train_classifiers({'criteria': xblock.rubric_criteria}, CLASSIFIER_SCORE_OVERRIDES)
# Submit, assess, and render the grade view
self._create_submission_and_assessments(
xblock, self.SUBMISSION, self.PEERS, self.ASSESSMENTS, self.ASSESSMENTS[0]
......@@ -68,6 +82,7 @@ class TestGrade(XBlockHandlerTestCase):
self.assertIn('self', resp.lower())
self.assertIn('complete', resp.lower())
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/feedback_per_criterion.xml', user_id='Bernard')
def test_render_grade_feedback_per_criterion(self, xblock):
# Submit, assess, and render the grade view
......@@ -99,18 +114,22 @@ class TestGrade(XBlockHandlerTestCase):
self.assertIn(u'Peer 2: ฝﻉɭɭ ɗѻกﻉ!', resp.decode('utf-8'))
self.assertIn(u'Peer 2: ƒαιя נσв', resp.decode('utf-8'))
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@ddt.file_data('data/waiting_scenarios.json')
@scenario('data/grade_scenario.xml', user_id='Omar')
def test_grade_waiting(self, xblock):
def test_grade_waiting(self, xblock, data):
train_classifiers({'criteria': xblock.rubric_criteria}, CLASSIFIER_SCORE_OVERRIDES)
# Waiting to be assessed by a peer
self._create_submission_and_assessments(
xblock, self.SUBMISSION, self.PEERS, self.ASSESSMENTS, self.ASSESSMENTS[0],
waiting_for_peer=True
waiting_for_peer=data["waiting_for_peer"], waiting_for_ai=data["waiting_for_ai"]
)
resp = self.request(xblock, 'render_grade', json.dumps(dict()))
# Verify that we're on the waiting template
self.assertIn(u'waiting for peer assessment', resp.decode('utf-8').lower())
self.assertIn(data["expected_response"], resp.decode('utf-8').lower())
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/grade_incomplete_scenario.xml', user_id='Bunk')
def test_grade_incomplete_missing_self(self, xblock):
# Graded peers, but haven't completed self assessment
......@@ -122,6 +141,7 @@ class TestGrade(XBlockHandlerTestCase):
# Verify that we're on the right template
self.assertIn(u'not completed', resp.decode('utf-8').lower())
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/grade_incomplete_scenario.xml', user_id='Daniels')
def test_grade_incomplete_missing_peer(self, xblock):
# Have not yet completed peer assessment
......@@ -133,6 +153,7 @@ class TestGrade(XBlockHandlerTestCase):
# Verify that we're on the right template
self.assertIn(u'not completed', resp.decode('utf-8').lower())
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/grade_scenario.xml', user_id='Greggs')
def test_submit_feedback(self, xblock):
# Create submissions and assessments
......@@ -156,6 +177,7 @@ class TestGrade(XBlockHandlerTestCase):
feedback['options'], [{'text': u'Option 1'}, {'text': u'Option 2'}]
)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/grade_scenario.xml', user_id='Bob')
def test_submit_feedback_no_options(self, xblock):
# Create submissions and assessments
......@@ -176,6 +198,7 @@ class TestGrade(XBlockHandlerTestCase):
self.assertIsNot(feedback, None)
self.assertItemsEqual(feedback['options'], [])
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/grade_scenario.xml', user_id='Bob')
def test_submit_feedback_invalid_options(self, xblock):
# Create submissions and assessments
......@@ -194,7 +217,7 @@ class TestGrade(XBlockHandlerTestCase):
def _create_submission_and_assessments(
self, xblock, submission_text, peers, peer_assessments, self_assessment,
waiting_for_peer=False
waiting_for_peer=False, waiting_for_ai=False
):
"""
Create a submission and peer/self assessments, so that the user can receive a grade.
......@@ -208,6 +231,7 @@ class TestGrade(XBlockHandlerTestCase):
Kwargs:
waiting_for_peer (bool): If true, skip creation of peer assessments for the user's submission.
waiting_for_ai (bool): If True, skip creation of ai assessment.
Returns:
None
......@@ -216,6 +240,10 @@ class TestGrade(XBlockHandlerTestCase):
# Create a submission from the user
student_item = xblock.get_student_item_dict()
student_id = student_item['student_id']
example_based_assessment = xblock.get_assessment_module('example-based-assessment')
if not waiting_for_ai and example_based_assessment:
train_classifiers({'criteria': xblock.rubric_criteria}, CLASSIFIER_SCORE_OVERRIDES)
example_based_assessment['algorithm_id'] = ALGORITHM_ID
submission = xblock.create_submission(student_item, submission_text)
# Create submissions and assessments from other users
......@@ -263,4 +291,4 @@ class TestGrade(XBlockHandlerTestCase):
self_api.create_assessment(
submission['uuid'], student_id, self_assessment['options_selected'],
{'criteria': xblock.rubric_criteria}
)
)
\ No newline at end of file
......@@ -3,6 +3,7 @@
"""
Tests for message handlers in Open Assessment XBlock.
"""
import copy
import mock
import pytz
......@@ -29,9 +30,21 @@ class TestMessageRender(XBlockHandlerTestCase):
PAST = TODAY - dt.timedelta(days=10)
FAR_PAST = TODAY - dt.timedelta(days=100)
DEFAULT_STATUS_DETAILS = {
'peer': {
'completed': TODAY,
'graded': TODAY,
},
'ai': {
'completed': TODAY,
'graded': TODAY,
},
}
def _assert_path_and_context(
self, xblock, expected_path, expected_context,
workflow_status, deadline_information, has_peers_to_grade
workflow_status, deadline_information, has_peers_to_grade,
workflow_status_details=DEFAULT_STATUS_DETAILS
):
"""
Complete all of the logic behind rendering the message and verify
......@@ -49,12 +62,14 @@ class TestMessageRender(XBlockHandlerTestCase):
- deadline_information.get("self-assessment") has the same properties as is_closed("self-assessment")
- (WILL BE DEFAULT) deadline_information.get("over-all") has the same properties as is_closed()
has_peers_to_grade (bool): A boolean which indicates whether the queue of peer responses is empty
workflow_status_details (dict): A dictionary of status details
"""
# Simulate the response from the workflow API
workflow_info = {
'status': workflow_status
'status': workflow_status,
'status_details': workflow_status_details,
}
xblock.get_workflow_info = mock.Mock(return_value=workflow_info)
......@@ -692,6 +707,8 @@ class TestMessageRender(XBlockHandlerTestCase):
def test_waiting_due(self, xblock):
status = 'waiting'
status_details = copy.deepcopy(self.DEFAULT_STATUS_DETAILS)
status_details["peer"]["graded"] = None
deadline_information = {
'submission': (True, 'due', self.FAR_PAST, self.YESTERDAY),
......@@ -705,18 +722,20 @@ class TestMessageRender(XBlockHandlerTestCase):
expected_path = 'openassessmentblock/message/oa_message_complete.html'
expected_context = {
"waiting": True
"waiting": "peer"
}
self._assert_path_and_context(
xblock, expected_path, expected_context,
status, deadline_information, has_peers_to_grade
status, deadline_information, has_peers_to_grade, status_details
)
@scenario('data/message_scenario.xml', user_id = "Linda")
def test_waiting_not_due(self, xblock):
status = 'waiting'
status_details = copy.deepcopy(self.DEFAULT_STATUS_DETAILS)
status_details["peer"]["graded"] = None
deadline_information = {
'submission': (True, 'due', self.FAR_PAST, self.YESTERDAY),
......@@ -730,12 +749,67 @@ class TestMessageRender(XBlockHandlerTestCase):
expected_path = 'openassessmentblock/message/oa_message_complete.html'
expected_context = {
"waiting": True
"waiting": "peer"
}
self._assert_path_and_context(
xblock, expected_path, expected_context,
status, deadline_information, has_peers_to_grade
status, deadline_information, has_peers_to_grade, status_details
)
@scenario('data/message_scenario.xml', user_id="Linda")
def test_waiting_on_ai(self, xblock):
status = 'waiting'
status_details = copy.deepcopy(self.DEFAULT_STATUS_DETAILS)
status_details["ai"]["graded"] = None
deadline_information = {
'submission': (True, 'due', self.FAR_PAST, self.YESTERDAY),
'peer-assessment': (True, 'due', self.YESTERDAY, self.TODAY),
'self-assessment': (True, 'due', self.YESTERDAY, self.TODAY),
'over-all': (True, 'due', self.FAR_PAST, self.TODAY)
}
has_peers_to_grade = False
expected_path = 'openassessmentblock/message/oa_message_complete.html'
expected_context = {
"waiting": "example-based"
}
self._assert_path_and_context(
xblock, expected_path, expected_context,
status, deadline_information, has_peers_to_grade, status_details
)
@scenario('data/message_scenario.xml', user_id="Linda")
def test_waiting_on_all(self, xblock):
status = 'waiting'
status_details = copy.deepcopy(self.DEFAULT_STATUS_DETAILS)
status_details["ai"]["graded"] = None
status_details["peer"]["graded"] = None
deadline_information = {
'submission': (True, 'due', self.FAR_PAST, self.YESTERDAY),
'peer-assessment': (True, 'due', self.YESTERDAY, self.TODAY),
'self-assessment': (True, 'due', self.YESTERDAY, self.TODAY),
'over-all': (True, 'due', self.FAR_PAST, self.TODAY)
}
has_peers_to_grade = False
expected_path = 'openassessmentblock/message/oa_message_complete.html'
expected_context = {
"waiting": "all"
}
self._assert_path_and_context(
xblock, expected_path, expected_context,
status, deadline_information, has_peers_to_grade, status_details
)
@scenario('data/message_scenario.xml', user_id = "Linda")
......@@ -755,7 +829,7 @@ class TestMessageRender(XBlockHandlerTestCase):
expected_path = 'openassessmentblock/message/oa_message_complete.html'
expected_context = {
"waiting": False
"waiting": None
}
self._assert_path_and_context(
......@@ -780,10 +854,11 @@ class TestMessageRender(XBlockHandlerTestCase):
expected_path = 'openassessmentblock/message/oa_message_complete.html'
expected_context = {
"waiting": False
"waiting": None
}
self._assert_path_and_context(
xblock, expected_path, expected_context,
status, deadline_information, has_peers_to_grade
)
\ No newline at end of file
)
......@@ -3,6 +3,7 @@ from collections import namedtuple
import pytz
import json
from mock import Mock, patch
from django.test.utils import override_settings
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
from openassessment.assessment.api import ai as ai_api
......@@ -10,6 +11,10 @@ from openassessment.workflow import api as workflow_api
from openassessment.assessment.errors.ai import AIError
from submissions import api as sub_api
from openassessment.xblock.test.base import scenario, XBlockHandlerTestCase
# Test dependency on Stub AI Algorithm configuration
from openassessment.assessment.test.test_ai import (
ALGORITHM_ID, AI_ALGORITHMS, train_classifiers
)
STUDENT_ITEM = dict(
student_id="Bob",
......@@ -60,6 +65,13 @@ EXAMPLE_BASED_ASSESSMENT = {
]
}
# Rubric-specific classifier score override
CLASSIFIER_SCORE_OVERRIDES = {
u"Ideas": {'score_override': 1},
u"Content": {'score_override': 2}
}
class TestCourseStaff(XBlockHandlerTestCase):
"""
Tests for course staff debug panel.
......@@ -298,8 +310,12 @@ class TestCourseStaff(XBlockHandlerTestCase):
self.assertEquals('openassessmentblock/staff_debug/staff_debug.html', path)
self.assertTrue(context['display_schedule_training'])
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@scenario('data/example_based_assessment.xml', user_id='Bob')
def test_schedule_training(self, xblock):
example_based_assessment = xblock.get_assessment_module('example-based-assessment')
example_based_assessment['algorithm_id'] = ALGORITHM_ID
train_classifiers({'criteria': xblock.rubric_criteria}, CLASSIFIER_SCORE_OVERRIDES)
xblock.rubric_assessments.append(EXAMPLE_BASED_ASSESSMENT)
xblock.xmodule_runtime = self._create_mock_runtime(
xblock.scope_ids.usage_id, True, True, "Bob"
......@@ -339,7 +355,7 @@ class TestCourseStaff(XBlockHandlerTestCase):
self.assertFalse(response['success'])
self.assertTrue('error' in response['msg'])
@scenario('data/example_based_assessment.xml', user_id='Bob')
@scenario('data/peer_only_scenario.xml', user_id='Bob')
def test_no_example_based_assessment(self, xblock):
xblock.xmodule_runtime = self._create_mock_runtime(
xblock.scope_ids.usage_id, True, True, "Bob"
......
......@@ -14,6 +14,7 @@ class WorkflowMixin(object):
# Dictionary mapping assessment names (e.g. peer-assessment)
# to the corresponding workflow step names.
ASSESSMENT_STEP_NAMES = {
"example-based-assessment": "ai",
"self-assessment": "self",
"peer-assessment": "peer",
"student-training": "training",
......@@ -49,7 +50,12 @@ class WorkflowMixin(object):
"""
steps = self._create_step_list()
workflow_api.create_workflow(submission_uuid, steps)
rubric_dict = {
'criteria': self.rubric_criteria
}
ai_module = self.get_assessment_module('example-based-assessment')
algorithm_id = ai_module["algorithm_id"] if ai_module else None
workflow_api.create_workflow(submission_uuid, steps, rubric_dict, algorithm_id)
def workflow_requirements(self):
"""
......
......@@ -48,7 +48,7 @@ EDX_ORA2["EVENT_LOGGER"] = "openassessment.workflow.test.events.fake_event_logge
# We run Celery in "always eager" mode in the test suite,
# which executes tasks synchronously instead of using the task queue.
CELERY_ALWAYS_EAGER = True
CELERY_EAGER_PROPAGATES_EXCEPTIONS=True
# Silence cache key warnings
# https://docs.djangoproject.com/en/1.4/topics/cache/#cache-key-warnings
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment