Skip to content

E
edx-ora2
Commit 733dd4fe by Will Daly
Options

Student training handles changes to rubric and examples mid-flight.

parent f94334ce
Showing with 190 additions and 265 deletions
apps/openassessment/assessment/api/student_training.py
......@@ -31,18 +31,29 @@ def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W06
Args:
submission_uuid (str): The UUID of the student's submission.
requirements (dict): Not used.
requirements (dict): Must contain "num_required" indicating
the number of examples the student must assess.
Returns:
bool
Raises:
StudentTrainingRequestError
"""
try:
num_required = int(requirements['num_required'])
except KeyError:
raise StudentTrainingRequestError(u'Requirements dict must contain "num_required" key')
except ValueError:
raise StudentTrainingRequestError(u'Number of requirements must be an integer')
try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
except StudentTrainingWorkflow.DoesNotExist:
return False
else:
return workflow.is_complete
return workflow.num_completed >= num_required
def assessment_is_finished(submission_uuid, requirements): # pylint:disable=W0613
......@@ -147,8 +158,9 @@ def validate_training_examples(rubric, examples):
]
for criterion in rubric['criteria']
}
except (ValueError, KeyError):
except (ValueError, KeyError) as ex:
msg = _(u"Could not parse serialized rubric")
logger.warning("{}: {}".format(msg, ex))
return [msg]
# Check each example
......@@ -189,161 +201,31 @@ def validate_training_examples(rubric, examples):
return errors
def create_training_workflow(submission_uuid, rubric, examples):
"""
Start the training workflow.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): The serialized training examples the student will need to assess.
Returns:
None
Raises:
StudentTrainingRequestError
StudentTrainingInternalError
Example usage:
>>> options = [
>>> {
>>> "order_num": 0,
>>> "name": "poor",
>>> "explanation": "Poor job!",
>>> "points": 0,
>>> },
>>> {
>>> "order_num": 1,
>>> "name": "good",
>>> "explanation": "Good job!",
>>> "points": 1,
>>> },
>>> {
>>> "order_num": 2,
>>> "name": "excellent",
>>> "explanation": "Excellent job!",
>>> "points": 2,
>>> },
>>> ]
>>>
>>> rubric = {
>>> "prompt": "Write an essay!",
>>> "criteria": [
>>> {
>>> "order_num": 0,
>>> "name": "vocabulary",
>>> "prompt": "How varied is the vocabulary?",
>>> "options": options
>>> },
>>> {
>>> "order_num": 1,
>>> "name": "grammar",
>>> "prompt": "How correct is the grammar?",
>>> "options": options
>>> }
>>> ]
>>> }
>>>
>>> examples = [
>>> {
>>> 'answer': u'Lorem ipsum',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'excellent'
>>> }
>>> },
>>> {
>>> 'answer': u'Doler',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> create_training_workflow("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
"""
try:
# Check that examples were provided
if len(examples) == 0:
msg = (
u"No examples provided for student training workflow "
u"(attempted to create workflow for student with submission UUID {})"
).format(submission_uuid)
raise StudentTrainingRequestError(msg)
# Ensure that a workflow doesn't already exist for this submission
already_exists = StudentTrainingWorkflow.objects.filter(
submission_uuid=submission_uuid
).exists()
if already_exists:
msg = (
u"Student training workflow already exists for the student "
u"associated with submission UUID {}"
).format(submission_uuid)
raise StudentTrainingRequestError(msg)
# Create the training examples
try:
examples = deserialize_training_examples(examples, rubric)
except (InvalidRubric, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex.message)
# Create the workflow
try:
StudentTrainingWorkflow.create_workflow(submission_uuid, examples)
except sub_api.SubmissionNotFoundError as ex:
raise StudentTrainingRequestError(ex.message)
except DatabaseError:
msg = (
u"Could not create student training workflow "
u"with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
def get_workflow_status(submission_uuid):
def get_num_completed(submission_uuid):
"""
Get the student's position in the training workflow.
Get the number of training examples the student has assessed successfully.
Args:
submission_uuid (str): The UUID of the student's submission.
Returns:
dict: Serialized TrainingStatus
int: The number of completed training examples
Raises:
StudentTrainingRequestError
StudentTrainingInternalError
Example usage:
>>> get_workflow_status("5443ebbbe2297b30f503736e26be84f6c7303c57")
{
'num_items_completed': 1,
'num_items_available': 3
}
>>> get_num_completed("5443ebbbe2297b30f503736e26be84f6c7303c57")
2
"""
try:
try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
except StudentTrainingWorkflow.DoesNotExist:
msg = u"Student training workflow does not exist for submission UUID {}".format(submission_uuid)
raise StudentTrainingRequestError(msg)
num_completed, num_total = workflow.status
return {
"num_completed": num_completed,
"num_total": num_total
}
return 0
else:
return workflow.num_completed
except DatabaseError:
msg = (
u"An unexpected error occurred while "
......@@ -353,12 +235,22 @@ def get_workflow_status(submission_uuid):
raise StudentTrainingInternalError(msg)
def get_training_example(submission_uuid):
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
......@@ -380,7 +272,7 @@ def get_training_example(submission_uuid):
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57")
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': u'Lorem ipsum',
'rubric': {
......@@ -407,26 +299,38 @@ def get_training_example(submission_uuid):
}
"""
# Find a workflow for the student
try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if len(errors) > 0:
msg = _(u"Training examples do not match the rubric: {errors}").format(
errors="\n".join(errors)
)
raise StudentTrainingRequestError(msg)
# Find the next incomplete item in the workflow
item = workflow.next_incomplete_item
if item is None:
return None
else:
return serialize_training_example(item.training_example)
except StudentTrainingWorkflow.DoesNotExist:
msg = (
u"No student training workflow exists for the student "
u"associated with submission UUID {}"
).format(submission_uuid)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_or_create_workflow(submission_uuid=submission_uuid)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
item = workflow.next_incomplete_item(examples)
return None if item is None else serialize_training_example(item.training_example)
except (InvalidRubric, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex.message)
except sub_api.SubmissionNotFoundError as ex:
msg = _(u"Could not retrieve the submission with UUID {}").format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg)
except DatabaseError:
msg = (
u"Could not retrieve next item in"
u" student training workflow with submission UUID {}"
msg = _(
u"Could not retrieve a training example "
u"for the student with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
......@@ -436,6 +340,8 @@ def assess_training_example(submission_uuid, options_selected, update_workflow=T
"""
Assess a training example and update the workflow.
This must be called *after* `get_training_example()`.
Args:
submission_uuid (str): The UUID of the student's submission.
options_selected (dict): The options the student selected.
......@@ -466,8 +372,8 @@ def assess_training_example(submission_uuid, options_selected, update_workflow=T
try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
# Find the next incomplete item in the workflow
item = workflow.next_incomplete_item
# Find the item the student is currently working on
item = workflow.current_item
if item is None:
msg = (
u"No items are available in the student training workflow associated with "
......
apps/openassessment/assessment/models/student_training.py
......@@ -27,14 +27,12 @@ class StudentTrainingWorkflow(models.Model):
app_label = "assessment"
@classmethod
@transaction.commit_on_success
def create_workflow(cls, submission_uuid, examples):
def get_or_create_workflow(cls, submission_uuid):
"""
Create a student training workflow.
Args:
submission_uuid (str): The UUID of the submission from the student being trained.
examples (list of TrainingExamples): The training examples to show the student.
Returns:
StudentTrainingWorkflow
......@@ -43,27 +41,48 @@ class StudentTrainingWorkflow(models.Model):
SubmissionError: There was an error retrieving the submission.
"""
# Try to retrieve an existing workflow
# If we find one, return it immediately
try:
return cls.objects.get(submission_uuid=submission_uuid) # pylint:disable=E1101
except cls.DoesNotExist:
pass
# Retrieve the student item info
submission = sub_api.get_submission_and_student(submission_uuid)
student_item = submission['student_item']
# Create the workflow
workflow = cls.objects.create(
return cls.objects.create(
submission_uuid=submission_uuid,
student_id=student_item['student_id'],
item_id=student_item['item_id'],
course_id=student_item['course_id']
)
# Create workflow items for each example
for order_num, example in enumerate(examples):
StudentTrainingWorkflowItem.objects.create(
workflow=workflow,
order_num=order_num,
training_example=example,
)
@transaction.commit_on_success
def create_workflow_item(self, training_example):
"""
Create a workflow item for a training example
and add it to the workflow.
Args:
training_example (TrainingExample): The training example model
associated with the next workflow item.
Returns:
StudentTrainingWorkflowItem
return workflow
"""
order_num = self.items.count() + 1 # pylint:disable=E1101
item = StudentTrainingWorkflowItem.objects.create(
workflow=self,
order_num=order_num,
training_example=training_example
)
self.items.add(item) # pylint:disable=E1101
self.save()
return item
@property
def status(self):
......@@ -80,21 +99,57 @@ class StudentTrainingWorkflow(models.Model):
return num_complete, num_total
@property
def is_complete(self):
def num_completed(self):
"""
Check whether all items in the workflow are complete.
Return the number of training examples that the
student successfully assessed.
Returns:
bool
int
"""
num_incomplete = self.items.filter(completed_at__isnull=True).count() # pylint:disable=E1101
return num_incomplete == 0
return self.items.filter(completed_at__isnull=False).count() # pylint:disable=E1101
@property
def next_incomplete_item(self):
def next_incomplete_item(self, examples):
"""
Find the next incomplete item in the workflow.
Args:
examples (list of TrainingExample): Training examples to choose from.
Returns:
StudentTrainingWorkflowItem or None
"""
# If we're already working on an item, then return that item
current_item = self.current_item
if current_item is not None:
return current_item
# Otherwise, pick an item that we have not completed
# from the list of examples.
completed_examples = [
item.training_example for item in self.items.all() # pylint:disable=E1101
]
available_examples = [
available for available in examples
if available not in completed_examples
]
# If there are no more items available, return None
if len(available_examples) == 0:
return None
# Otherwise, create a new workflow item for the example
# and add it to the workflow
else:
return self.create_workflow_item(available_examples[0])
@property
def current_item(self):
"""
Return the item the student is currently working on,
or None.
Returns:
StudentTrainingWorkflowItem or None
......
apps/openassessment/assessment/test/test_student_training.py
......@@ -44,8 +44,8 @@ class StudentTrainingAssessmentTest(CacheResetTest):
},
{
"order_num": 2,
"name": "єχ¢єℓℓєηт",
"explanation": "乇ﾒc乇ﾚﾚ乇刀ｲ ﾌo乃!",
"name": u"єχ¢єℓℓєηт",
"explanation": u"乇ﾒc乇ﾚﾚ乇刀ｲ ﾌo乃!",
"points": 2,
},
]
......@@ -97,10 +97,6 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self.submission_uuid = submission['uuid']
def test_training_workflow(self):
# Start a workflow
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# Initially, we should be on the first step
self._assert_workflow_status(self.submission_uuid, 0, 2)
......@@ -141,12 +137,9 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self._assert_workflow_status(self.submission_uuid, 2, 2)
def test_assess_without_update(self):
# Start a workflow
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# Assess the first training example the same way the instructor did
# but do NOT update the workflow
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
corrections = training_api.assess_training_example(
self.submission_uuid,
self.EXAMPLES[0]['options_selected'],
......@@ -157,6 +150,15 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self.assertEqual(corrections, dict())
self._assert_workflow_status(self.submission_uuid, 0, 2)
def test_get_same_example(self):
# Retrieve a training example
retrieved = training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# If we retrieve an example without completing the current example,
# we should get the same one.
next_retrieved = training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
self.assertEqual(retrieved, next_retrieved)
@ddt.file_data('data/validate_training_examples.json')
def test_validate_training_examples(self, data):
errors = training_api.validate_training_examples(
......@@ -167,17 +169,15 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_is_finished_no_workflow(self):
# Without creating a workflow, we should not be finished
self.assertFalse(training_api.submitter_is_finished(self.submission_uuid, dict()))
requirements = {'num_required': 1}
self.assertFalse(training_api.submitter_is_finished(self.submission_uuid, requirements))
# But since we're not being assessed by others, the "assessment" should be finished.
self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, dict()))
self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, requirements))
def test_get_training_example_none_available(self):
# Start a workflow and assess all training examples
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
self._assert_workflow_status(self.submission_uuid, 0, 2)
for example in self.EXAMPLES:
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
training_api.assess_training_example(self.submission_uuid, example['options_selected'])
# Now we should be complete
......@@ -185,40 +185,13 @@ class StudentTrainingAssessmentTest(CacheResetTest):
# ... and if we try to get another example, we should get None
self.assertIs(
training_api.get_training_example(self.submission_uuid), None
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES),
None
)
def test_get_training_example_no_workflow(self):
# With no workflow defined, we should get an error
with self.assertRaises(StudentTrainingRequestError):
training_api.get_training_example(self.submission_uuid)
def test_create_training_workflow_already_started(self):
# Create a workflow for training
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# Try to create a second workflow for the same submission,
# expecting an error.
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
def test_create_training_workflow_no_examples(self):
# Try to create a training workflow with no examples
# and expect an error.
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, [])
def test_create_training_workflow_no_submission(self):
# Try to create a training workflow with an invalid submission UUID
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow("not a submission!", self.RUBRIC, self.EXAMPLES)
def test_assess_training_example_completed_workflow(self):
# Start a workflow and assess all training examples
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
self._assert_workflow_status(self.submission_uuid, 0, 2)
for example in self.EXAMPLES:
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
training_api.assess_training_example(self.submission_uuid, example['options_selected'])
# Try to assess again, and expect an error
......@@ -228,66 +201,62 @@ class StudentTrainingAssessmentTest(CacheResetTest):
)
def test_assess_training_example_no_workflow(self):
# With no workflow defined, we should get an error
# If we try to assess without first retrieving an example
# (which implicitly creates a workflow)
# then we should get a request error.
with self.assertRaises(StudentTrainingRequestError):
training_api.assess_training_example(
self.submission_uuid, self.EXAMPLES[0]['options_selected']
)
def test_get_workflow_status_no_workflow(self):
# With no workflow defined, we should get an error
# when we try to request the status.
with self.assertRaises(StudentTrainingRequestError):
training_api.get_workflow_status(self.submission_uuid)
def test_get_num_completed_no_workflow(self):
num_completed = training_api.get_num_completed(self.submission_uuid)
self.assertEqual(num_completed, 0)
def test_create_workflow_invalid_rubric(self):
def test_get_training_example_invalid_rubric(self):
# Rubric is missing a very important key!
invalid_rubric = copy.deepcopy(self.RUBRIC)
del invalid_rubric['criteria']
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, invalid_rubric, self.EXAMPLES)
training_api.get_training_example(self.submission_uuid, invalid_rubric, self.EXAMPLES)
def test_create_workflow_invalid_examples(self):
# Training example is not a dictionary!
def test_get_training_example_no_submission(self):
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, ["not a dict!"])
@patch.object(StudentTrainingWorkflow, 'create_workflow')
def test_create_workflow_database_error(self, mock_db):
mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
training_api.get_training_example("no_such_submission", self.RUBRIC, self.EXAMPLES)
@patch.object(StudentTrainingWorkflow.objects, 'get')
def test_get_workflow_status_database_error(self, mock_db):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
def test_get_num_completed_database_error(self, mock_db):
mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError):
training_api.get_workflow_status(self.submission_uuid)
training_api.get_num_completed(self.submission_uuid)
@patch.object(StudentTrainingWorkflow.objects, 'get')
def test_get_training_example_database_error(self, mock_db):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError):
training_api.get_training_example(self.submission_uuid)
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
@patch.object(StudentTrainingWorkflow.objects, 'get')
def test_assess_training_example_database_error(self, mock_db):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError):
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected'])
def _assert_workflow_status(self, submission_uuid, num_completed, num_total):
@ddt.data({}, {'num_required': 'not an integer!'})
def test_submitter_is_finished_invalid_requirements(self, requirements):
with self.assertRaises(StudentTrainingRequestError):
training_api.submitter_is_finished(self.submission_uuid, requirements)
def _assert_workflow_status(self, submission_uuid, num_completed, num_required):
"""
Check that the training workflow is on the expected step.
Args:
submission_uuid (str): Submission UUID of the student being trained.
num_completed (int): The expected number of examples assessed correctly.
num_total (int): The expected number of available examples.
num_total (int): The required number of examples to assess.
Returns:
None
......@@ -296,27 +265,22 @@ class StudentTrainingAssessmentTest(CacheResetTest):
AssertionError
"""
# Check the workflow status (what step are we on?)
status = training_api.get_workflow_status(submission_uuid)
self.assertEqual(status['num_completed'], num_completed)
self.assertEqual(status['num_total'], num_total)
# Check the number of steps we've completed
actual_num_completed = training_api.get_num_completed(submission_uuid)
self.assertEqual(actual_num_completed, num_completed)
# Check whether the assessment step is completed
# (used by the workflow API)
is_finished = bool(num_completed == num_total)
self.assertEqual(
training_api.submitter_is_finished(submission_uuid, dict()),
is_finished
)
requirements = {'num_required': num_required}
is_finished = training_api.submitter_is_finished(submission_uuid, requirements)
self.assertEqual(is_finished, bool(num_completed >= num_required))
# Assessment is finished should always be true,
# since we're not being assessed by others.
self.assertTrue(
training_api.assessment_is_finished(submission_uuid, dict()),
)
self.assertTrue(training_api.assessment_is_finished(submission_uuid, requirements))
# At no point should we receive a score!
self.assertIs(training_api.get_score(submission_uuid, dict()), None)
self.assertIs(training_api.get_score(submission_uuid, requirements), None)
def _expected_example(self, input_example, rubric):
"""
......@@ -352,6 +316,6 @@ class StudentTrainingAssessmentTest(CacheResetTest):
AssertionError
"""
example = training_api.get_training_example(submission_uuid)
example = training_api.get_training_example(submission_uuid, input_rubric, input_examples)
expected_example = self._expected_example(input_examples[order_num], input_rubric)
self.assertItemsEqual(example, expected_example)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment