Skip to content

E
edx-ora2
Commit 733dd4fe by Will Daly
Options

Student training handles changes to rubric and examples mid-flight.

parent f94334ce
Showing with 190 additions and 265 deletions
apps/openassessment/assessment/api/student_training.py
...@@ -31,18 +31,29 @@ def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W06 ...@@ -31,18 +31,29 @@ def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W06
Args: Args:
submission_uuid (str): The UUID of the student's submission. submission_uuid (str): The UUID of the student's submission.
requirements (dict): Not used. requirements (dict): Must contain "num_required" indicating
the number of examples the student must assess.
Returns: Returns:
bool bool
Raises:
StudentTrainingRequestError
""" """
try: try:
num_required = int(requirements['num_required'])
except KeyError:
raise StudentTrainingRequestError(u'Requirements dict must contain "num_required" key')
except ValueError:
raise StudentTrainingRequestError(u'Number of requirements must be an integer')
try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid) workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
except StudentTrainingWorkflow.DoesNotExist: except StudentTrainingWorkflow.DoesNotExist:
return False return False
else: else:
return workflow.is_complete return workflow.num_completed >= num_required
def assessment_is_finished(submission_uuid, requirements): # pylint:disable=W0613 def assessment_is_finished(submission_uuid, requirements): # pylint:disable=W0613
...@@ -147,8 +158,9 @@ def validate_training_examples(rubric, examples): ...@@ -147,8 +158,9 @@ def validate_training_examples(rubric, examples):
] ]
for criterion in rubric['criteria'] for criterion in rubric['criteria']
} }
except (ValueError, KeyError): except (ValueError, KeyError) as ex:
msg = _(u"Could not parse serialized rubric") msg = _(u"Could not parse serialized rubric")
logger.warning("{}: {}".format(msg, ex))
return [msg] return [msg]
# Check each example # Check each example
...@@ -189,161 +201,31 @@ def validate_training_examples(rubric, examples): ...@@ -189,161 +201,31 @@ def validate_training_examples(rubric, examples):
return errors return errors
def create_training_workflow(submission_uuid, rubric, examples): def get_num_completed(submission_uuid):
"""
Start the training workflow.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): The serialized training examples the student will need to assess.
Returns:
None
Raises:
StudentTrainingRequestError
StudentTrainingInternalError
Example usage:
>>> options = [
>>> {
>>> "order_num": 0,
>>> "name": "poor",
>>> "explanation": "Poor job!",
>>> "points": 0,
>>> },
>>> {
>>> "order_num": 1,
>>> "name": "good",
>>> "explanation": "Good job!",
>>> "points": 1,
>>> },
>>> {
>>> "order_num": 2,
>>> "name": "excellent",
>>> "explanation": "Excellent job!",
>>> "points": 2,
>>> },
>>> ]
>>>
>>> rubric = {
>>> "prompt": "Write an essay!",
>>> "criteria": [
>>> {
>>> "order_num": 0,
>>> "name": "vocabulary",
>>> "prompt": "How varied is the vocabulary?",
>>> "options": options
>>> },
>>> {
>>> "order_num": 1,
>>> "name": "grammar",
>>> "prompt": "How correct is the grammar?",
>>> "options": options
>>> }
>>> ]
>>> }
>>>
>>> examples = [
>>> {
>>> 'answer': u'Lorem ipsum',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'excellent'
>>> }
>>> },
>>> {
>>> 'answer': u'Doler',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> create_training_workflow("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
"""
try:
# Check that examples were provided
if len(examples) == 0:
msg = (
u"No examples provided for student training workflow "
u"(attempted to create workflow for student with submission UUID {})"
).format(submission_uuid)
raise StudentTrainingRequestError(msg)
# Ensure that a workflow doesn't already exist for this submission
already_exists = StudentTrainingWorkflow.objects.filter(
submission_uuid=submission_uuid
).exists()
if already_exists:
msg = (
u"Student training workflow already exists for the student "
u"associated with submission UUID {}"
).format(submission_uuid)
raise StudentTrainingRequestError(msg)
# Create the training examples
try:
examples = deserialize_training_examples(examples, rubric)
except (InvalidRubric, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex.message)
# Create the workflow
try:
StudentTrainingWorkflow.create_workflow(submission_uuid, examples)
except sub_api.SubmissionNotFoundError as ex:
raise StudentTrainingRequestError(ex.message)
except DatabaseError:
msg = (
u"Could not create student training workflow "
u"with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
def get_workflow_status(submission_uuid):
""" """
Get the student's position in the training workflow. Get the number of training examples the student has assessed successfully.
Args: Args:
submission_uuid (str): The UUID of the student's submission. submission_uuid (str): The UUID of the student's submission.
Returns: Returns:
dict: Serialized TrainingStatus int: The number of completed training examples
Raises: Raises:
StudentTrainingRequestError
StudentTrainingInternalError StudentTrainingInternalError
Example usage: Example usage:
>>> get_workflow_status("5443ebbbe2297b30f503736e26be84f6c7303c57") >>> get_num_completed("5443ebbbe2297b30f503736e26be84f6c7303c57")
{ 2
'num_items_completed': 1,
'num_items_available': 3
}
""" """
try: try:
try: try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid) workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
except StudentTrainingWorkflow.DoesNotExist: except StudentTrainingWorkflow.DoesNotExist:
msg = u"Student training workflow does not exist for submission UUID {}".format(submission_uuid) return 0
raise StudentTrainingRequestError(msg) else:
return workflow.num_completed
num_completed, num_total = workflow.status
return {
"num_completed": num_completed,
"num_total": num_total
}
except DatabaseError: except DatabaseError:
msg = ( msg = (
u"An unexpected error occurred while " u"An unexpected error occurred while "
...@@ -353,12 +235,22 @@ def get_workflow_status(submission_uuid): ...@@ -353,12 +235,22 @@ def get_workflow_status(submission_uuid):
raise StudentTrainingInternalError(msg) raise StudentTrainingInternalError(msg)
def get_training_example(submission_uuid): def get_training_example(submission_uuid, rubric, examples):
""" """
Retrieve a training example for the student to assess. Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args: Args:
submission_uuid (str): The UUID of the student's submission. submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns: Returns:
dict: The training example with keys "answer", "rubric", and "options_selected". dict: The training example with keys "answer", "rubric", and "options_selected".
...@@ -380,7 +272,7 @@ def get_training_example(submission_uuid): ...@@ -380,7 +272,7 @@ def get_training_example(submission_uuid):
>>> } >>> }
>>> ] >>> ]
>>> >>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57") >>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{ {
'answer': u'Lorem ipsum', 'answer': u'Lorem ipsum',
'rubric': { 'rubric': {
...@@ -407,26 +299,38 @@ def get_training_example(submission_uuid): ...@@ -407,26 +299,38 @@ def get_training_example(submission_uuid):
} }
""" """
# Find a workflow for the student
try: try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid) # Validate the training examples
errors = validate_training_examples(rubric, examples)
if len(errors) > 0:
msg = _(u"Training examples do not match the rubric: {errors}").format(
errors="\n".join(errors)
)
raise StudentTrainingRequestError(msg)
# Find the next incomplete item in the workflow # Get or create the workflow
item = workflow.next_incomplete_item workflow = StudentTrainingWorkflow.get_or_create_workflow(submission_uuid=submission_uuid)
if item is None:
return None # Get or create the training examples
else: examples = deserialize_training_examples(examples, rubric)
return serialize_training_example(item.training_example)
except StudentTrainingWorkflow.DoesNotExist: # Pick a training example that the student has not yet completed
msg = ( # If the student already started a training example, then return that instead.
u"No student training workflow exists for the student " item = workflow.next_incomplete_item(examples)
u"associated with submission UUID {}" return None if item is None else serialize_training_example(item.training_example)
).format(submission_uuid) except (InvalidRubric, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex.message)
except sub_api.SubmissionNotFoundError as ex:
msg = _(u"Could not retrieve the submission with UUID {}").format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg) raise StudentTrainingRequestError(msg)
except DatabaseError: except DatabaseError:
msg = ( msg = _(
u"Could not retrieve next item in" u"Could not retrieve a training example "
u" student training workflow with submission UUID {}" u"for the student with submission UUID {}"
).format(submission_uuid) ).format(submission_uuid)
logger.exception(msg) logger.exception(msg)
raise StudentTrainingInternalError(msg) raise StudentTrainingInternalError(msg)
...@@ -436,6 +340,8 @@ def assess_training_example(submission_uuid, options_selected, update_workflow=T ...@@ -436,6 +340,8 @@ def assess_training_example(submission_uuid, options_selected, update_workflow=T
""" """
Assess a training example and update the workflow. Assess a training example and update the workflow.
This must be called *after* `get_training_example()`.
Args: Args:
submission_uuid (str): The UUID of the student's submission. submission_uuid (str): The UUID of the student's submission.
options_selected (dict): The options the student selected. options_selected (dict): The options the student selected.
...@@ -466,8 +372,8 @@ def assess_training_example(submission_uuid, options_selected, update_workflow=T ...@@ -466,8 +372,8 @@ def assess_training_example(submission_uuid, options_selected, update_workflow=T
try: try:
workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid) workflow = StudentTrainingWorkflow.objects.get(submission_uuid=submission_uuid)
# Find the next incomplete item in the workflow # Find the item the student is currently working on
item = workflow.next_incomplete_item item = workflow.current_item
if item is None: if item is None:
msg = ( msg = (
u"No items are available in the student training workflow associated with " u"No items are available in the student training workflow associated with "
......
apps/openassessment/assessment/models/student_training.py
...@@ -27,14 +27,12 @@ class StudentTrainingWorkflow(models.Model): ...@@ -27,14 +27,12 @@ class StudentTrainingWorkflow(models.Model):
app_label = "assessment" app_label = "assessment"
@classmethod @classmethod
@transaction.commit_on_success def get_or_create_workflow(cls, submission_uuid):
def create_workflow(cls, submission_uuid, examples):
""" """
Create a student training workflow. Create a student training workflow.
Args: Args:
submission_uuid (str): The UUID of the submission from the student being trained. submission_uuid (str): The UUID of the submission from the student being trained.
examples (list of TrainingExamples): The training examples to show the student.
Returns: Returns:
StudentTrainingWorkflow StudentTrainingWorkflow
...@@ -43,27 +41,48 @@ class StudentTrainingWorkflow(models.Model): ...@@ -43,27 +41,48 @@ class StudentTrainingWorkflow(models.Model):
SubmissionError: There was an error retrieving the submission. SubmissionError: There was an error retrieving the submission.
""" """
# Try to retrieve an existing workflow
# If we find one, return it immediately
try:
return cls.objects.get(submission_uuid=submission_uuid) # pylint:disable=E1101
except cls.DoesNotExist:
pass
# Retrieve the student item info # Retrieve the student item info
submission = sub_api.get_submission_and_student(submission_uuid) submission = sub_api.get_submission_and_student(submission_uuid)
student_item = submission['student_item'] student_item = submission['student_item']
# Create the workflow # Create the workflow
workflow = cls.objects.create( return cls.objects.create(
submission_uuid=submission_uuid, submission_uuid=submission_uuid,
student_id=student_item['student_id'], student_id=student_item['student_id'],
item_id=student_item['item_id'], item_id=student_item['item_id'],
course_id=student_item['course_id'] course_id=student_item['course_id']
) )
# Create workflow items for each example @transaction.commit_on_success
for order_num, example in enumerate(examples): def create_workflow_item(self, training_example):
StudentTrainingWorkflowItem.objects.create( """
workflow=workflow, Create a workflow item for a training example
order_num=order_num, and add it to the workflow.
training_example=example,
) Args:
training_example (TrainingExample): The training example model
associated with the next workflow item.
Returns:
StudentTrainingWorkflowItem
return workflow """
order_num = self.items.count() + 1 # pylint:disable=E1101
item = StudentTrainingWorkflowItem.objects.create(
workflow=self,
order_num=order_num,
training_example=training_example
)
self.items.add(item) # pylint:disable=E1101
self.save()
return item
@property @property
def status(self): def status(self):
...@@ -80,21 +99,57 @@ class StudentTrainingWorkflow(models.Model): ...@@ -80,21 +99,57 @@ class StudentTrainingWorkflow(models.Model):
return num_complete, num_total return num_complete, num_total
@property @property
def is_complete(self): def num_completed(self):
""" """
Check whether all items in the workflow are complete. Return the number of training examples that the
student successfully assessed.
Returns: Returns:
bool int
""" """
num_incomplete = self.items.filter(completed_at__isnull=True).count() # pylint:disable=E1101 return self.items.filter(completed_at__isnull=False).count() # pylint:disable=E1101
return num_incomplete == 0
@property def next_incomplete_item(self, examples):
def next_incomplete_item(self):
""" """
Find the next incomplete item in the workflow. Find the next incomplete item in the workflow.
Args:
examples (list of TrainingExample): Training examples to choose from.
Returns:
StudentTrainingWorkflowItem or None
"""
# If we're already working on an item, then return that item
current_item = self.current_item
if current_item is not None:
return current_item
# Otherwise, pick an item that we have not completed
# from the list of examples.
completed_examples = [
item.training_example for item in self.items.all() # pylint:disable=E1101
]
available_examples = [
available for available in examples
if available not in completed_examples
]
# If there are no more items available, return None
if len(available_examples) == 0:
return None
# Otherwise, create a new workflow item for the example
# and add it to the workflow
else:
return self.create_workflow_item(available_examples[0])
@property
def current_item(self):
"""
Return the item the student is currently working on,
or None.
Returns: Returns:
StudentTrainingWorkflowItem or None StudentTrainingWorkflowItem or None
......
apps/openassessment/assessment/test/test_student_training.py
...@@ -44,8 +44,8 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -44,8 +44,8 @@ class StudentTrainingAssessmentTest(CacheResetTest):
}, },
{ {
"order_num": 2, "order_num": 2,
"name": "єχ¢єℓℓєηт", "name": u"єχ¢єℓℓєηт",
"explanation": "乇ﾒc乇ﾚﾚ乇刀ｲ ﾌo乃!", "explanation": u"乇ﾒc乇ﾚﾚ乇刀ｲ ﾌo乃!",
"points": 2, "points": 2,
}, },
] ]
...@@ -97,10 +97,6 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -97,10 +97,6 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self.submission_uuid = submission['uuid'] self.submission_uuid = submission['uuid']
def test_training_workflow(self): def test_training_workflow(self):
# Start a workflow
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# Initially, we should be on the first step # Initially, we should be on the first step
self._assert_workflow_status(self.submission_uuid, 0, 2) self._assert_workflow_status(self.submission_uuid, 0, 2)
...@@ -141,12 +137,9 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -141,12 +137,9 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self._assert_workflow_status(self.submission_uuid, 2, 2) self._assert_workflow_status(self.submission_uuid, 2, 2)
def test_assess_without_update(self): def test_assess_without_update(self):
# Start a workflow
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# Assess the first training example the same way the instructor did # Assess the first training example the same way the instructor did
# but do NOT update the workflow # but do NOT update the workflow
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
corrections = training_api.assess_training_example( corrections = training_api.assess_training_example(
self.submission_uuid, self.submission_uuid,
self.EXAMPLES[0]['options_selected'], self.EXAMPLES[0]['options_selected'],
...@@ -157,6 +150,15 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -157,6 +150,15 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self.assertEqual(corrections, dict()) self.assertEqual(corrections, dict())
self._assert_workflow_status(self.submission_uuid, 0, 2) self._assert_workflow_status(self.submission_uuid, 0, 2)
def test_get_same_example(self):
# Retrieve a training example
retrieved = training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# If we retrieve an example without completing the current example,
# we should get the same one.
next_retrieved = training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
self.assertEqual(retrieved, next_retrieved)
@ddt.file_data('data/validate_training_examples.json') @ddt.file_data('data/validate_training_examples.json')
def test_validate_training_examples(self, data): def test_validate_training_examples(self, data):
errors = training_api.validate_training_examples( errors = training_api.validate_training_examples(
...@@ -167,17 +169,15 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -167,17 +169,15 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_is_finished_no_workflow(self): def test_is_finished_no_workflow(self):
# Without creating a workflow, we should not be finished # Without creating a workflow, we should not be finished
self.assertFalse(training_api.submitter_is_finished(self.submission_uuid, dict())) requirements = {'num_required': 1}
self.assertFalse(training_api.submitter_is_finished(self.submission_uuid, requirements))
# But since we're not being assessed by others, the "assessment" should be finished. # But since we're not being assessed by others, the "assessment" should be finished.
self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, dict())) self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, requirements))
def test_get_training_example_none_available(self): def test_get_training_example_none_available(self):
# Start a workflow and assess all training examples
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
self._assert_workflow_status(self.submission_uuid, 0, 2)
for example in self.EXAMPLES: for example in self.EXAMPLES:
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
training_api.assess_training_example(self.submission_uuid, example['options_selected']) training_api.assess_training_example(self.submission_uuid, example['options_selected'])
# Now we should be complete # Now we should be complete
...@@ -185,40 +185,13 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -185,40 +185,13 @@ class StudentTrainingAssessmentTest(CacheResetTest):
# ... and if we try to get another example, we should get None # ... and if we try to get another example, we should get None
self.assertIs( self.assertIs(
training_api.get_training_example(self.submission_uuid), None training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES),
None
) )
def test_get_training_example_no_workflow(self):
# With no workflow defined, we should get an error
with self.assertRaises(StudentTrainingRequestError):
training_api.get_training_example(self.submission_uuid)
def test_create_training_workflow_already_started(self):
# Create a workflow for training
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
# Try to create a second workflow for the same submission,
# expecting an error.
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
def test_create_training_workflow_no_examples(self):
# Try to create a training workflow with no examples
# and expect an error.
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, [])
def test_create_training_workflow_no_submission(self):
# Try to create a training workflow with an invalid submission UUID
with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow("not a submission!", self.RUBRIC, self.EXAMPLES)
def test_assess_training_example_completed_workflow(self): def test_assess_training_example_completed_workflow(self):
# Start a workflow and assess all training examples
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
self._assert_workflow_status(self.submission_uuid, 0, 2)
for example in self.EXAMPLES: for example in self.EXAMPLES:
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
training_api.assess_training_example(self.submission_uuid, example['options_selected']) training_api.assess_training_example(self.submission_uuid, example['options_selected'])
# Try to assess again, and expect an error # Try to assess again, and expect an error
...@@ -228,66 +201,62 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -228,66 +201,62 @@ class StudentTrainingAssessmentTest(CacheResetTest):
) )
def test_assess_training_example_no_workflow(self): def test_assess_training_example_no_workflow(self):
# With no workflow defined, we should get an error # If we try to assess without first retrieving an example
# (which implicitly creates a workflow)
# then we should get a request error.
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.assess_training_example( training_api.assess_training_example(
self.submission_uuid, self.EXAMPLES[0]['options_selected'] self.submission_uuid, self.EXAMPLES[0]['options_selected']
) )
def test_get_workflow_status_no_workflow(self): def test_get_num_completed_no_workflow(self):
# With no workflow defined, we should get an error num_completed = training_api.get_num_completed(self.submission_uuid)
# when we try to request the status. self.assertEqual(num_completed, 0)
with self.assertRaises(StudentTrainingRequestError):
training_api.get_workflow_status(self.submission_uuid)
def test_create_workflow_invalid_rubric(self): def test_get_training_example_invalid_rubric(self):
# Rubric is missing a very important key! # Rubric is missing a very important key!
invalid_rubric = copy.deepcopy(self.RUBRIC) invalid_rubric = copy.deepcopy(self.RUBRIC)
del invalid_rubric['criteria'] del invalid_rubric['criteria']
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, invalid_rubric, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, invalid_rubric, self.EXAMPLES)
def test_create_workflow_invalid_examples(self): def test_get_training_example_no_submission(self):
# Training example is not a dictionary!
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, ["not a dict!"]) training_api.get_training_example("no_such_submission", self.RUBRIC, self.EXAMPLES)
@patch.object(StudentTrainingWorkflow, 'create_workflow')
def test_create_workflow_database_error(self, mock_db):
mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
@patch.object(StudentTrainingWorkflow.objects, 'get') @patch.object(StudentTrainingWorkflow.objects, 'get')
def test_get_workflow_status_database_error(self, mock_db): def test_get_num_completed_database_error(self, mock_db):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
mock_db.side_effect = DatabaseError("Kaboom!") mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError): with self.assertRaises(StudentTrainingInternalError):
training_api.get_workflow_status(self.submission_uuid) training_api.get_num_completed(self.submission_uuid)
@patch.object(StudentTrainingWorkflow.objects, 'get') @patch.object(StudentTrainingWorkflow.objects, 'get')
def test_get_training_example_database_error(self, mock_db): def test_get_training_example_database_error(self, mock_db):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
mock_db.side_effect = DatabaseError("Kaboom!") mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError): with self.assertRaises(StudentTrainingInternalError):
training_api.get_training_example(self.submission_uuid) training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
@patch.object(StudentTrainingWorkflow.objects, 'get') @patch.object(StudentTrainingWorkflow.objects, 'get')
def test_assess_training_example_database_error(self, mock_db): def test_assess_training_example_database_error(self, mock_db):
training_api.create_training_workflow(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES)
mock_db.side_effect = DatabaseError("Kaboom!") mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError): with self.assertRaises(StudentTrainingInternalError):
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected']) training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected'])
def _assert_workflow_status(self, submission_uuid, num_completed, num_total): @ddt.data({}, {'num_required': 'not an integer!'})
def test_submitter_is_finished_invalid_requirements(self, requirements):
with self.assertRaises(StudentTrainingRequestError):
training_api.submitter_is_finished(self.submission_uuid, requirements)
def _assert_workflow_status(self, submission_uuid, num_completed, num_required):
""" """
Check that the training workflow is on the expected step. Check that the training workflow is on the expected step.
Args: Args:
submission_uuid (str): Submission UUID of the student being trained. submission_uuid (str): Submission UUID of the student being trained.
num_completed (int): The expected number of examples assessed correctly. num_completed (int): The expected number of examples assessed correctly.
num_total (int): The expected number of available examples. num_total (int): The required number of examples to assess.
Returns: Returns:
None None
...@@ -296,27 +265,22 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -296,27 +265,22 @@ class StudentTrainingAssessmentTest(CacheResetTest):
AssertionError AssertionError
""" """
# Check the workflow status (what step are we on?) # Check the number of steps we've completed
status = training_api.get_workflow_status(submission_uuid) actual_num_completed = training_api.get_num_completed(submission_uuid)
self.assertEqual(status['num_completed'], num_completed) self.assertEqual(actual_num_completed, num_completed)
self.assertEqual(status['num_total'], num_total)
# Check whether the assessment step is completed # Check whether the assessment step is completed
# (used by the workflow API) # (used by the workflow API)
is_finished = bool(num_completed == num_total) requirements = {'num_required': num_required}
self.assertEqual( is_finished = training_api.submitter_is_finished(submission_uuid, requirements)
training_api.submitter_is_finished(submission_uuid, dict()), self.assertEqual(is_finished, bool(num_completed >= num_required))
is_finished
)
# Assessment is finished should always be true, # Assessment is finished should always be true,
# since we're not being assessed by others. # since we're not being assessed by others.
self.assertTrue( self.assertTrue(training_api.assessment_is_finished(submission_uuid, requirements))
training_api.assessment_is_finished(submission_uuid, dict()),
)
# At no point should we receive a score! # At no point should we receive a score!
self.assertIs(training_api.get_score(submission_uuid, dict()), None) self.assertIs(training_api.get_score(submission_uuid, requirements), None)
def _expected_example(self, input_example, rubric): def _expected_example(self, input_example, rubric):
""" """
...@@ -352,6 +316,6 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -352,6 +316,6 @@ class StudentTrainingAssessmentTest(CacheResetTest):
AssertionError AssertionError
""" """
example = training_api.get_training_example(submission_uuid) example = training_api.get_training_example(submission_uuid, input_rubric, input_examples)
expected_example = self._expected_example(input_examples[order_num], input_rubric) expected_example = self._expected_example(input_examples[order_num], input_rubric)
self.assertItemsEqual(example, expected_example) self.assertItemsEqual(example, expected_example)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment