Commit 2a75972c by Will Daly

Merge pull request #423 from edx/will/more-workflow-refactoring

More workflow refactoring
parents 52b4bf78 6567fadd
...@@ -31,6 +31,10 @@ def submitter_is_finished(submission_uuid, requirements): ...@@ -31,6 +31,10 @@ def submitter_is_finished(submission_uuid, requirements):
""" """
Check whether the submitter has made the required number of assessments. Check whether the submitter has made the required number of assessments.
If the requirements dict is None (because we're being updated
asynchronously or when the workflow is first created),
then automatically return False.
Args: Args:
submission_uuid (str): The UUID of the submission being tracked. submission_uuid (str): The UUID of the submission being tracked.
requirements (dict): Dictionary with the key "must_grade" indicating requirements (dict): Dictionary with the key "must_grade" indicating
...@@ -40,6 +44,9 @@ def submitter_is_finished(submission_uuid, requirements): ...@@ -40,6 +44,9 @@ def submitter_is_finished(submission_uuid, requirements):
bool bool
""" """
if requirements is None:
return False
try: try:
workflow = PeerWorkflow.objects.get(submission_uuid=submission_uuid) workflow = PeerWorkflow.objects.get(submission_uuid=submission_uuid)
if workflow.completed_at is not None: if workflow.completed_at is not None:
...@@ -58,6 +65,10 @@ def assessment_is_finished(submission_uuid, requirements): ...@@ -58,6 +65,10 @@ def assessment_is_finished(submission_uuid, requirements):
Check whether the submitter has received enough assessments Check whether the submitter has received enough assessments
to get a score. to get a score.
If the requirements dict is None (because we're being updated
asynchronously or when the workflow is first created),
then automatically return False.
Args: Args:
submission_uuid (str): The UUID of the submission being tracked. submission_uuid (str): The UUID of the submission being tracked.
requirements (dict): Dictionary with the key "must_be_graded_by" requirements (dict): Dictionary with the key "must_be_graded_by"
...@@ -68,6 +79,8 @@ def assessment_is_finished(submission_uuid, requirements): ...@@ -68,6 +79,8 @@ def assessment_is_finished(submission_uuid, requirements):
bool bool
""" """
if requirements is None:
return False
return bool(get_score(submission_uuid, requirements)) return bool(get_score(submission_uuid, requirements))
...@@ -126,6 +139,7 @@ def get_score(submission_uuid, requirements): ...@@ -126,6 +139,7 @@ def get_score(submission_uuid, requirements):
dict with keys "points_earned" and "points_possible". dict with keys "points_earned" and "points_possible".
""" """
# User hasn't completed their own submission yet # User hasn't completed their own submission yet
if not submitter_is_finished(submission_uuid, requirements): if not submitter_is_finished(submission_uuid, requirements):
return None return None
......
...@@ -41,6 +41,9 @@ def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W06 ...@@ -41,6 +41,9 @@ def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W06
StudentTrainingRequestError StudentTrainingRequestError
""" """
if requirements is None:
return False
try: try:
num_required = int(requirements['num_required']) num_required = int(requirements['num_required'])
except KeyError: except KeyError:
......
...@@ -1095,12 +1095,12 @@ class TestPeerApi(CacheResetTest): ...@@ -1095,12 +1095,12 @@ class TestPeerApi(CacheResetTest):
1 1
) )
@patch.object(Assessment.objects, 'filter')
@raises(peer_api.PeerAssessmentInternalError) @raises(peer_api.PeerAssessmentInternalError)
def test_max_score_db_error(self, mock_filter): def test_max_score_db_error(self):
mock_filter.side_effect = DatabaseError("Bad things happened")
tim, _ = self._create_student_and_submission("Tim", "Tim's answer") tim, _ = self._create_student_and_submission("Tim", "Tim's answer")
peer_api.get_rubric_max_scores(tim["uuid"]) with patch.object(Assessment.objects, 'filter') as mock_filter:
mock_filter.side_effect = DatabaseError("Bad things happened")
peer_api.get_rubric_max_scores(tim["uuid"])
@patch.object(PeerWorkflow.objects, 'get') @patch.object(PeerWorkflow.objects, 'get')
@raises(peer_api.PeerAssessmentInternalError) @raises(peer_api.PeerAssessmentInternalError)
......
...@@ -63,7 +63,8 @@ def create_workflow(submission_uuid, steps): ...@@ -63,7 +63,8 @@ def create_workflow(submission_uuid, steps):
) )
try: try:
submission_dict = sub_api.get_submission_and_student(submission_uuid) workflow = AssessmentWorkflow.start_workflow(submission_uuid, steps)
return AssessmentWorkflowSerializer(workflow).data
except sub_api.SubmissionNotFoundError: except sub_api.SubmissionNotFoundError:
err_msg = sub_err_msg("submission not found") err_msg = sub_err_msg("submission not found")
logger.error(err_msg) logger.error(err_msg)
...@@ -78,61 +79,17 @@ def create_workflow(submission_uuid, steps): ...@@ -78,61 +79,17 @@ def create_workflow(submission_uuid, steps):
u"retrieving submission {} failed with unknown error: {}" u"retrieving submission {} failed with unknown error: {}"
.format(submission_uuid, err) .format(submission_uuid, err)
) )
except DatabaseError:
# Raise an error if they specify a step we don't recognize... err_msg = u"Could not create assessment workflow for submission UUID: {}".format(submission_uuid)
invalid_steps = set(steps) - set(AssessmentWorkflow.STEPS) logger.exception(err_msg)
if invalid_steps: raise AssessmentWorkflowInternalError(err_msg)
raise AssessmentWorkflowRequestError( except:
u"The following steps were not recognized: {}; Must be one of {}".format( err_msg = (
invalid_steps, AssessmentWorkflow.STEPS u"An unexpected error occurred while creating "
) u"the workflow for submission UUID {}"
) ).format(submission_uuid)
# We're not using a serializer to deserialize this because the only variable
# we're getting from the outside is the submission_uuid, which is already
# validated by this point.
status = AssessmentWorkflow.STATUS.peer
if steps[0] == "peer":
try:
peer_api.on_start(submission_uuid)
except PeerAssessmentError as err:
err_msg = u"Could not create assessment workflow: {}".format(err)
logger.exception(err_msg)
raise AssessmentWorkflowInternalError(err_msg)
elif steps[0] == "self":
status = AssessmentWorkflow.STATUS.self
elif steps[0] == "training":
status = AssessmentWorkflow.STATUS.training
try:
training_api.on_start(submission_uuid)
except StudentTrainingInternalError as err:
err_msg = u"Could not create assessment workflow: {}".format(err)
logger.exception(err_msg)
raise AssessmentWorkflowInternalError(err_msg)
try:
workflow = AssessmentWorkflow.objects.create(
submission_uuid=submission_uuid,
status=status,
course_id=submission_dict['student_item']['course_id'],
item_id=submission_dict['student_item']['item_id'],
)
workflow_steps = [
AssessmentWorkflowStep(
workflow=workflow, name=step, order_num=i
)
for i, step in enumerate(steps)
]
workflow.steps.add(*workflow_steps)
except (
DatabaseError,
sub_api.SubmissionError
) as err:
err_msg = u"Could not create assessment workflow: {}".format(err)
logger.exception(err_msg) logger.exception(err_msg)
raise AssessmentWorkflowInternalError(err_msg) raise AssessmentWorkflowInternalError(err_msg)
return AssessmentWorkflowSerializer(workflow).data
def get_workflow_for_submission(submission_uuid, assessment_requirements): def get_workflow_for_submission(submission_uuid, assessment_requirements):
......
...@@ -12,7 +12,7 @@ need to then generate a matching migration for it using: ...@@ -12,7 +12,7 @@ need to then generate a matching migration for it using:
import logging import logging
import importlib import importlib
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models, transaction
from django_extensions.db.fields import UUIDField from django_extensions.db.fields import UUIDField
from django.utils.timezone import now from django.utils.timezone import now
from model_utils import Choices from model_utils import Choices
...@@ -93,6 +93,78 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel): ...@@ -93,6 +93,78 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
ordering = ["-created"] ordering = ["-created"]
# TODO: In migration, need a non-unique index on (course_id, item_id, status) # TODO: In migration, need a non-unique index on (course_id, item_id, status)
@classmethod
@transaction.commit_on_success
def start_workflow(cls, submission_uuid, step_names):
"""
Start a new workflow.
Args:
submission_uuid (str): The UUID of the submission associated with this workflow.
step_names (list): The names of the assessment steps in the workflow.
Returns:
AssessmentWorkflow
Raises:
SubmissionNotFoundError
SubmissionRequestError
SubmissionInternalError
DatabaseError
Assessment-module specific errors
"""
submission_dict = sub_api.get_submission_and_student(submission_uuid)
# Create the workflow and step models in the database
# For now, set the status to waiting; we'll modify it later
# based on the first step in the workflow.
workflow = cls.objects.create(
submission_uuid=submission_uuid,
status=AssessmentWorkflow.STATUS.waiting,
course_id=submission_dict['student_item']['course_id'],
item_id=submission_dict['student_item']['item_id']
)
workflow_steps = [
AssessmentWorkflowStep(
workflow=workflow, name=step, order_num=i
)
for i, step in enumerate(step_names)
]
workflow.steps.add(*workflow_steps)
# Initialize the assessment APIs
has_started_first_step = False
for step in workflow_steps:
api = step.api()
if api is not None:
# Initialize the assessment module
# We do this for every assessment module
on_init_func = getattr(api, 'on_init', lambda submission_uuid: None)
on_init_func(submission_uuid)
# For the first valid step, update the workflow status
# and notify the assessment module that it's being started
if not has_started_first_step:
# Update the workflow
workflow.status = step.name
workflow.save()
# Notify the assessment module that it's being started
on_start_func = getattr(api, 'on_start', lambda submission_uuid: None)
on_start_func(submission_uuid)
# Remember that we've already started the first step
has_started_first_step = True
# Update the workflow (in case some of the assessment modules are automatically complete)
# We do NOT pass in requirements, on the assumption that any assessment module
# that accepts requirements would NOT automatically complete.
workflow.update_from_assessments(None)
# Return the newly created workflow
return workflow
@property @property
def score(self): def score(self):
"""Latest score for the submission we're tracking. """Latest score for the submission we're tracking.
...@@ -132,6 +204,14 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel): ...@@ -132,6 +204,14 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
this submission, then we record the score in the submissions API and move our this submission, then we record the score in the submissions API and move our
`status` to `done`. `status` to `done`.
By convention, if `assessment_requirements` is `None`, then assessment
modules that need requirements should automatically say that they're incomplete.
This allows us to update the workflow even when we don't know the
current state of the problem. For example, if we're updating the workflow
at the completion of an asynchronous call, we won't necessarily know the
current state of the problem, but we would still want to update assessments
that don't have any requirements.
Args: Args:
assessment_requirements (dict): Dictionary passed to the assessment API. assessment_requirements (dict): Dictionary passed to the assessment API.
This defines the requirements for each assessment step; the APIs This defines the requirements for each assessment step; the APIs
...@@ -290,7 +370,7 @@ class AssessmentWorkflowStep(models.Model): ...@@ -290,7 +370,7 @@ class AssessmentWorkflowStep(models.Model):
msg = ( msg = (
u"No assessment configured for '{name}'. " u"No assessment configured for '{name}'. "
u"Check the ORA2_ASSESSMENTS Django setting." u"Check the ORA2_ASSESSMENTS Django setting."
).format(self.name) ).format(name=self.name)
logger.warning(msg) logger.warning(msg)
return None return None
...@@ -304,7 +384,10 @@ class AssessmentWorkflowStep(models.Model): ...@@ -304,7 +384,10 @@ class AssessmentWorkflowStep(models.Model):
""" """
# Once a step is completed, it will not be revisited based on updated requirements. # Once a step is completed, it will not be revisited based on updated requirements.
step_changed = False step_changed = False
step_reqs = assessment_requirements.get(self.name, {}) if assessment_requirements is None:
step_reqs = None
else:
step_reqs = assessment_requirements.get(self.name, {})
default_finished = lambda submission_uuid, step_reqs: True default_finished = lambda submission_uuid, step_reqs: True
submitter_finished = getattr(self.api(), 'submitter_is_finished', default_finished) submitter_finished = getattr(self.api(), 'submitter_is_finished', default_finished)
......
...@@ -14,7 +14,7 @@ import submissions.api as sub_api ...@@ -14,7 +14,7 @@ import submissions.api as sub_api
from openassessment.assessment.api import peer as peer_api from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api from openassessment.assessment.api import self as self_api
from openassessment.workflow.models import AssessmentWorkflow from openassessment.workflow.models import AssessmentWorkflow
from openassessment.workflow.errors import AssessmentApiLoadError from openassessment.workflow.errors import AssessmentWorkflowInternalError
ITEM_1 = { ITEM_1 = {
...@@ -264,9 +264,10 @@ class TestAssessmentWorkflowApi(CacheResetTest): ...@@ -264,9 +264,10 @@ class TestAssessmentWorkflowApi(CacheResetTest):
"item_type": "openassessment", "item_type": "openassessment",
}, "test answer") }, "test answer")
workflow_api.create_workflow(submission['uuid'], ['self']) with self.assertRaises(AssessmentWorkflowInternalError):
workflow_api.create_workflow(submission['uuid'], ['self'])
with self.assertRaises(AssessmentApiLoadError): with self.assertRaises(AssessmentWorkflowInternalError):
workflow_api.update_from_assessments(submission['uuid'], {}) workflow_api.update_from_assessments(submission['uuid'], {})
def _create_workflow_with_status( def _create_workflow_with_status(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment