Skip to content

E
edx-ora2
Commit 74a9d63e by Will Daly
Options

Implement AI grading task, database models, and API calls.

parent ec584292
Showing with 942 additions and 187 deletions
apps/openassessment/assessment/api/ai.py
......@@ -2,22 +2,29 @@
Public interface for AI training and grading, used by students/course authors.
"""
import logging
from django.db import DatabaseError
from submissions import api as sub_api
from openassessment.assessment.serializers import (
deserialize_training_examples, InvalidTrainingExample, InvalidRubric
deserialize_training_examples, InvalidTrainingExample, InvalidRubric,
full_assessment_dict
)
from openassessment.assessment.errors import (
AITrainingRequestError, AITrainingInternalError
AITrainingRequestError, AITrainingInternalError,
AIGradingRequestError, AIGradingInternalError
)
from openassessment.assessment.models import (
AITrainingWorkflow, InvalidOptionSelection, NoTrainingExamples
AITrainingWorkflow, InvalidOptionSelection, NoTrainingExamples,
Assessment, AITrainingWorkflow, AIGradingWorkflow,
AIClassifierSet, AI_ASSESSMENT_TYPE
)
from openassessment.assessment.worker import training as training_tasks
from openassessment.assessment.worker import grading as grading_tasks
logger = logging.getLogger(__name__)
def submit(submission_uuid, rubric):
def submit(submission_uuid, rubric, algorithm_id):
"""
Submit a response for AI assessment.
This will:
......@@ -27,6 +34,7 @@ def submit(submission_uuid, rubric):
Args:
submission_uuid (str): The UUID of the submission to assess.
rubric (dict): Serialized rubric model.
algorithm_id (unicode): Use only classifiers trained with the specified algorithm.
Returns:
grading_workflow_uuid (str): The UUID of the grading workflow.
......@@ -39,7 +47,50 @@ def submit(submission_uuid, rubric):
AIGradingInternalError
"""
pass
try:
workflow = AIGradingWorkflow.start_workflow(submission_uuid, rubric, algorithm_id)
except (sub_api.SubmissionNotFoundError, sub_api.SubmissionRequestError) as ex:
msg = (
u"An error occurred while retrieving the "
u"submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
raise AIGradingRequestError(msg)
except InvalidRubric as ex:
msg = (
u"An error occurred while parsing the serialized "
u"rubric {rubric}: {ex}"
).format(rubric=rubric, ex=ex)
raise AIGradingRequestError(msg)
except (sub_api.SubmissionInternalError, DatabaseError) as ex:
msg = (
u"An unexpected error occurred while submitting an "
u"essay for AI grading: {ex}"
).format(ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
try:
classifier_set_candidates = AIClassifierSet.objects.filter(
rubric=workflow.rubric, algorithm_id=algorithm_id
).order_by('-created_at')[:1]
# If we find classifiers for this rubric/algorithm
# then associate the classifiers with the workflow
# and schedule a grading task.
# Otherwise, the task will need to be scheduled later,
# once the classifiers have been trained.
if len(classifier_set_candidates) > 0:
workflow.classifier_set = classifier_set_candidates[0]
workflow.save()
grading_tasks.grade_essay.apply_async(args=[workflow.uuid])
return workflow.uuid
except Exception as ex:
msg = (
u"An unexpected error occurred while scheduling the "
u"AI grading task for the submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
raise AIGradingInternalError(msg)
def get_latest_assessment(submission_uuid):
......@@ -51,13 +102,29 @@ def get_latest_assessment(submission_uuid):
Returns:
dict: The serialized assessment model
or None if no assessments are available
Raises:
AIGradingRequestError
AIGradingInternalError
"""
pass
try:
assessments = Assessment.objects.filter(
submission_uuid=submission_uuid,
score_type=AI_ASSESSMENT_TYPE,
)[:1]
except DatabaseError as ex:
msg = (
u"An error occurred while retrieving AI graded assessments "
u"for the submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
if len(assessments) > 0:
return full_assessment_dict(assessments[0])
else:
return None
def train_classifiers(rubric_dict, examples, algorithm_id):
......
apps/openassessment/assessment/api/ai_worker.py
......@@ -5,63 +5,85 @@ import logging
from django.utils.timezone import now
from django.db import DatabaseError
from openassessment.assessment.models import (
AITrainingWorkflow, AIClassifierSet,
AITrainingWorkflow, AIGradingWorkflow, AIClassifierSet,
ClassifierUploadError, ClassifierSerializeError,
IncompleteClassifierSet, NoTrainingExamples
)
from openassessment.assessment.errors import (
AITrainingRequestError, AITrainingInternalError
AITrainingRequestError, AITrainingInternalError,
AIGradingRequestError, AIGradingInternalError
)
logger = logging.getLogger(__name__)
def get_submission(grading_workflow_uuid):
def get_grading_task_params(grading_workflow_uuid):
"""
Retrieve the submission associated with a particular grading workflow.
Retrieve the classifier set and algorithm ID
associated with a particular grading workflow.
Args:
grading_workflow_uuid (str): The UUID of the grading workflow.
Returns:
submission (JSON-serializable): submission from the student.
dict with keys:
* essay_text (unicode): The text of the essay submission.
* classifier_set (dict): Maps criterion names to serialized classifiers.
* algorithm_id (unicode): ID of the algorithm used to perform training.
Raises:
AIGradingRequestError
AIGradingInternalError
"""
pass
def get_classifier_set(grading_workflow_uuid):
"""
Retrieve the classifier set associated with a particular grading workflow.
Args:
grading_workflow_uuid (str): The UUID of the grading workflow.
Returns:
dict: Maps criterion names to serialized classifiers.
(binary classifiers are base-64 encoded).
try:
workflow = AIGradingWorkflow.objects.get(uuid=grading_workflow_uuid)
except AIGradingWorkflow.DoesNotExist:
msg = (
u"Could not retrieve the AI grading workflow with uuid {}"
).format(grading_workflow_uuid)
raise AIGradingRequestError(msg)
except DatabaseError as ex:
msg = (
u"An unexpected error occurred while retrieving the "
u"AI grading workflow with uuid {uuid}: {ex}"
).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
Raises:
AIGradingRequestError
AIGradingInternalError
classifier_set = workflow.classifier_set
# Tasks shouldn't be scheduled until a classifier set is
# available, so this is a serious internal error.
if classifier_set is None:
msg = (
u"AI grading workflow with UUID {} has no classifier set"
).format(grading_workflow_uuid)
logger.exception(msg)
raise AIGradingInternalError(msg)
"""
pass
try:
return {
'essay_text': workflow.essay_text,
'classifier_set': classifier_set.classifiers_dict,
'algorithm_id': workflow.algorithm_id,
}
except (ValueError, IOError, DatabaseError) as ex:
msg = (
u"An unexpected error occurred while retrieving "
u"classifiers for the grading workflow with UUID {uuid}: {ex}"
).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
def create_assessment(grading_workflow_uuid, assessment):
def create_assessment(grading_workflow_uuid, criterion_scores):
"""
Create an AI assessment (complete the AI grading task).
Args:
grading_workflow_uuid (str): The UUID of the grading workflow.
assessment (dict): The serialized assessment.
criterion_scores (dict): Dictionary mapping criteria names to integer scores.
Returns:
None
......@@ -71,57 +93,59 @@ def create_assessment(grading_workflow_uuid, assessment):
AIGradingInternalError
"""
pass
def get_algorithm_id(training_workflow_uuid):
"""
Retrieve the ID of the algorithm to use.
Args:
training_workflow_uuid (str): The UUID of the training workflow.
Returns:
unicode: The algorithm ID associated with the training task.
Raises:
AITrainingRequestError
AITrainingInternalError
"""
try:
workflow = AITrainingWorkflow.objects.get(uuid=training_workflow_uuid)
return workflow.algorithm_id
except AITrainingWorkflow.DoesNotExist:
workflow = AIGradingWorkflow.objects.get(uuid=grading_workflow_uuid)
except AIGradingWorkflow.DoesNotExist:
msg = (
u"Could not retrieve AI training workflow with UUID {}"
).format(training_workflow_uuid)
raise AITrainingRequestError(msg)
except DatabaseError:
u"Could not retrieve the AI grading workflow with uuid {}"
).format(grading_workflow_uuid)
raise AIGradingRequestError(msg)
except DatabaseError as ex:
msg = (
u"An unexpected error occurred while retrieving "
u"the algorithm ID for training workflow with UUID {}"
).format(training_workflow_uuid)
u"An unexpected error occurred while retrieving the "
u"AI grading workflow with uuid {uuid}: {ex}"
).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg)
raise AITrainingInternalError(msg)
raise AIGradingInternalError(msg)
# Optimization: if the workflow has already been marked complete
# (perhaps the task was picked up by multiple workers),
# then we don't need to do anything.
# Otherwise, create the assessment mark the workflow complete.
try:
if not workflow.is_complete:
workflow.complete(criterion_scores)
except DatabaseError as ex:
msg = (
u"An unexpected error occurred while creating the assessment "
u"for AI grading workflow with uuid {uuid}: {ex}"
).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
def get_training_examples(training_workflow_uuid):
def get_training_task_params(training_workflow_uuid):
"""
Retrieve the training examples associated with a training task.
Retrieve the training examples and algorithm ID
associated with a training task.
Args:
training_workflow_uuid (str): The UUID of the training workflow.
Returns:
list of dict: Serialized training examples, of the form:
dict with keys:
* training_examples (list of dict): The examples used to train the classifiers.
* algorithm_id (unicode): The ID of the algorithm to use for training.
Raises:
AITrainingRequestError
AITrainingInternalError
Example usage:
>>> get_training_examples('abcd1234')
>>> params = get_training_task_params('abcd1234')
>>> params['algorithm_id']
u'ease'
>>> params['training_examples']
[
{
"text": u"Example answer number one",
......@@ -161,7 +185,10 @@ def get_training_examples(training_workflow_uuid):
'scores': scores
})
return returned_examples
return {
'training_examples': returned_examples,
'algorithm_id': workflow.algorithm_id
}
except AITrainingWorkflow.DoesNotExist:
msg = (
u"Could not retrieve AI training workflow with UUID {}"
......
apps/openassessment/assessment/migrations/0012_move_algorithm_id_to_classifier_set.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'AIGradingWorkflow'
db.create_table('assessment_aigradingworkflow', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('uuid', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=36, blank=True)),
('scheduled_at', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, db_index=True)),
('completed_at', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
('submission_uuid', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
('classifier_set', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='+', null=True, to=orm['assessment.AIClassifierSet'])),
('algorithm_id', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
('rubric', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['assessment.Rubric'])),
('assessment', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='+', null=True, to=orm['assessment.Assessment'])),
('student_id', self.gf('django.db.models.fields.CharField')(max_length=40, db_index=True)),
('item_id', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
('course_id', self.gf('django.db.models.fields.CharField')(max_length=40, db_index=True)),
))
db.send_create_signal('assessment', ['AIGradingWorkflow'])
# Adding field 'AIClassifierSet.algorithm_id'
db.add_column('assessment_aiclassifierset', 'algorithm_id',
self.gf('django.db.models.fields.CharField')(default='ease', max_length=128, db_index=True),
keep_default=False)
# Deleting field 'AIClassifier.algorithm_id'
db.delete_column('assessment_aiclassifier', 'algorithm_id')
def backwards(self, orm):
# Deleting model 'AIGradingWorkflow'
db.delete_table('assessment_aigradingworkflow')
# Deleting field 'AIClassifierSet.algorithm_id'
db.delete_column('assessment_aiclassifierset', 'algorithm_id')
# Adding field 'AIClassifier.algorithm_id'
db.add_column('assessment_aiclassifier', 'algorithm_id',
self.gf('django.db.models.fields.CharField')(default='ease', max_length=128, db_index=True),
keep_default=False)
models = {
'assessment.aiclassifier': {
'Meta': {'object_name': 'AIClassifier'},
'classifier_data': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'classifiers'", 'to': "orm['assessment.AIClassifierSet']"}),
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Criterion']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.aiclassifierset': {
'Meta': {'ordering': "['-created_at']", 'object_name': 'AIClassifierSet'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"})
},
'assessment.aigradingworkflow': {
'Meta': {'object_name': 'AIGradingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.Assessment']"}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.aitrainingworkflow': {
'Meta': {'object_name': 'AITrainingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'training_examples': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'+'", 'symmetrical': 'False', 'to': "orm['assessment.TrainingExample']"}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.assessment': {
'Meta': {'ordering': "['-scored_at', '-id']", 'object_name': 'Assessment'},
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"}),
'score_type': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
'scored_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'scorer_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedback': {
'Meta': {'object_name': 'AssessmentFeedback'},
'assessments': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.Assessment']"}),
'feedback_text': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.AssessmentFeedbackOption']"}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedbackoption': {
'Meta': {'object_name': 'AssessmentFeedbackOption'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'text': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'assessment.assessmentpart': {
'Meta': {'object_name': 'AssessmentPart'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parts'", 'to': "orm['assessment.Assessment']"}),
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'option': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.CriterionOption']"})
},
'assessment.criterion': {
'Meta': {'ordering': "['rubric', 'order_num']", 'object_name': 'Criterion'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'prompt': ('django.db.models.fields.TextField', [], {'max_length': '10000'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'criteria'", 'to': "orm['assessment.Rubric']"})
},
'assessment.criterionoption': {
'Meta': {'ordering': "['criterion', 'order_num']", 'object_name': 'CriterionOption'},
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['assessment.Criterion']"}),
'explanation': ('django.db.models.fields.TextField', [], {'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'points': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'assessment.peerworkflow': {
'Meta': {'ordering': "['created_at', 'id']", 'object_name': 'PeerWorkflow'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'grading_completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.peerworkflowitem': {
'Meta': {'ordering': "['started_at', 'id']", 'object_name': 'PeerWorkflowItem'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Assessment']", 'null': 'True'}),
'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded_by'", 'to': "orm['assessment.PeerWorkflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scored': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'scorer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded'", 'to': "orm['assessment.PeerWorkflow']"}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.rubric': {
'Meta': {'object_name': 'Rubric'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.studenttrainingworkflow': {
'Meta': {'object_name': 'StudentTrainingWorkflow'},
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.studenttrainingworkflowitem': {
'Meta': {'ordering': "['workflow', 'order_num']", 'unique_together': "(('workflow', 'order_num'),)", 'object_name': 'StudentTrainingWorkflowItem'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'training_example': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.TrainingExample']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['assessment.StudentTrainingWorkflow']"})
},
'assessment.trainingexample': {
'Meta': {'object_name': 'TrainingExample'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options_selected': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['assessment.CriterionOption']", 'symmetrical': 'False'}),
'raw_answer': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"})
}
}
complete_apps = ['assessment']
\ No newline at end of file
apps/openassessment/assessment/migrations/0013_auto__add_field_aigradingworkflow_essay_text.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'AIGradingWorkflow.essay_text'
db.add_column('assessment_aigradingworkflow', 'essay_text',
self.gf('django.db.models.fields.TextField')(default='', blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'AIGradingWorkflow.essay_text'
db.delete_column('assessment_aigradingworkflow', 'essay_text')
models = {
'assessment.aiclassifier': {
'Meta': {'object_name': 'AIClassifier'},
'classifier_data': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'classifiers'", 'to': "orm['assessment.AIClassifierSet']"}),
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Criterion']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.aiclassifierset': {
'Meta': {'ordering': "['-created_at']", 'object_name': 'AIClassifierSet'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"})
},
'assessment.aigradingworkflow': {
'Meta': {'object_name': 'AIGradingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.Assessment']"}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'essay_text': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.aitrainingworkflow': {
'Meta': {'object_name': 'AITrainingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'training_examples': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'+'", 'symmetrical': 'False', 'to': "orm['assessment.TrainingExample']"}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.assessment': {
'Meta': {'ordering': "['-scored_at', '-id']", 'object_name': 'Assessment'},
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"}),
'score_type': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
'scored_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'scorer_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedback': {
'Meta': {'object_name': 'AssessmentFeedback'},
'assessments': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.Assessment']"}),
'feedback_text': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.AssessmentFeedbackOption']"}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedbackoption': {
'Meta': {'object_name': 'AssessmentFeedbackOption'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'text': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'assessment.assessmentpart': {
'Meta': {'object_name': 'AssessmentPart'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parts'", 'to': "orm['assessment.Assessment']"}),
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'option': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.CriterionOption']"})
},
'assessment.criterion': {
'Meta': {'ordering': "['rubric', 'order_num']", 'object_name': 'Criterion'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'prompt': ('django.db.models.fields.TextField', [], {'max_length': '10000'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'criteria'", 'to': "orm['assessment.Rubric']"})
},
'assessment.criterionoption': {
'Meta': {'ordering': "['criterion', 'order_num']", 'object_name': 'CriterionOption'},
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['assessment.Criterion']"}),
'explanation': ('django.db.models.fields.TextField', [], {'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'points': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'assessment.peerworkflow': {
'Meta': {'ordering': "['created_at', 'id']", 'object_name': 'PeerWorkflow'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'grading_completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.peerworkflowitem': {
'Meta': {'ordering': "['started_at', 'id']", 'object_name': 'PeerWorkflowItem'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Assessment']", 'null': 'True'}),
'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded_by'", 'to': "orm['assessment.PeerWorkflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scored': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'scorer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded'", 'to': "orm['assessment.PeerWorkflow']"}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.rubric': {
'Meta': {'object_name': 'Rubric'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.studenttrainingworkflow': {
'Meta': {'object_name': 'StudentTrainingWorkflow'},
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.studenttrainingworkflowitem': {
'Meta': {'ordering': "['workflow', 'order_num']", 'unique_together': "(('workflow', 'order_num'),)", 'object_name': 'StudentTrainingWorkflowItem'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'training_example': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.TrainingExample']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['assessment.StudentTrainingWorkflow']"})
},
'assessment.trainingexample': {
'Meta': {'object_name': 'TrainingExample'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options_selected': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['assessment.CriterionOption']", 'symmetrical': 'False'}),
'raw_answer': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"})
}
}
complete_apps = ['assessment']
\ No newline at end of file
apps/openassessment/assessment/models/ai.py
......@@ -7,16 +7,27 @@ from django.core.files.base import ContentFile
from django.db import models, transaction
from django.utils.timezone import now
from django_extensions.db.fields import UUIDField
from .base import Rubric, Criterion
from submissions import api as sub_api
from openassessment.assessment.serializers import rubric_from_dict
from .base import Rubric, Criterion, Assessment, AssessmentPart
from .training import TrainingExample
AI_ASSESSMENT_TYPE = "AI"
class IncompleteClassifierSet(Exception):
"""
The classifier set is missing a classifier for a criterion in the rubric.
"""
def __init__(self, expected_criteria, actual_criteria):
"""
Construct an error message that explains which criteria were missing.
Args:
expected_criteria (iterable of unicode): The criteria in the rubric.
actual_criteria (iterable of unicode): The criteria specified by the classifier set.
"""
missing_criteria = set(expected_criteria) - set(actual_criteria)
msg = (
......@@ -60,6 +71,7 @@ class AIClassifierSet(models.Model):
class Meta:
app_label = "assessment"
ordering = ['-created_at']
# The rubric associated with this set of classifiers
# We should have one classifier for each of the criteria in the rubric.
......@@ -69,6 +81,9 @@ class AIClassifierSet(models.Model):
# This allows us to find the most recently trained set of classifiers.
created_at = models.DateTimeField(default=now, db_index=True)
# The ID of the algorithm that was used to train classifiers in this set.
algorithm_id = models.CharField(max_length=128, db_index=True)
@classmethod
@transaction.commit_on_success
def create_classifier_set(cls, classifiers_dict, rubric, algorithm_id):
......@@ -91,7 +106,7 @@ class AIClassifierSet(models.Model):
"""
# Create the classifier set
classifier_set = cls.objects.create(rubric=rubric)
classifier_set = cls.objects.create(rubric=rubric, algorithm_id=algorithm_id)
# Retrieve the criteria for this rubric,
# then organize them by criterion name
......@@ -109,8 +124,7 @@ class AIClassifierSet(models.Model):
criterion = criteria.get(criterion_name)
classifier = AIClassifier.objects.create(
classifier_set=classifier_set,
criterion=criterion,
algorithm_id=algorithm_id
criterion=criterion
)
# Serialize the classifier data and upload
......@@ -180,9 +194,6 @@ class AIClassifier(models.Model):
# which allows us to plug in different storage backends (such as S3)
classifier_data = models.FileField(upload_to=AI_CLASSIFIER_STORAGE)
# The ID of the algorithm that was used to train this classifier.
algorithm_id = models.CharField(max_length=128, db_index=True)
def download_classifier_data(self):
"""
Download and deserialize the classifier data.
......@@ -198,28 +209,75 @@ class AIClassifier(models.Model):
return json.loads(self.classifier_data.read()) # pylint:disable=E1101
class AITrainingWorkflow(models.Model):
class AIWorkflow(models.Model):
"""
Used to track all training tasks.
Training tasks take as input an algorithm ID and a set of training examples
(which are associated with a rubric).
On successful completion, training tasks output a set of trained classifiers.
Abstract base class for AI workflow database models.
"""
class Meta:
app_label = "assessment"
abstract = True
# Unique identifier used to track this workflow
uuid = UUIDField(version=1, db_index=True)
# Timestamps
# The task is *scheduled* as soon as a client asks the API to
# train classifiers.
# The task is *completed* when a worker has successfully created a
# classifier set based on the training examples.
scheduled_at = models.DateTimeField(default=now, db_index=True)
completed_at = models.DateTimeField(null=True, db_index=True)
# The ID of the algorithm used to train the classifiers
# This is a parameter passed to and interpreted by the workers.
# Django settings allow the users to map algorithm ID strings
# to the Python code they should use to perform the training.
algorithm_id = models.CharField(max_length=128, db_index=True)
# The set of trained classifiers.
# In the training task, this field will be set when the task completes successfully.
# In the grading task, this may be set to null if no classifiers are available
# when the student submits an essay for grading.
classifier_set = models.ForeignKey(
AIClassifierSet, related_name='+',
null=True, default=None
)
@property
def is_complete(self):
"""
Check whether the workflow is complete.
Returns:
bool
"""
return self.completed_at is not None
def mark_complete_and_save(self):
"""
Mark the workflow as complete.
Returns:
None
"""
self.completed_at = now()
self.save()
class AITrainingWorkflow(AIWorkflow):
"""
Used to track AI training tasks.
Training tasks take as input an algorithm ID and a set of training examples
(which are associated with a rubric).
On successful completion, training tasks output a set of trained classifiers.
"""
class Meta:
app_label = "assessment"
# The training examples (essays + scores) used to train the classifiers.
# This is a many-to-many field because
# (a) we need multiple training examples to train a classifier, and
......@@ -227,21 +285,6 @@ class AITrainingWorkflow(models.Model):
# (for example, if a training task is executed by Celery workers multiple times)
training_examples = models.ManyToManyField(TrainingExample, related_name="+")
# The set of trained classifiers.
# Until the task completes successfully, this will be set to null.
classifier_set = models.ForeignKey(
AIClassifierSet, related_name='training_workflow',
null=True, default=None
)
# Timestamps
# The task is *scheduled* as soon as a client asks the API to
# train classifiers.
# The task is *completed* when a worker has successfully created a
# classifier set based on the training examples.
scheduled_at = models.DateTimeField(default=now, db_index=True)
completed_at = models.DateTimeField(null=True, db_index=True)
@classmethod
@transaction.commit_on_success
def start_workflow(cls, examples, algorithm_id):
......@@ -288,17 +331,6 @@ class AITrainingWorkflow(models.Model):
else:
raise NoTrainingExamples(workflow_uuid=self.uuid)
@property
def is_complete(self):
"""
Check whether the workflow is complete (classifiers have been trained).
Returns:
bool
"""
return self.completed_at is not None
def complete(self, classifier_set):
"""
Add a classifier set to the workflow and mark it complete.
......@@ -319,5 +351,120 @@ class AITrainingWorkflow(models.Model):
self.classifier_set = AIClassifierSet.create_classifier_set(
classifier_set, self.rubric, self.algorithm_id
)
self.completed_at = now()
self.save()
self.mark_complete_and_save()
class AIGradingWorkflow(AIWorkflow):
"""
Used to track AI grading tasks.
Grading tasks take as input an essay submission
and a set of classifiers; the tasks select options
for each criterion in the rubric.
"""
class Meta:
app_label = "assessment"
# The UUID of the submission being graded
submission_uuid = models.CharField(max_length=128, db_index=True)
# The text of the essay submission to grade
# We duplicate this here to avoid having to repeatedly look up
# the submission. Since submissions are immutable, this is safe.
essay_text = models.TextField(blank=True)
# The rubric used to evaluate the submission.
# We store this so we can look for classifiers for the same rubric
# if none are available when the workflow is created.
rubric = models.ForeignKey(Rubric, related_name="+")
# The assessment produced by the AI grading algorithm
# Until the task completes successfully, this will be set to null
assessment = models.ForeignKey(
Assessment, related_name="+", null=True, default=None
)
# Identifier information associated with the student's submission
# Useful for finding workflows for a particular course/item/student
# Since submissions are immutable, and since the workflow is
# associated with one submission, it's safe to duplicate
# this information here from the submissions models.
student_id = models.CharField(max_length=40, db_index=True)
item_id = models.CharField(max_length=128, db_index=True)
course_id = models.CharField(max_length=40, db_index=True)
@classmethod
@transaction.commit_on_success
def start_workflow(cls, submission_uuid, rubric_dict, algorithm_id):
"""
Start a grading workflow.
Args:
submission_uuid (str): The UUID of the submission to grade.
rubric_dict (dict): The serialized rubric model.
algorithm_id (unicode): The ID of the algorithm to use for grading.
Returns:
AIGradingWorkflow
Raises:
SubmissionNotFoundError
SubmissionRequestError
SubmissionInternalError
InvalidRubric
DatabaseError
"""
# Retrieve info about the submission
submission = sub_api.get_submission_and_student(submission_uuid)
# Get or create the rubric
rubric = rubric_from_dict(rubric_dict)
# Retrieve the submission text
# Submissions are arbitrary JSON-blobs, which *should*
# contain a single key, "answer", containing the essay
# submission text. If not, though, assume we've been
# given the essay text directly (convenient for testing).
if isinstance(submission, dict):
essay_text = submission.get('answer')
else:
essay_text = unicode(submission)
# Create the workflow
return cls.objects.create(
submission_uuid=submission_uuid,
essay_text=essay_text,
algorithm_id=algorithm_id,
student_id=submission['student_item']['student_id'],
item_id=submission['student_item']['item_id'],
course_id=submission['student_item']['course_id'],
rubric=rubric
)
@transaction.commit_on_success
def complete(self, criterion_scores):
"""
Create an assessment with scores from the AI classifiers
and mark the workflow complete.
Args:
criterion_scores (dict): Dictionary mapping criteria names to integer scores.
Raises:
DatabaseError
"""
assessment = Assessment.objects.create(
submission_uuid=self.submission_uuid,
rubric=self.rubric,
scorer_id=self.algorithm_id,
score_type=AI_ASSESSMENT_TYPE
)
option_ids = self.rubric.options_ids_for_points(criterion_scores)
AssessmentPart.add_to_assessment(assessment, option_ids)
self.assessment = assessment
self.mark_complete_and_save()
apps/openassessment/assessment/models/base.py
......@@ -166,6 +166,48 @@ class Rubric(models.Model):
return option_id_set
def options_ids_for_points(self, criterion_points):
"""
Given a mapping of selected point values, return the option IDs.
If there are multiple options with the same point value,
this will return the first one (lower order number).
Args:
criterion_points (dict): Mapping of criteria names to point values.
Returns:
list of option IDs
Raises:
InvalidOptionSelection
"""
# This is a really inefficient initial implementation
# TODO -- refactor to add caching
rubric_options = CriterionOption.objects.filter(
criterion__rubric=self
).select_related()
rubric_points_dict = defaultdict(dict)
for option in rubric_options:
if option.points not in rubric_points_dict[option.criterion.name]:
rubric_points_dict[option.criterion.name][option.points] = option.id
option_id_set = set()
for criterion_name, option_points in criterion_points.iteritems():
if (criterion_name in rubric_points_dict and
option_points in rubric_points_dict[criterion_name]
):
option_id = rubric_points_dict[criterion_name][option_points]
option_id_set.add(option_id)
else:
msg = _("{criterion} option with point value {points} not found in rubric").format(
criterion=criterion_name, points=option_points
)
raise InvalidOptionSelection(msg)
return option_id_set
class Criterion(models.Model):
"""A single aspect of a submission that needs assessment.
......
apps/openassessment/assessment/tasks.py
......@@ -4,3 +4,4 @@ so import the tasks we want the workers to implement.
"""
# pylint:disable=W0611
from .worker.training import train_classifiers
from .worker.grading import grade_essay
apps/openassessment/assessment/test/test_ai.py
......@@ -7,11 +7,13 @@ import mock
from django.db import DatabaseError
from django.test.utils import override_settings
from openassessment.test_utils import CacheResetTest
from submissions import api as sub_api
from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.models import AITrainingWorkflow
from openassessment.assessment.models import AITrainingWorkflow, AIClassifierSet
from openassessment.assessment.worker.algorithm import AIAlgorithm
from openassessment.assessment.serializers import rubric_from_dict
from openassessment.assessment.errors import AITrainingRequestError, AITrainingInternalError
from openassessment.assessment.test.constants import RUBRIC, EXAMPLES
from openassessment.assessment.test.constants import RUBRIC, EXAMPLES, STUDENT_ITEM, ANSWER
class StubAIAlgorithm(AIAlgorithm):
......@@ -34,14 +36,25 @@ class StubAIAlgorithm(AIAlgorithm):
# so we can test that the correct inputs were used
classifier = copy.copy(self.FAKE_CLASSIFIER)
classifier['examples'] = examples
classifier['score_override'] = 0
return classifier
def score(self, text, classifier):
"""
Not implemented, but we need to make the abstact
method concrete.
Stub implementation that returns whatever scores were
provided in the serialized classifier data.
Expect `classifier` to be a dict with a single key,
"score_override" containing the score to return.
"""
raise NotImplementedError
return classifier['score_override']
ALGORITHM_ID = "test-stub"
AI_ALGORITHMS = {
ALGORITHM_ID: '{module}.StubAIAlgorithm'.format(module=__name__)
}
class AITrainingTest(CacheResetTest):
......@@ -49,11 +62,6 @@ class AITrainingTest(CacheResetTest):
Tests for AI training tasks.
"""
ALGORITHM_ID = "test-stub"
AI_ALGORITHMS = {
ALGORITHM_ID: '{module}.StubAIAlgorithm'.format(module=__name__)
}
EXPECTED_INPUT_SCORES = {
u'vøȼȺƀᵾłȺɍɏ': [1, 0],
u'ﻭɼค๓๓คɼ': [0, 2]
......@@ -65,7 +73,7 @@ class AITrainingTest(CacheResetTest):
# Schedule a training task
# Because Celery is configured in "always eager" mode,
# expect the task to be executed synchronously.
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, self.ALGORITHM_ID)
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
# Retrieve the classifier set from the database
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
......@@ -106,12 +114,12 @@ class AITrainingTest(CacheResetTest):
# Expect a request error
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, mutated_examples, self.ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, mutated_examples, ALGORITHM_ID)
def test_train_classifiers_no_examples(self):
# Empty list of training examples
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, [], self.ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, [], ALGORITHM_ID)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch.object(AITrainingWorkflow.objects, 'create')
......@@ -119,7 +127,7 @@ class AITrainingTest(CacheResetTest):
# Simulate a database error when creating the training workflow
mock_create.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, self.ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.api.ai.training_tasks')
......@@ -127,4 +135,47 @@ class AITrainingTest(CacheResetTest):
# Simulate an exception raised when scheduling a training task
mock_training_tasks.train_classifiers.apply_async.side_effect = Exception("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, self.ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
class AIGradingTest(CacheResetTest):
"""
Tests for AI grading tasks.
"""
CLASSIFIER_SCORE_OVERRIDES = {
u"vøȼȺƀᵾłȺɍɏ": {'score_override': 1},
u"ﻭɼค๓๓คɼ": {'score_override': 2}
}
def setUp(self):
"""
Create a submission and a fake classifier set.
"""
# Create a submission
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
# Create the classifier set for our fake AI algorithm
# To isolate these tests from the tests for the training
# task, we use the database models directly.
# We also use a stub AI algorithm that simply returns
# whatever scores we specify in the classifier data.
rubric = rubric_from_dict(RUBRIC)
AIClassifierSet.create_classifier_set(
self.CLASSIFIER_SCORE_OVERRIDES, rubric, ALGORITHM_ID
)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_grade_essay(self):
# Schedule a grading task
# Because Celery is configured in "always eager" mode, this will
# be executed synchronously.
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
# Verify that we got the scores we provided to the stub AI algorithm
assessment = ai_api.get_latest_assessment(self.submission_uuid)
for part in assessment['parts']:
criterion_name = part['option']['criterion']['name']
expected_score = self.CLASSIFIER_SCORE_OVERRIDES[criterion_name]['score_override']
self.assertEqual(part['option']['points'], expected_score)
apps/openassessment/assessment/test/test_ai_worker.py
......@@ -45,22 +45,8 @@ class AIWorkerTrainingTest(CacheResetTest):
workflow = AITrainingWorkflow.start_workflow(examples, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def test_get_algorithm_id(self):
algorithm_id = ai_worker_api.get_algorithm_id(self.workflow_uuid)
self.assertEqual(algorithm_id, self.ALGORITHM_ID)
def test_get_algorithm_id_no_workflow(self):
with self.assertRaises(AITrainingRequestError):
ai_worker_api.get_algorithm_id("invalid_uuid")
@mock.patch.object(AITrainingWorkflow.objects, 'get')
def test_get_algorithm_id_database_error(self, mock_get):
mock_get.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_worker_api.get_algorithm_id(self.workflow_uuid)
def test_get_training_examples(self):
examples = ai_worker_api.get_training_examples(self.workflow_uuid)
def test_get_training_task_params(self):
params = ai_worker_api.get_training_task_params(self.workflow_uuid)
expected_examples = [
{
'text': EXAMPLES[0]['answer'],
......@@ -77,17 +63,18 @@ class AIWorkerTrainingTest(CacheResetTest):
}
},
]
self.assertItemsEqual(examples, expected_examples)
self.assertItemsEqual(params['training_examples'], expected_examples)
self.assertItemsEqual(params['algorithm_id'], self.ALGORITHM_ID)
def test_get_training_examples_no_workflow(self):
def test_get_training_task_params_no_workflow(self):
with self.assertRaises(AITrainingRequestError):
ai_worker_api.get_training_examples("invalid_uuid")
ai_worker_api.get_training_task_params("invalid_uuid")
@mock.patch.object(AITrainingWorkflow.objects, 'get')
def test_get_training_examples_database_error(self, mock_get):
def test_get_training_task_params_database_error(self, mock_get):
mock_get.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_worker_api.get_training_examples(self.workflow_uuid)
ai_worker_api.get_training_task_params(self.workflow_uuid)
def test_create_classifiers(self):
ai_worker_api.create_classifiers(self.workflow_uuid, self.CLASSIFIERS)
......
apps/openassessment/assessment/test/test_worker.py
......@@ -3,7 +3,6 @@
Tests for AI worker tasks.
"""
from contextlib import contextmanager
import datetime
import mock
from django.test.utils import override_settings
from openassessment.test_utils import CacheResetTest
......@@ -64,13 +63,6 @@ class AITrainingTaskTest(CacheResetTest):
workflow = AITrainingWorkflow.start_workflow(examples, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.worker.training.ai_worker_api.get_algorithm_id')
def test_get_algorithm_id_api_error(self, mock_call):
mock_call.side_effect = AITrainingRequestError("Test error!")
with self._assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers(self.workflow_uuid)
def test_unknown_algorithm(self):
# Since we haven't overridden settings to configure the algorithms,
# the worker will not recognize the workflow's algorithm ID.
......@@ -92,8 +84,8 @@ class AITrainingTaskTest(CacheResetTest):
train_classifiers(self.workflow_uuid)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.worker.training.ai_worker_api.get_training_examples')
def test_get_training_examples_api_error(self, mock_call):
@mock.patch('openassessment.assessment.worker.training.ai_worker_api.get_training_task_params')
def test_get_training_task_params_api_error(self, mock_call):
mock_call.side_effect = AITrainingRequestError("Test error!")
with self._assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers(self.workflow_uuid)
......@@ -160,12 +152,12 @@ class AITrainingTaskTest(CacheResetTest):
AssertionError
"""
examples = ai_worker_api.get_training_examples(self.workflow_uuid)
mutate_func(examples)
params = ai_worker_api.get_training_task_params(self.workflow_uuid)
mutate_func(params['training_examples'])
call_signature = 'openassessment.assessment.worker.training.ai_worker_api.get_training_examples'
call_signature = 'openassessment.assessment.worker.training.ai_worker_api.get_training_task_params'
with mock.patch(call_signature) as mock_call:
mock_call.return_value = examples
mock_call.return_value = params
with self._assert_retry(train_classifiers, InvalidExample):
train_classifiers(self.workflow_uuid)
......
apps/openassessment/assessment/worker/grading.py 0 → 100644
"""
Asynchronous tasks for grading essays using text classifiers.
"""
from celery import task
from celery.utils.log import get_task_logger
from openassessment.assessment.api import ai_worker as ai_worker_api
from openassessment.assessment.errors import AIError
from .algorithm import AIAlgorithm, AIAlgorithmError
MAX_RETRIES = 2
logger = get_task_logger(__name__)
@task(max_retries=MAX_RETRIES) # pylint: disable=E1102
def grade_essay(workflow_uuid):
"""
Asynchronous task to grade an essay using a text classifier
(trained using a supervised ML algorithm).
If the task could not be completed successfully,
it will be retried a few times; if it continues to fail,
it is left incomplete. Incomplate tasks can be rescheduled
manually through the AI API.
Args:
workflow_uuid (str): The UUID of the workflow associated
with this grading task.
Returns:
None
Raises:
AIError: An error occurred while making an AI worker API call.
AIAlgorithmError: An error occurred while retrieving or using an AI algorithm.
"""
# Retrieve the task parameters
try:
params = ai_worker_api.get_grading_task_params(workflow_uuid)
essay_text = params['essay_text']
classifier_set = params['classifier_set']
algorithm_id = params['algorithm_id']
except (AIError, KeyError):
msg = (
u"An error occurred while retrieving the AI grading task "
u"parameters for the workflow with UUID {}"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Retrieve the AI algorithm
try:
algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
except AIAlgorithmError:
msg = (
u"An error occurred while retrieving "
u"the algorithm ID (grading workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Use the algorithm to evaluate the essay for each criterion
try:
scores_by_criterion = {
criterion_name: algorithm.score(essay_text, classifier)
for criterion_name, classifier in classifier_set.iteritems()
}
except AIAlgorithmError:
msg = (
u"An error occurred while scoring essays using "
u"an AI algorithm (worker workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Create the assessment and mark the workflow complete
try:
ai_worker_api.create_assessment(workflow_uuid, scores_by_criterion)
except AIError:
msg = (
u"An error occurred while creating assessments "
u"for the AI grading workflow with UUID {uuid}. "
u"The assessment scores were: {scores}"
).format(uuid=workflow_uuid, scores=scores_by_criterion)
logger.exception(msg)
raise grade_essay.retry()
apps/openassessment/assessment/worker/training.py
......@@ -53,10 +53,22 @@ def train_classifiers(workflow_uuid):
InvalidExample: The training examples provided by the AI API were not valid.
"""
# Retrieve task parameters
try:
params = ai_worker_api.get_training_task_params(workflow_uuid)
examples = params['training_examples']
algorithm_id = params['algorithm_id']
except (AIError, KeyError):
msg = (
u"An error occurred while retrieving AI training "
u"task parameters for the workflow with UUID {}"
).format(workflow_uuid)
logger.exception(msg)
raise train_classifiers.retry()
# Retrieve the ML algorithm to use for training
# (based on task params and worker configuration)
try:
algorithm_id = ai_worker_api.get_algorithm_id(workflow_uuid)
algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
except AIAlgorithmError:
msg = (
......@@ -73,19 +85,6 @@ def train_classifiers(workflow_uuid):
logger.exception(msg)
raise train_classifiers.retry()
# Retrieve training examples, then transform them into the
# data structures we use internally.
try:
examples = ai_worker_api.get_training_examples(workflow_uuid)
except AIError:
msg = (
u"An error occurred while retrieving "
u"training examples for AI training "
u"(training workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise train_classifiers.retry()
# Train a classifier for each criterion
# The AIAlgorithm subclass is responsible for ensuring that
# the trained classifiers are JSON-serializable.
......
docs/en_us/developers/source/architecture/ai_grading.rst
......@@ -240,17 +240,15 @@ Data Model
1. **GradingWorkflow**
a. Submission UUID (varchar)
b. Rubric UUID (varchar)
c. ClassifierSet (Foreign Key, Nullable)
d. Assessment (Foreign Key, Nullable)
e. Scheduled at (timestamp): The time the task was placed on the queue.
f. Started at (timestamp): The time the task was picked up by the worker.
b. ClassifierSet (Foreign Key, Nullable)
c. Assessment (Foreign Key, Nullable)
d. Rubric (Foreign Key): Used to search for classifier sets if none are available when the workflow is started.
e. Algorithm ID (varchar): Used to search for classifier sets if none are available when the workflow is started.
f. Scheduled at (timestamp): The time the task was placed on the queue.
g. Completed at (timestamp): The time the task was completed. If set, the task is considered complete.
h. Course ID (varchar): The ID of the course associated with the submission. Useful for rescheduling
failed grading tasks in a particular course.
i. Item ID (varchar): The ID of the item (problem) associated with the submission. Useful for rescheduling
failed grading tasks in a particular item in a course.
j. Worker version (varchar): Identifier for the code running on the worker when the task was started. Useful for error tracking.
h. Course ID (varchar): The ID of the course associated with the submission. Useful for rescheduling failed grading tasks in a particular course.
i. Item ID (varchar): The ID of the item (problem) associated with the submission. Useful for rescheduling failed grading tasks in a particular item in a course.
2. **TrainingWorkflow**
......@@ -269,13 +267,13 @@ Data Model
a. Rubric (Foreign Key)
b. Created at (timestamp)
c. Algorithm ID (varchar)
5. **Classifier**
a. ClassifierSet (Foreign Key)
b. URL for trained classifier (varchar)
c. Algorithm ID (varchar)
d. Criterion (Foreign Key)
c. Criterion (Foreign Key)
6. **Assessment** (same as current implementation)
......
scripts/test.sh
......@@ -4,5 +4,5 @@
set -e
cd `dirname $BASH_SOURCE` && cd ..
./scripts/test-python.sh
./scripts/test-python.sh $1
./scripts/test-js.sh
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment