Skip to content

E
edx-ora2
Commit 74a9d63e by Will Daly
Options

Implement AI grading task, database models, and API calls.

parent ec584292
Showing with 942 additions and 187 deletions
apps/openassessment/assessment/api/ai.py
...@@ -2,22 +2,29 @@ ...@@ -2,22 +2,29 @@
Public interface for AI training and grading, used by students/course authors. Public interface for AI training and grading, used by students/course authors.
""" """
import logging import logging
from django.db import DatabaseError
from submissions import api as sub_api
from openassessment.assessment.serializers import ( from openassessment.assessment.serializers import (
deserialize_training_examples, InvalidTrainingExample, InvalidRubric deserialize_training_examples, InvalidTrainingExample, InvalidRubric,
full_assessment_dict
) )
from openassessment.assessment.errors import ( from openassessment.assessment.errors import (
AITrainingRequestError, AITrainingInternalError AITrainingRequestError, AITrainingInternalError,
AIGradingRequestError, AIGradingInternalError
) )
from openassessment.assessment.models import ( from openassessment.assessment.models import (
AITrainingWorkflow, InvalidOptionSelection, NoTrainingExamples AITrainingWorkflow, InvalidOptionSelection, NoTrainingExamples,
Assessment, AITrainingWorkflow, AIGradingWorkflow,
AIClassifierSet, AI_ASSESSMENT_TYPE
) )
from openassessment.assessment.worker import training as training_tasks from openassessment.assessment.worker import training as training_tasks
from openassessment.assessment.worker import grading as grading_tasks
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def submit(submission_uuid, rubric): def submit(submission_uuid, rubric, algorithm_id):
""" """
Submit a response for AI assessment. Submit a response for AI assessment.
This will: This will:
...@@ -27,6 +34,7 @@ def submit(submission_uuid, rubric): ...@@ -27,6 +34,7 @@ def submit(submission_uuid, rubric):
Args: Args:
submission_uuid (str): The UUID of the submission to assess. submission_uuid (str): The UUID of the submission to assess.
rubric (dict): Serialized rubric model. rubric (dict): Serialized rubric model.
algorithm_id (unicode): Use only classifiers trained with the specified algorithm.
Returns: Returns:
grading_workflow_uuid (str): The UUID of the grading workflow. grading_workflow_uuid (str): The UUID of the grading workflow.
...@@ -39,7 +47,50 @@ def submit(submission_uuid, rubric): ...@@ -39,7 +47,50 @@ def submit(submission_uuid, rubric):
AIGradingInternalError AIGradingInternalError
""" """
pass try:
workflow = AIGradingWorkflow.start_workflow(submission_uuid, rubric, algorithm_id)
except (sub_api.SubmissionNotFoundError, sub_api.SubmissionRequestError) as ex:
msg = (
u"An error occurred while retrieving the "
u"submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
raise AIGradingRequestError(msg)
except InvalidRubric as ex:
msg = (
u"An error occurred while parsing the serialized "
u"rubric {rubric}: {ex}"
).format(rubric=rubric, ex=ex)
raise AIGradingRequestError(msg)
except (sub_api.SubmissionInternalError, DatabaseError) as ex:
msg = (
u"An unexpected error occurred while submitting an "
u"essay for AI grading: {ex}"
).format(ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
try:
classifier_set_candidates = AIClassifierSet.objects.filter(
rubric=workflow.rubric, algorithm_id=algorithm_id
).order_by('-created_at')[:1]
# If we find classifiers for this rubric/algorithm
# then associate the classifiers with the workflow
# and schedule a grading task.
# Otherwise, the task will need to be scheduled later,
# once the classifiers have been trained.
if len(classifier_set_candidates) > 0:
workflow.classifier_set = classifier_set_candidates[0]
workflow.save()
grading_tasks.grade_essay.apply_async(args=[workflow.uuid])
return workflow.uuid
except Exception as ex:
msg = (
u"An unexpected error occurred while scheduling the "
u"AI grading task for the submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
raise AIGradingInternalError(msg)
def get_latest_assessment(submission_uuid): def get_latest_assessment(submission_uuid):
...@@ -51,13 +102,29 @@ def get_latest_assessment(submission_uuid): ...@@ -51,13 +102,29 @@ def get_latest_assessment(submission_uuid):
Returns: Returns:
dict: The serialized assessment model dict: The serialized assessment model
or None if no assessments are available
Raises: Raises:
AIGradingRequestError
AIGradingInternalError AIGradingInternalError
""" """
pass try:
assessments = Assessment.objects.filter(
submission_uuid=submission_uuid,
score_type=AI_ASSESSMENT_TYPE,
)[:1]
except DatabaseError as ex:
msg = (
u"An error occurred while retrieving AI graded assessments "
u"for the submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
if len(assessments) > 0:
return full_assessment_dict(assessments[0])
else:
return None
def train_classifiers(rubric_dict, examples, algorithm_id): def train_classifiers(rubric_dict, examples, algorithm_id):
......
apps/openassessment/assessment/api/ai_worker.py
...@@ -5,63 +5,85 @@ import logging ...@@ -5,63 +5,85 @@ import logging
from django.utils.timezone import now from django.utils.timezone import now
from django.db import DatabaseError from django.db import DatabaseError
from openassessment.assessment.models import ( from openassessment.assessment.models import (
AITrainingWorkflow, AIClassifierSet, AITrainingWorkflow, AIGradingWorkflow, AIClassifierSet,
ClassifierUploadError, ClassifierSerializeError, ClassifierUploadError, ClassifierSerializeError,
IncompleteClassifierSet, NoTrainingExamples IncompleteClassifierSet, NoTrainingExamples
) )
from openassessment.assessment.errors import ( from openassessment.assessment.errors import (
AITrainingRequestError, AITrainingInternalError AITrainingRequestError, AITrainingInternalError,
AIGradingRequestError, AIGradingInternalError
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def get_grading_task_params(grading_workflow_uuid):
def get_submission(grading_workflow_uuid):
""" """
Retrieve the submission associated with a particular grading workflow. Retrieve the classifier set and algorithm ID
associated with a particular grading workflow.
Args: Args:
grading_workflow_uuid (str): The UUID of the grading workflow. grading_workflow_uuid (str): The UUID of the grading workflow.
Returns: Returns:
submission (JSON-serializable): submission from the student. dict with keys:
* essay_text (unicode): The text of the essay submission.
* classifier_set (dict): Maps criterion names to serialized classifiers.
* algorithm_id (unicode): ID of the algorithm used to perform training.
Raises: Raises:
AIGradingRequestError AIGradingRequestError
AIGradingInternalError AIGradingInternalError
""" """
pass try:
workflow = AIGradingWorkflow.objects.get(uuid=grading_workflow_uuid)
except AIGradingWorkflow.DoesNotExist:
def get_classifier_set(grading_workflow_uuid): msg = (
""" u"Could not retrieve the AI grading workflow with uuid {}"
Retrieve the classifier set associated with a particular grading workflow. ).format(grading_workflow_uuid)
raise AIGradingRequestError(msg)
Args: except DatabaseError as ex:
grading_workflow_uuid (str): The UUID of the grading workflow. msg = (
u"An unexpected error occurred while retrieving the "
Returns: u"AI grading workflow with uuid {uuid}: {ex}"
dict: Maps criterion names to serialized classifiers. ).format(uuid=grading_workflow_uuid, ex=ex)
(binary classifiers are base-64 encoded). logger.exception(msg)
raise AIGradingInternalError(msg)
Raises: classifier_set = workflow.classifier_set
AIGradingRequestError # Tasks shouldn't be scheduled until a classifier set is
AIGradingInternalError # available, so this is a serious internal error.
if classifier_set is None:
msg = (
u"AI grading workflow with UUID {} has no classifier set"
).format(grading_workflow_uuid)
logger.exception(msg)
raise AIGradingInternalError(msg)
""" try:
pass return {
'essay_text': workflow.essay_text,
'classifier_set': classifier_set.classifiers_dict,
'algorithm_id': workflow.algorithm_id,
}
except (ValueError, IOError, DatabaseError) as ex:
msg = (
u"An unexpected error occurred while retrieving "
u"classifiers for the grading workflow with UUID {uuid}: {ex}"
).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
def create_assessment(grading_workflow_uuid, assessment): def create_assessment(grading_workflow_uuid, criterion_scores):
""" """
Create an AI assessment (complete the AI grading task). Create an AI assessment (complete the AI grading task).
Args: Args:
grading_workflow_uuid (str): The UUID of the grading workflow. grading_workflow_uuid (str): The UUID of the grading workflow.
assessment (dict): The serialized assessment. criterion_scores (dict): Dictionary mapping criteria names to integer scores.
Returns: Returns:
None None
...@@ -71,57 +93,59 @@ def create_assessment(grading_workflow_uuid, assessment): ...@@ -71,57 +93,59 @@ def create_assessment(grading_workflow_uuid, assessment):
AIGradingInternalError AIGradingInternalError
""" """
pass
def get_algorithm_id(training_workflow_uuid):
"""
Retrieve the ID of the algorithm to use.
Args:
training_workflow_uuid (str): The UUID of the training workflow.
Returns:
unicode: The algorithm ID associated with the training task.
Raises:
AITrainingRequestError
AITrainingInternalError
"""
try: try:
workflow = AITrainingWorkflow.objects.get(uuid=training_workflow_uuid) workflow = AIGradingWorkflow.objects.get(uuid=grading_workflow_uuid)
return workflow.algorithm_id except AIGradingWorkflow.DoesNotExist:
except AITrainingWorkflow.DoesNotExist:
msg = ( msg = (
u"Could not retrieve AI training workflow with UUID {}" u"Could not retrieve the AI grading workflow with uuid {}"
).format(training_workflow_uuid) ).format(grading_workflow_uuid)
raise AITrainingRequestError(msg) raise AIGradingRequestError(msg)
except DatabaseError: except DatabaseError as ex:
msg = ( msg = (
u"An unexpected error occurred while retrieving " u"An unexpected error occurred while retrieving the "
u"the algorithm ID for training workflow with UUID {}" u"AI grading workflow with uuid {uuid}: {ex}"
).format(training_workflow_uuid) ).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg) logger.exception(msg)
raise AITrainingInternalError(msg) raise AIGradingInternalError(msg)
# Optimization: if the workflow has already been marked complete
# (perhaps the task was picked up by multiple workers),
# then we don't need to do anything.
# Otherwise, create the assessment mark the workflow complete.
try:
if not workflow.is_complete:
workflow.complete(criterion_scores)
except DatabaseError as ex:
msg = (
u"An unexpected error occurred while creating the assessment "
u"for AI grading workflow with uuid {uuid}: {ex}"
).format(uuid=grading_workflow_uuid, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(msg)
def get_training_examples(training_workflow_uuid): def get_training_task_params(training_workflow_uuid):
""" """
Retrieve the training examples associated with a training task. Retrieve the training examples and algorithm ID
associated with a training task.
Args: Args:
training_workflow_uuid (str): The UUID of the training workflow. training_workflow_uuid (str): The UUID of the training workflow.
Returns: Returns:
list of dict: Serialized training examples, of the form: dict with keys:
* training_examples (list of dict): The examples used to train the classifiers.
* algorithm_id (unicode): The ID of the algorithm to use for training.
Raises: Raises:
AITrainingRequestError AITrainingRequestError
AITrainingInternalError AITrainingInternalError
Example usage: Example usage:
>>> get_training_examples('abcd1234') >>> params = get_training_task_params('abcd1234')
>>> params['algorithm_id']
u'ease'
>>> params['training_examples']
[ [
{ {
"text": u"Example answer number one", "text": u"Example answer number one",
...@@ -161,7 +185,10 @@ def get_training_examples(training_workflow_uuid): ...@@ -161,7 +185,10 @@ def get_training_examples(training_workflow_uuid):
'scores': scores 'scores': scores
}) })
return returned_examples return {
'training_examples': returned_examples,
'algorithm_id': workflow.algorithm_id
}
except AITrainingWorkflow.DoesNotExist: except AITrainingWorkflow.DoesNotExist:
msg = ( msg = (
u"Could not retrieve AI training workflow with UUID {}" u"Could not retrieve AI training workflow with UUID {}"
......
apps/openassessment/assessment/migrations/0012_move_algorithm_id_to_classifier_set.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'AIGradingWorkflow'
db.create_table('assessment_aigradingworkflow', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('uuid', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=36, blank=True)),
('scheduled_at', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, db_index=True)),
('completed_at', self.gf('django.db.models.fields.DateTimeField')(null=True, db_index=True)),
('submission_uuid', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
('classifier_set', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='+', null=True, to=orm['assessment.AIClassifierSet'])),
('algorithm_id', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
('rubric', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['assessment.Rubric'])),
('assessment', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='+', null=True, to=orm['assessment.Assessment'])),
('student_id', self.gf('django.db.models.fields.CharField')(max_length=40, db_index=True)),
('item_id', self.gf('django.db.models.fields.CharField')(max_length=128, db_index=True)),
('course_id', self.gf('django.db.models.fields.CharField')(max_length=40, db_index=True)),
))
db.send_create_signal('assessment', ['AIGradingWorkflow'])
# Adding field 'AIClassifierSet.algorithm_id'
db.add_column('assessment_aiclassifierset', 'algorithm_id',
self.gf('django.db.models.fields.CharField')(default='ease', max_length=128, db_index=True),
keep_default=False)
# Deleting field 'AIClassifier.algorithm_id'
db.delete_column('assessment_aiclassifier', 'algorithm_id')
def backwards(self, orm):
# Deleting model 'AIGradingWorkflow'
db.delete_table('assessment_aigradingworkflow')
# Deleting field 'AIClassifierSet.algorithm_id'
db.delete_column('assessment_aiclassifierset', 'algorithm_id')
# Adding field 'AIClassifier.algorithm_id'
db.add_column('assessment_aiclassifier', 'algorithm_id',
self.gf('django.db.models.fields.CharField')(default='ease', max_length=128, db_index=True),
keep_default=False)
models = {
'assessment.aiclassifier': {
'Meta': {'object_name': 'AIClassifier'},
'classifier_data': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'classifiers'", 'to': "orm['assessment.AIClassifierSet']"}),
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Criterion']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.aiclassifierset': {
'Meta': {'ordering': "['-created_at']", 'object_name': 'AIClassifierSet'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"})
},
'assessment.aigradingworkflow': {
'Meta': {'object_name': 'AIGradingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.Assessment']"}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.aitrainingworkflow': {
'Meta': {'object_name': 'AITrainingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'training_examples': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'+'", 'symmetrical': 'False', 'to': "orm['assessment.TrainingExample']"}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.assessment': {
'Meta': {'ordering': "['-scored_at', '-id']", 'object_name': 'Assessment'},
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"}),
'score_type': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
'scored_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'scorer_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedback': {
'Meta': {'object_name': 'AssessmentFeedback'},
'assessments': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.Assessment']"}),
'feedback_text': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.AssessmentFeedbackOption']"}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedbackoption': {
'Meta': {'object_name': 'AssessmentFeedbackOption'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'text': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'assessment.assessmentpart': {
'Meta': {'object_name': 'AssessmentPart'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parts'", 'to': "orm['assessment.Assessment']"}),
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'option': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.CriterionOption']"})
},
'assessment.criterion': {
'Meta': {'ordering': "['rubric', 'order_num']", 'object_name': 'Criterion'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'prompt': ('django.db.models.fields.TextField', [], {'max_length': '10000'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'criteria'", 'to': "orm['assessment.Rubric']"})
},
'assessment.criterionoption': {
'Meta': {'ordering': "['criterion', 'order_num']", 'object_name': 'CriterionOption'},
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['assessment.Criterion']"}),
'explanation': ('django.db.models.fields.TextField', [], {'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'points': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'assessment.peerworkflow': {
'Meta': {'ordering': "['created_at', 'id']", 'object_name': 'PeerWorkflow'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'grading_completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.peerworkflowitem': {
'Meta': {'ordering': "['started_at', 'id']", 'object_name': 'PeerWorkflowItem'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Assessment']", 'null': 'True'}),
'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded_by'", 'to': "orm['assessment.PeerWorkflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scored': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'scorer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded'", 'to': "orm['assessment.PeerWorkflow']"}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.rubric': {
'Meta': {'object_name': 'Rubric'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.studenttrainingworkflow': {
'Meta': {'object_name': 'StudentTrainingWorkflow'},
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.studenttrainingworkflowitem': {
'Meta': {'ordering': "['workflow', 'order_num']", 'unique_together': "(('workflow', 'order_num'),)", 'object_name': 'StudentTrainingWorkflowItem'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'training_example': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.TrainingExample']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['assessment.StudentTrainingWorkflow']"})
},
'assessment.trainingexample': {
'Meta': {'object_name': 'TrainingExample'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options_selected': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['assessment.CriterionOption']", 'symmetrical': 'False'}),
'raw_answer': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"})
}
}
complete_apps = ['assessment']
\ No newline at end of file
apps/openassessment/assessment/migrations/0013_auto__add_field_aigradingworkflow_essay_text.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'AIGradingWorkflow.essay_text'
db.add_column('assessment_aigradingworkflow', 'essay_text',
self.gf('django.db.models.fields.TextField')(default='', blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'AIGradingWorkflow.essay_text'
db.delete_column('assessment_aigradingworkflow', 'essay_text')
models = {
'assessment.aiclassifier': {
'Meta': {'object_name': 'AIClassifier'},
'classifier_data': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'classifiers'", 'to': "orm['assessment.AIClassifierSet']"}),
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Criterion']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.aiclassifierset': {
'Meta': {'ordering': "['-created_at']", 'object_name': 'AIClassifierSet'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"})
},
'assessment.aigradingworkflow': {
'Meta': {'object_name': 'AIGradingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.Assessment']"}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'essay_text': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.aitrainingworkflow': {
'Meta': {'object_name': 'AITrainingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'training_examples': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'+'", 'symmetrical': 'False', 'to': "orm['assessment.TrainingExample']"}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.assessment': {
'Meta': {'ordering': "['-scored_at', '-id']", 'object_name': 'Assessment'},
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"}),
'score_type': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
'scored_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'scorer_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedback': {
'Meta': {'object_name': 'AssessmentFeedback'},
'assessments': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.Assessment']"}),
'feedback_text': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.AssessmentFeedbackOption']"}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedbackoption': {
'Meta': {'object_name': 'AssessmentFeedbackOption'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'text': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'assessment.assessmentpart': {
'Meta': {'object_name': 'AssessmentPart'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parts'", 'to': "orm['assessment.Assessment']"}),
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'option': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.CriterionOption']"})
},
'assessment.criterion': {
'Meta': {'ordering': "['rubric', 'order_num']", 'object_name': 'Criterion'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'prompt': ('django.db.models.fields.TextField', [], {'max_length': '10000'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'criteria'", 'to': "orm['assessment.Rubric']"})
},
'assessment.criterionoption': {
'Meta': {'ordering': "['criterion', 'order_num']", 'object_name': 'CriterionOption'},
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['assessment.Criterion']"}),
'explanation': ('django.db.models.fields.TextField', [], {'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'points': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'assessment.peerworkflow': {
'Meta': {'ordering': "['created_at', 'id']", 'object_name': 'PeerWorkflow'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'grading_completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.peerworkflowitem': {
'Meta': {'ordering': "['started_at', 'id']", 'object_name': 'PeerWorkflowItem'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Assessment']", 'null': 'True'}),
'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded_by'", 'to': "orm['assessment.PeerWorkflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scored': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'scorer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded'", 'to': "orm['assessment.PeerWorkflow']"}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.rubric': {
'Meta': {'object_name': 'Rubric'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.studenttrainingworkflow': {
'Meta': {'object_name': 'StudentTrainingWorkflow'},
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.studenttrainingworkflowitem': {
'Meta': {'ordering': "['workflow', 'order_num']", 'unique_together': "(('workflow', 'order_num'),)", 'object_name': 'StudentTrainingWorkflowItem'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'training_example': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.TrainingExample']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['assessment.StudentTrainingWorkflow']"})
},
'assessment.trainingexample': {
'Meta': {'object_name': 'TrainingExample'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options_selected': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['assessment.CriterionOption']", 'symmetrical': 'False'}),
'raw_answer': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"})
}
}
complete_apps = ['assessment']
\ No newline at end of file
apps/openassessment/assessment/models/ai.py
...@@ -7,16 +7,27 @@ from django.core.files.base import ContentFile ...@@ -7,16 +7,27 @@ from django.core.files.base import ContentFile
from django.db import models, transaction from django.db import models, transaction
from django.utils.timezone import now from django.utils.timezone import now
from django_extensions.db.fields import UUIDField from django_extensions.db.fields import UUIDField
from .base import Rubric, Criterion from submissions import api as sub_api
from openassessment.assessment.serializers import rubric_from_dict
from .base import Rubric, Criterion, Assessment, AssessmentPart
from .training import TrainingExample from .training import TrainingExample
AI_ASSESSMENT_TYPE = "AI"
class IncompleteClassifierSet(Exception): class IncompleteClassifierSet(Exception):
""" """
The classifier set is missing a classifier for a criterion in the rubric. The classifier set is missing a classifier for a criterion in the rubric.
""" """
def __init__(self, expected_criteria, actual_criteria): def __init__(self, expected_criteria, actual_criteria):
""" """
Construct an error message that explains which criteria were missing.
Args:
expected_criteria (iterable of unicode): The criteria in the rubric.
actual_criteria (iterable of unicode): The criteria specified by the classifier set.
""" """
missing_criteria = set(expected_criteria) - set(actual_criteria) missing_criteria = set(expected_criteria) - set(actual_criteria)
msg = ( msg = (
...@@ -60,6 +71,7 @@ class AIClassifierSet(models.Model): ...@@ -60,6 +71,7 @@ class AIClassifierSet(models.Model):
class Meta: class Meta:
app_label = "assessment" app_label = "assessment"
ordering = ['-created_at']
# The rubric associated with this set of classifiers # The rubric associated with this set of classifiers
# We should have one classifier for each of the criteria in the rubric. # We should have one classifier for each of the criteria in the rubric.
...@@ -69,6 +81,9 @@ class AIClassifierSet(models.Model): ...@@ -69,6 +81,9 @@ class AIClassifierSet(models.Model):
# This allows us to find the most recently trained set of classifiers. # This allows us to find the most recently trained set of classifiers.
created_at = models.DateTimeField(default=now, db_index=True) created_at = models.DateTimeField(default=now, db_index=True)
# The ID of the algorithm that was used to train classifiers in this set.
algorithm_id = models.CharField(max_length=128, db_index=True)
@classmethod @classmethod
@transaction.commit_on_success @transaction.commit_on_success
def create_classifier_set(cls, classifiers_dict, rubric, algorithm_id): def create_classifier_set(cls, classifiers_dict, rubric, algorithm_id):
...@@ -91,7 +106,7 @@ class AIClassifierSet(models.Model): ...@@ -91,7 +106,7 @@ class AIClassifierSet(models.Model):
""" """
# Create the classifier set # Create the classifier set
classifier_set = cls.objects.create(rubric=rubric) classifier_set = cls.objects.create(rubric=rubric, algorithm_id=algorithm_id)
# Retrieve the criteria for this rubric, # Retrieve the criteria for this rubric,
# then organize them by criterion name # then organize them by criterion name
...@@ -109,8 +124,7 @@ class AIClassifierSet(models.Model): ...@@ -109,8 +124,7 @@ class AIClassifierSet(models.Model):
criterion = criteria.get(criterion_name) criterion = criteria.get(criterion_name)
classifier = AIClassifier.objects.create( classifier = AIClassifier.objects.create(
classifier_set=classifier_set, classifier_set=classifier_set,
criterion=criterion, criterion=criterion
algorithm_id=algorithm_id
) )
# Serialize the classifier data and upload # Serialize the classifier data and upload
...@@ -180,9 +194,6 @@ class AIClassifier(models.Model): ...@@ -180,9 +194,6 @@ class AIClassifier(models.Model):
# which allows us to plug in different storage backends (such as S3) # which allows us to plug in different storage backends (such as S3)
classifier_data = models.FileField(upload_to=AI_CLASSIFIER_STORAGE) classifier_data = models.FileField(upload_to=AI_CLASSIFIER_STORAGE)
# The ID of the algorithm that was used to train this classifier.
algorithm_id = models.CharField(max_length=128, db_index=True)
def download_classifier_data(self): def download_classifier_data(self):
""" """
Download and deserialize the classifier data. Download and deserialize the classifier data.
...@@ -198,28 +209,75 @@ class AIClassifier(models.Model): ...@@ -198,28 +209,75 @@ class AIClassifier(models.Model):
return json.loads(self.classifier_data.read()) # pylint:disable=E1101 return json.loads(self.classifier_data.read()) # pylint:disable=E1101
class AITrainingWorkflow(models.Model): class AIWorkflow(models.Model):
""" """
Used to track all training tasks. Abstract base class for AI workflow database models.
Training tasks take as input an algorithm ID and a set of training examples
(which are associated with a rubric).
On successful completion, training tasks output a set of trained classifiers.
""" """
class Meta: class Meta:
app_label = "assessment" app_label = "assessment"
abstract = True
# Unique identifier used to track this workflow # Unique identifier used to track this workflow
uuid = UUIDField(version=1, db_index=True) uuid = UUIDField(version=1, db_index=True)
# Timestamps
# The task is *scheduled* as soon as a client asks the API to
# train classifiers.
# The task is *completed* when a worker has successfully created a
# classifier set based on the training examples.
scheduled_at = models.DateTimeField(default=now, db_index=True)
completed_at = models.DateTimeField(null=True, db_index=True)
# The ID of the algorithm used to train the classifiers # The ID of the algorithm used to train the classifiers
# This is a parameter passed to and interpreted by the workers. # This is a parameter passed to and interpreted by the workers.
# Django settings allow the users to map algorithm ID strings # Django settings allow the users to map algorithm ID strings
# to the Python code they should use to perform the training. # to the Python code they should use to perform the training.
algorithm_id = models.CharField(max_length=128, db_index=True) algorithm_id = models.CharField(max_length=128, db_index=True)
# The set of trained classifiers.
# In the training task, this field will be set when the task completes successfully.
# In the grading task, this may be set to null if no classifiers are available
# when the student submits an essay for grading.
classifier_set = models.ForeignKey(
AIClassifierSet, related_name='+',
null=True, default=None
)
@property
def is_complete(self):
"""
Check whether the workflow is complete.
Returns:
bool
"""
return self.completed_at is not None
def mark_complete_and_save(self):
"""
Mark the workflow as complete.
Returns:
None
"""
self.completed_at = now()
self.save()
class AITrainingWorkflow(AIWorkflow):
"""
Used to track AI training tasks.
Training tasks take as input an algorithm ID and a set of training examples
(which are associated with a rubric).
On successful completion, training tasks output a set of trained classifiers.
"""
class Meta:
app_label = "assessment"
# The training examples (essays + scores) used to train the classifiers. # The training examples (essays + scores) used to train the classifiers.
# This is a many-to-many field because # This is a many-to-many field because
# (a) we need multiple training examples to train a classifier, and # (a) we need multiple training examples to train a classifier, and
...@@ -227,21 +285,6 @@ class AITrainingWorkflow(models.Model): ...@@ -227,21 +285,6 @@ class AITrainingWorkflow(models.Model):
# (for example, if a training task is executed by Celery workers multiple times) # (for example, if a training task is executed by Celery workers multiple times)
training_examples = models.ManyToManyField(TrainingExample, related_name="+") training_examples = models.ManyToManyField(TrainingExample, related_name="+")
# The set of trained classifiers.
# Until the task completes successfully, this will be set to null.
classifier_set = models.ForeignKey(
AIClassifierSet, related_name='training_workflow',
null=True, default=None
)
# Timestamps
# The task is *scheduled* as soon as a client asks the API to
# train classifiers.
# The task is *completed* when a worker has successfully created a
# classifier set based on the training examples.
scheduled_at = models.DateTimeField(default=now, db_index=True)
completed_at = models.DateTimeField(null=True, db_index=True)
@classmethod @classmethod
@transaction.commit_on_success @transaction.commit_on_success
def start_workflow(cls, examples, algorithm_id): def start_workflow(cls, examples, algorithm_id):
...@@ -288,17 +331,6 @@ class AITrainingWorkflow(models.Model): ...@@ -288,17 +331,6 @@ class AITrainingWorkflow(models.Model):
else: else:
raise NoTrainingExamples(workflow_uuid=self.uuid) raise NoTrainingExamples(workflow_uuid=self.uuid)
@property
def is_complete(self):
"""
Check whether the workflow is complete (classifiers have been trained).
Returns:
bool
"""
return self.completed_at is not None
def complete(self, classifier_set): def complete(self, classifier_set):
""" """
Add a classifier set to the workflow and mark it complete. Add a classifier set to the workflow and mark it complete.
...@@ -319,5 +351,120 @@ class AITrainingWorkflow(models.Model): ...@@ -319,5 +351,120 @@ class AITrainingWorkflow(models.Model):
self.classifier_set = AIClassifierSet.create_classifier_set( self.classifier_set = AIClassifierSet.create_classifier_set(
classifier_set, self.rubric, self.algorithm_id classifier_set, self.rubric, self.algorithm_id
) )
self.completed_at = now() self.mark_complete_and_save()
self.save()
class AIGradingWorkflow(AIWorkflow):
"""
Used to track AI grading tasks.
Grading tasks take as input an essay submission
and a set of classifiers; the tasks select options
for each criterion in the rubric.
"""
class Meta:
app_label = "assessment"
# The UUID of the submission being graded
submission_uuid = models.CharField(max_length=128, db_index=True)
# The text of the essay submission to grade
# We duplicate this here to avoid having to repeatedly look up
# the submission. Since submissions are immutable, this is safe.
essay_text = models.TextField(blank=True)
# The rubric used to evaluate the submission.
# We store this so we can look for classifiers for the same rubric
# if none are available when the workflow is created.
rubric = models.ForeignKey(Rubric, related_name="+")
# The assessment produced by the AI grading algorithm
# Until the task completes successfully, this will be set to null
assessment = models.ForeignKey(
Assessment, related_name="+", null=True, default=None
)
# Identifier information associated with the student's submission
# Useful for finding workflows for a particular course/item/student
# Since submissions are immutable, and since the workflow is
# associated with one submission, it's safe to duplicate
# this information here from the submissions models.
student_id = models.CharField(max_length=40, db_index=True)
item_id = models.CharField(max_length=128, db_index=True)
course_id = models.CharField(max_length=40, db_index=True)
@classmethod
@transaction.commit_on_success
def start_workflow(cls, submission_uuid, rubric_dict, algorithm_id):
"""
Start a grading workflow.
Args:
submission_uuid (str): The UUID of the submission to grade.
rubric_dict (dict): The serialized rubric model.
algorithm_id (unicode): The ID of the algorithm to use for grading.
Returns:
AIGradingWorkflow
Raises:
SubmissionNotFoundError
SubmissionRequestError
SubmissionInternalError
InvalidRubric
DatabaseError
"""
# Retrieve info about the submission
submission = sub_api.get_submission_and_student(submission_uuid)
# Get or create the rubric
rubric = rubric_from_dict(rubric_dict)
# Retrieve the submission text
# Submissions are arbitrary JSON-blobs, which *should*
# contain a single key, "answer", containing the essay
# submission text. If not, though, assume we've been
# given the essay text directly (convenient for testing).
if isinstance(submission, dict):
essay_text = submission.get('answer')
else:
essay_text = unicode(submission)
# Create the workflow
return cls.objects.create(
submission_uuid=submission_uuid,
essay_text=essay_text,
algorithm_id=algorithm_id,
student_id=submission['student_item']['student_id'],
item_id=submission['student_item']['item_id'],
course_id=submission['student_item']['course_id'],
rubric=rubric
)
@transaction.commit_on_success
def complete(self, criterion_scores):
"""
Create an assessment with scores from the AI classifiers
and mark the workflow complete.
Args:
criterion_scores (dict): Dictionary mapping criteria names to integer scores.
Raises:
DatabaseError
"""
assessment = Assessment.objects.create(
submission_uuid=self.submission_uuid,
rubric=self.rubric,
scorer_id=self.algorithm_id,
score_type=AI_ASSESSMENT_TYPE
)
option_ids = self.rubric.options_ids_for_points(criterion_scores)
AssessmentPart.add_to_assessment(assessment, option_ids)
self.assessment = assessment
self.mark_complete_and_save()
apps/openassessment/assessment/models/base.py
...@@ -166,6 +166,48 @@ class Rubric(models.Model): ...@@ -166,6 +166,48 @@ class Rubric(models.Model):
return option_id_set return option_id_set
def options_ids_for_points(self, criterion_points):
"""
Given a mapping of selected point values, return the option IDs.
If there are multiple options with the same point value,
this will return the first one (lower order number).
Args:
criterion_points (dict): Mapping of criteria names to point values.
Returns:
list of option IDs
Raises:
InvalidOptionSelection
"""
# This is a really inefficient initial implementation
# TODO -- refactor to add caching
rubric_options = CriterionOption.objects.filter(
criterion__rubric=self
).select_related()
rubric_points_dict = defaultdict(dict)
for option in rubric_options:
if option.points not in rubric_points_dict[option.criterion.name]:
rubric_points_dict[option.criterion.name][option.points] = option.id
option_id_set = set()
for criterion_name, option_points in criterion_points.iteritems():
if (criterion_name in rubric_points_dict and
option_points in rubric_points_dict[criterion_name]
):
option_id = rubric_points_dict[criterion_name][option_points]
option_id_set.add(option_id)
else:
msg = _("{criterion} option with point value {points} not found in rubric").format(
criterion=criterion_name, points=option_points
)
raise InvalidOptionSelection(msg)
return option_id_set
class Criterion(models.Model): class Criterion(models.Model):
"""A single aspect of a submission that needs assessment. """A single aspect of a submission that needs assessment.
......
apps/openassessment/assessment/tasks.py
...@@ -4,3 +4,4 @@ so import the tasks we want the workers to implement. ...@@ -4,3 +4,4 @@ so import the tasks we want the workers to implement.
""" """
# pylint:disable=W0611 # pylint:disable=W0611
from .worker.training import train_classifiers from .worker.training import train_classifiers
from .worker.grading import grade_essay
apps/openassessment/assessment/test/test_ai.py
...@@ -7,11 +7,13 @@ import mock ...@@ -7,11 +7,13 @@ import mock
from django.db import DatabaseError from django.db import DatabaseError
from django.test.utils import override_settings from django.test.utils import override_settings
from openassessment.test_utils import CacheResetTest from openassessment.test_utils import CacheResetTest
from submissions import api as sub_api
from openassessment.assessment.api import ai as ai_api from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.models import AITrainingWorkflow from openassessment.assessment.models import AITrainingWorkflow, AIClassifierSet
from openassessment.assessment.worker.algorithm import AIAlgorithm from openassessment.assessment.worker.algorithm import AIAlgorithm
from openassessment.assessment.serializers import rubric_from_dict
from openassessment.assessment.errors import AITrainingRequestError, AITrainingInternalError from openassessment.assessment.errors import AITrainingRequestError, AITrainingInternalError
from openassessment.assessment.test.constants import RUBRIC, EXAMPLES from openassessment.assessment.test.constants import RUBRIC, EXAMPLES, STUDENT_ITEM, ANSWER
class StubAIAlgorithm(AIAlgorithm): class StubAIAlgorithm(AIAlgorithm):
...@@ -34,14 +36,25 @@ class StubAIAlgorithm(AIAlgorithm): ...@@ -34,14 +36,25 @@ class StubAIAlgorithm(AIAlgorithm):
# so we can test that the correct inputs were used # so we can test that the correct inputs were used
classifier = copy.copy(self.FAKE_CLASSIFIER) classifier = copy.copy(self.FAKE_CLASSIFIER)
classifier['examples'] = examples classifier['examples'] = examples
classifier['score_override'] = 0
return classifier return classifier
def score(self, text, classifier): def score(self, text, classifier):
""" """
Not implemented, but we need to make the abstact Stub implementation that returns whatever scores were
method concrete. provided in the serialized classifier data.
Expect `classifier` to be a dict with a single key,
"score_override" containing the score to return.
""" """
raise NotImplementedError return classifier['score_override']
ALGORITHM_ID = "test-stub"
AI_ALGORITHMS = {
ALGORITHM_ID: '{module}.StubAIAlgorithm'.format(module=__name__)
}
class AITrainingTest(CacheResetTest): class AITrainingTest(CacheResetTest):
...@@ -49,11 +62,6 @@ class AITrainingTest(CacheResetTest): ...@@ -49,11 +62,6 @@ class AITrainingTest(CacheResetTest):
Tests for AI training tasks. Tests for AI training tasks.
""" """
ALGORITHM_ID = "test-stub"
AI_ALGORITHMS = {
ALGORITHM_ID: '{module}.StubAIAlgorithm'.format(module=__name__)
}
EXPECTED_INPUT_SCORES = { EXPECTED_INPUT_SCORES = {
u'vøȼȺƀᵾłȺɍɏ': [1, 0], u'vøȼȺƀᵾłȺɍɏ': [1, 0],
u'ﻭɼค๓๓คɼ': [0, 2] u'ﻭɼค๓๓คɼ': [0, 2]
...@@ -65,7 +73,7 @@ class AITrainingTest(CacheResetTest): ...@@ -65,7 +73,7 @@ class AITrainingTest(CacheResetTest):
# Schedule a training task # Schedule a training task
# Because Celery is configured in "always eager" mode, # Because Celery is configured in "always eager" mode,
# expect the task to be executed synchronously. # expect the task to be executed synchronously.
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, self.ALGORITHM_ID) workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
# Retrieve the classifier set from the database # Retrieve the classifier set from the database
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid) workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
...@@ -106,12 +114,12 @@ class AITrainingTest(CacheResetTest): ...@@ -106,12 +114,12 @@ class AITrainingTest(CacheResetTest):
# Expect a request error # Expect a request error
with self.assertRaises(AITrainingRequestError): with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, mutated_examples, self.ALGORITHM_ID) ai_api.train_classifiers(RUBRIC, mutated_examples, ALGORITHM_ID)
def test_train_classifiers_no_examples(self): def test_train_classifiers_no_examples(self):
# Empty list of training examples # Empty list of training examples
with self.assertRaises(AITrainingRequestError): with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, [], self.ALGORITHM_ID) ai_api.train_classifiers(RUBRIC, [], ALGORITHM_ID)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS) @override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch.object(AITrainingWorkflow.objects, 'create') @mock.patch.object(AITrainingWorkflow.objects, 'create')
...@@ -119,7 +127,7 @@ class AITrainingTest(CacheResetTest): ...@@ -119,7 +127,7 @@ class AITrainingTest(CacheResetTest):
# Simulate a database error when creating the training workflow # Simulate a database error when creating the training workflow
mock_create.side_effect = DatabaseError("KABOOM!") mock_create.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError): with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, self.ALGORITHM_ID) ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS) @override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.api.ai.training_tasks') @mock.patch('openassessment.assessment.api.ai.training_tasks')
...@@ -127,4 +135,47 @@ class AITrainingTest(CacheResetTest): ...@@ -127,4 +135,47 @@ class AITrainingTest(CacheResetTest):
# Simulate an exception raised when scheduling a training task # Simulate an exception raised when scheduling a training task
mock_training_tasks.train_classifiers.apply_async.side_effect = Exception("KABOOM!") mock_training_tasks.train_classifiers.apply_async.side_effect = Exception("KABOOM!")
with self.assertRaises(AITrainingInternalError): with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, self.ALGORITHM_ID) ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
class AIGradingTest(CacheResetTest):
"""
Tests for AI grading tasks.
"""
CLASSIFIER_SCORE_OVERRIDES = {
u"vøȼȺƀᵾłȺɍɏ": {'score_override': 1},
u"ﻭɼค๓๓คɼ": {'score_override': 2}
}
def setUp(self):
"""
Create a submission and a fake classifier set.
"""
# Create a submission
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
# Create the classifier set for our fake AI algorithm
# To isolate these tests from the tests for the training
# task, we use the database models directly.
# We also use a stub AI algorithm that simply returns
# whatever scores we specify in the classifier data.
rubric = rubric_from_dict(RUBRIC)
AIClassifierSet.create_classifier_set(
self.CLASSIFIER_SCORE_OVERRIDES, rubric, ALGORITHM_ID
)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_grade_essay(self):
# Schedule a grading task
# Because Celery is configured in "always eager" mode, this will
# be executed synchronously.
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
# Verify that we got the scores we provided to the stub AI algorithm
assessment = ai_api.get_latest_assessment(self.submission_uuid)
for part in assessment['parts']:
criterion_name = part['option']['criterion']['name']
expected_score = self.CLASSIFIER_SCORE_OVERRIDES[criterion_name]['score_override']
self.assertEqual(part['option']['points'], expected_score)
apps/openassessment/assessment/test/test_ai_worker.py
...@@ -45,22 +45,8 @@ class AIWorkerTrainingTest(CacheResetTest): ...@@ -45,22 +45,8 @@ class AIWorkerTrainingTest(CacheResetTest):
workflow = AITrainingWorkflow.start_workflow(examples, self.ALGORITHM_ID) workflow = AITrainingWorkflow.start_workflow(examples, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid self.workflow_uuid = workflow.uuid
def test_get_algorithm_id(self): def test_get_training_task_params(self):
algorithm_id = ai_worker_api.get_algorithm_id(self.workflow_uuid) params = ai_worker_api.get_training_task_params(self.workflow_uuid)
self.assertEqual(algorithm_id, self.ALGORITHM_ID)
def test_get_algorithm_id_no_workflow(self):
with self.assertRaises(AITrainingRequestError):
ai_worker_api.get_algorithm_id("invalid_uuid")
@mock.patch.object(AITrainingWorkflow.objects, 'get')
def test_get_algorithm_id_database_error(self, mock_get):
mock_get.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_worker_api.get_algorithm_id(self.workflow_uuid)
def test_get_training_examples(self):
examples = ai_worker_api.get_training_examples(self.workflow_uuid)
expected_examples = [ expected_examples = [
{ {
'text': EXAMPLES[0]['answer'], 'text': EXAMPLES[0]['answer'],
...@@ -77,17 +63,18 @@ class AIWorkerTrainingTest(CacheResetTest): ...@@ -77,17 +63,18 @@ class AIWorkerTrainingTest(CacheResetTest):
} }
}, },
] ]
self.assertItemsEqual(examples, expected_examples) self.assertItemsEqual(params['training_examples'], expected_examples)
self.assertItemsEqual(params['algorithm_id'], self.ALGORITHM_ID)
def test_get_training_examples_no_workflow(self): def test_get_training_task_params_no_workflow(self):
with self.assertRaises(AITrainingRequestError): with self.assertRaises(AITrainingRequestError):
ai_worker_api.get_training_examples("invalid_uuid") ai_worker_api.get_training_task_params("invalid_uuid")
@mock.patch.object(AITrainingWorkflow.objects, 'get') @mock.patch.object(AITrainingWorkflow.objects, 'get')
def test_get_training_examples_database_error(self, mock_get): def test_get_training_task_params_database_error(self, mock_get):
mock_get.side_effect = DatabaseError("KABOOM!") mock_get.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError): with self.assertRaises(AITrainingInternalError):
ai_worker_api.get_training_examples(self.workflow_uuid) ai_worker_api.get_training_task_params(self.workflow_uuid)
def test_create_classifiers(self): def test_create_classifiers(self):
ai_worker_api.create_classifiers(self.workflow_uuid, self.CLASSIFIERS) ai_worker_api.create_classifiers(self.workflow_uuid, self.CLASSIFIERS)
......
apps/openassessment/assessment/test/test_worker.py
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
Tests for AI worker tasks. Tests for AI worker tasks.
""" """
from contextlib import contextmanager from contextlib import contextmanager
import datetime
import mock import mock
from django.test.utils import override_settings from django.test.utils import override_settings
from openassessment.test_utils import CacheResetTest from openassessment.test_utils import CacheResetTest
...@@ -64,13 +63,6 @@ class AITrainingTaskTest(CacheResetTest): ...@@ -64,13 +63,6 @@ class AITrainingTaskTest(CacheResetTest):
workflow = AITrainingWorkflow.start_workflow(examples, self.ALGORITHM_ID) workflow = AITrainingWorkflow.start_workflow(examples, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid self.workflow_uuid = workflow.uuid
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.worker.training.ai_worker_api.get_algorithm_id')
def test_get_algorithm_id_api_error(self, mock_call):
mock_call.side_effect = AITrainingRequestError("Test error!")
with self._assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers(self.workflow_uuid)
def test_unknown_algorithm(self): def test_unknown_algorithm(self):
# Since we haven't overridden settings to configure the algorithms, # Since we haven't overridden settings to configure the algorithms,
# the worker will not recognize the workflow's algorithm ID. # the worker will not recognize the workflow's algorithm ID.
...@@ -92,8 +84,8 @@ class AITrainingTaskTest(CacheResetTest): ...@@ -92,8 +84,8 @@ class AITrainingTaskTest(CacheResetTest):
train_classifiers(self.workflow_uuid) train_classifiers(self.workflow_uuid)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS) @override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.worker.training.ai_worker_api.get_training_examples') @mock.patch('openassessment.assessment.worker.training.ai_worker_api.get_training_task_params')
def test_get_training_examples_api_error(self, mock_call): def test_get_training_task_params_api_error(self, mock_call):
mock_call.side_effect = AITrainingRequestError("Test error!") mock_call.side_effect = AITrainingRequestError("Test error!")
with self._assert_retry(train_classifiers, AITrainingRequestError): with self._assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers(self.workflow_uuid) train_classifiers(self.workflow_uuid)
...@@ -160,12 +152,12 @@ class AITrainingTaskTest(CacheResetTest): ...@@ -160,12 +152,12 @@ class AITrainingTaskTest(CacheResetTest):
AssertionError AssertionError
""" """
examples = ai_worker_api.get_training_examples(self.workflow_uuid) params = ai_worker_api.get_training_task_params(self.workflow_uuid)
mutate_func(examples) mutate_func(params['training_examples'])
call_signature = 'openassessment.assessment.worker.training.ai_worker_api.get_training_examples' call_signature = 'openassessment.assessment.worker.training.ai_worker_api.get_training_task_params'
with mock.patch(call_signature) as mock_call: with mock.patch(call_signature) as mock_call:
mock_call.return_value = examples mock_call.return_value = params
with self._assert_retry(train_classifiers, InvalidExample): with self._assert_retry(train_classifiers, InvalidExample):
train_classifiers(self.workflow_uuid) train_classifiers(self.workflow_uuid)
......
apps/openassessment/assessment/worker/grading.py 0 → 100644
"""
Asynchronous tasks for grading essays using text classifiers.
"""
from celery import task
from celery.utils.log import get_task_logger
from openassessment.assessment.api import ai_worker as ai_worker_api
from openassessment.assessment.errors import AIError
from .algorithm import AIAlgorithm, AIAlgorithmError
MAX_RETRIES = 2
logger = get_task_logger(__name__)
@task(max_retries=MAX_RETRIES) # pylint: disable=E1102
def grade_essay(workflow_uuid):
"""
Asynchronous task to grade an essay using a text classifier
(trained using a supervised ML algorithm).
If the task could not be completed successfully,
it will be retried a few times; if it continues to fail,
it is left incomplete. Incomplate tasks can be rescheduled
manually through the AI API.
Args:
workflow_uuid (str): The UUID of the workflow associated
with this grading task.
Returns:
None
Raises:
AIError: An error occurred while making an AI worker API call.
AIAlgorithmError: An error occurred while retrieving or using an AI algorithm.
"""
# Retrieve the task parameters
try:
params = ai_worker_api.get_grading_task_params(workflow_uuid)
essay_text = params['essay_text']
classifier_set = params['classifier_set']
algorithm_id = params['algorithm_id']
except (AIError, KeyError):
msg = (
u"An error occurred while retrieving the AI grading task "
u"parameters for the workflow with UUID {}"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Retrieve the AI algorithm
try:
algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
except AIAlgorithmError:
msg = (
u"An error occurred while retrieving "
u"the algorithm ID (grading workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Use the algorithm to evaluate the essay for each criterion
try:
scores_by_criterion = {
criterion_name: algorithm.score(essay_text, classifier)
for criterion_name, classifier in classifier_set.iteritems()
}
except AIAlgorithmError:
msg = (
u"An error occurred while scoring essays using "
u"an AI algorithm (worker workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Create the assessment and mark the workflow complete
try:
ai_worker_api.create_assessment(workflow_uuid, scores_by_criterion)
except AIError:
msg = (
u"An error occurred while creating assessments "
u"for the AI grading workflow with UUID {uuid}. "
u"The assessment scores were: {scores}"
).format(uuid=workflow_uuid, scores=scores_by_criterion)
logger.exception(msg)
raise grade_essay.retry()
apps/openassessment/assessment/worker/training.py
...@@ -53,10 +53,22 @@ def train_classifiers(workflow_uuid): ...@@ -53,10 +53,22 @@ def train_classifiers(workflow_uuid):
InvalidExample: The training examples provided by the AI API were not valid. InvalidExample: The training examples provided by the AI API were not valid.
""" """
# Retrieve task parameters
try:
params = ai_worker_api.get_training_task_params(workflow_uuid)
examples = params['training_examples']
algorithm_id = params['algorithm_id']
except (AIError, KeyError):
msg = (
u"An error occurred while retrieving AI training "
u"task parameters for the workflow with UUID {}"
).format(workflow_uuid)
logger.exception(msg)
raise train_classifiers.retry()
# Retrieve the ML algorithm to use for training # Retrieve the ML algorithm to use for training
# (based on task params and worker configuration) # (based on task params and worker configuration)
try: try:
algorithm_id = ai_worker_api.get_algorithm_id(workflow_uuid)
algorithm = AIAlgorithm.algorithm_for_id(algorithm_id) algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
except AIAlgorithmError: except AIAlgorithmError:
msg = ( msg = (
...@@ -73,19 +85,6 @@ def train_classifiers(workflow_uuid): ...@@ -73,19 +85,6 @@ def train_classifiers(workflow_uuid):
logger.exception(msg) logger.exception(msg)
raise train_classifiers.retry() raise train_classifiers.retry()
# Retrieve training examples, then transform them into the
# data structures we use internally.
try:
examples = ai_worker_api.get_training_examples(workflow_uuid)
except AIError:
msg = (
u"An error occurred while retrieving "
u"training examples for AI training "
u"(training workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise train_classifiers.retry()
# Train a classifier for each criterion # Train a classifier for each criterion
# The AIAlgorithm subclass is responsible for ensuring that # The AIAlgorithm subclass is responsible for ensuring that
# the trained classifiers are JSON-serializable. # the trained classifiers are JSON-serializable.
......
docs/en_us/developers/source/architecture/ai_grading.rst
...@@ -240,17 +240,15 @@ Data Model ...@@ -240,17 +240,15 @@ Data Model
1. **GradingWorkflow** 1. **GradingWorkflow**
a. Submission UUID (varchar) a. Submission UUID (varchar)
b. Rubric UUID (varchar) b. ClassifierSet (Foreign Key, Nullable)
c. ClassifierSet (Foreign Key, Nullable) c. Assessment (Foreign Key, Nullable)
d. Assessment (Foreign Key, Nullable) d. Rubric (Foreign Key): Used to search for classifier sets if none are available when the workflow is started.
e. Scheduled at (timestamp): The time the task was placed on the queue. e. Algorithm ID (varchar): Used to search for classifier sets if none are available when the workflow is started.
f. Started at (timestamp): The time the task was picked up by the worker. f. Scheduled at (timestamp): The time the task was placed on the queue.
g. Completed at (timestamp): The time the task was completed. If set, the task is considered complete. g. Completed at (timestamp): The time the task was completed. If set, the task is considered complete.
h. Course ID (varchar): The ID of the course associated with the submission. Useful for rescheduling h. Course ID (varchar): The ID of the course associated with the submission. Useful for rescheduling failed grading tasks in a particular course.
failed grading tasks in a particular course. i. Item ID (varchar): The ID of the item (problem) associated with the submission. Useful for rescheduling failed grading tasks in a particular item in a course.
i. Item ID (varchar): The ID of the item (problem) associated with the submission. Useful for rescheduling
failed grading tasks in a particular item in a course.
j. Worker version (varchar): Identifier for the code running on the worker when the task was started. Useful for error tracking.
2. **TrainingWorkflow** 2. **TrainingWorkflow**
...@@ -269,13 +267,13 @@ Data Model ...@@ -269,13 +267,13 @@ Data Model
a. Rubric (Foreign Key) a. Rubric (Foreign Key)
b. Created at (timestamp) b. Created at (timestamp)
c. Algorithm ID (varchar)
5. **Classifier** 5. **Classifier**
a. ClassifierSet (Foreign Key) a. ClassifierSet (Foreign Key)
b. URL for trained classifier (varchar) b. URL for trained classifier (varchar)
c. Algorithm ID (varchar) c. Criterion (Foreign Key)
d. Criterion (Foreign Key)
6. **Assessment** (same as current implementation) 6. **Assessment** (same as current implementation)
......
scripts/test.sh
...@@ -4,5 +4,5 @@ ...@@ -4,5 +4,5 @@
set -e set -e
cd `dirname $BASH_SOURCE` && cd .. cd `dirname $BASH_SOURCE` && cd ..
./scripts/test-python.sh ./scripts/test-python.sh $1
./scripts/test-js.sh ./scripts/test-js.sh
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment