Skip to content

E
edx-ora2
Commit 8d6bb67d by Will Daly
Options

Merge pull request #336 from edx/will/uniqueness-and-integrity 

Uniqueness constraints for student training
parents 2b224a74 d8bcde15
Showing with 311 additions and 106 deletions
apps/openassessment/assessment/migrations/0009_auto__add_unique_studenttrainingworkflowitem_order_num_workflow.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding unique constraint on 'StudentTrainingWorkflowItem', fields ['order_num', 'workflow']
db.create_unique('assessment_studenttrainingworkflowitem', ['order_num', 'workflow_id'])
def backwards(self, orm):
# Removing unique constraint on 'StudentTrainingWorkflowItem', fields ['order_num', 'workflow']
db.delete_unique('assessment_studenttrainingworkflowitem', ['order_num', 'workflow_id'])
models = {
'assessment.assessment': {
'Meta': {'ordering': "['-scored_at', '-id']", 'object_name': 'Assessment'},
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"}),
'score_type': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
'scored_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'scorer_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedback': {
'Meta': {'object_name': 'AssessmentFeedback'},
'assessments': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.Assessment']"}),
'feedback_text': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.AssessmentFeedbackOption']"}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedbackoption': {
'Meta': {'object_name': 'AssessmentFeedbackOption'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'text': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'assessment.assessmentpart': {
'Meta': {'object_name': 'AssessmentPart'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parts'", 'to': "orm['assessment.Assessment']"}),
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'option': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.CriterionOption']"})
},
'assessment.criterion': {
'Meta': {'ordering': "['rubric', 'order_num']", 'object_name': 'Criterion'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'prompt': ('django.db.models.fields.TextField', [], {'max_length': '10000'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'criteria'", 'to': "orm['assessment.Rubric']"})
},
'assessment.criterionoption': {
'Meta': {'ordering': "['criterion', 'order_num']", 'object_name': 'CriterionOption'},
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['assessment.Criterion']"}),
'explanation': ('django.db.models.fields.TextField', [], {'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'points': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'assessment.peerworkflow': {
'Meta': {'ordering': "['created_at', 'id']", 'object_name': 'PeerWorkflow'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'grading_completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.peerworkflowitem': {
'Meta': {'ordering': "['started_at', 'id']", 'object_name': 'PeerWorkflowItem'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Assessment']", 'null': 'True'}),
'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded_by'", 'to': "orm['assessment.PeerWorkflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scored': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'scorer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded'", 'to': "orm['assessment.PeerWorkflow']"}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.rubric': {
'Meta': {'object_name': 'Rubric'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.studenttrainingworkflow': {
'Meta': {'object_name': 'StudentTrainingWorkflow'},
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.studenttrainingworkflowitem': {
'Meta': {'ordering': "['workflow', 'order_num']", 'unique_together': "(('workflow', 'order_num'),)", 'object_name': 'StudentTrainingWorkflowItem'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'training_example': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.TrainingExample']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['assessment.StudentTrainingWorkflow']"})
},
'assessment.trainingexample': {
'Meta': {'object_name': 'TrainingExample'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options_selected': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['assessment.CriterionOption']", 'symmetrical': 'False'}),
'raw_answer': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"})
}
}
complete_apps = ['assessment']
\ No newline at end of file
apps/openassessment/assessment/models/student_training.py
""" """
Django models specific to the student training assessment type. Django models specific to the student training assessment type.
""" """
from django.db import models from django.db import models, IntegrityError
from django.utils import timezone from django.utils import timezone
from submissions import api as sub_api from submissions import api as sub_api
from .training import TrainingExample from .training import TrainingExample
...@@ -53,12 +53,18 @@ class StudentTrainingWorkflow(models.Model): ...@@ -53,12 +53,18 @@ class StudentTrainingWorkflow(models.Model):
student_item = submission['student_item'] student_item = submission['student_item']
# Create the workflow # Create the workflow
try:
return cls.objects.create( return cls.objects.create(
submission_uuid=submission_uuid, submission_uuid=submission_uuid,
student_id=student_item['student_id'], student_id=student_item['student_id'],
item_id=student_item['item_id'], item_id=student_item['item_id'],
course_id=student_item['course_id'] course_id=student_item['course_id']
) )
# If we get an integrity error, it means we've violated a uniqueness constraint
# (someone has created this object after we checked if it existed)
# We can therefore assume that the object exists and we can retrieve it.
except IntegrityError:
return cls.objects.get(submission_uuid=submission_uuid)
@property @property
def num_completed(self): def num_completed(self):
...@@ -116,11 +122,24 @@ class StudentTrainingWorkflow(models.Model): ...@@ -116,11 +122,24 @@ class StudentTrainingWorkflow(models.Model):
else: else:
order_num = len(items) + 1 order_num = len(items) + 1
next_example = available_examples[0] next_example = available_examples[0]
try:
StudentTrainingWorkflowItem.objects.create( StudentTrainingWorkflowItem.objects.create(
workflow=self, workflow=self,
order_num=order_num, order_num=order_num,
training_example=next_example training_example=next_example
) )
# If we get an integrity error, it means we've violated a uniqueness constraint
# (someone has created this object after we checked if it existed)
# Since the object already exists, we don't need to do anything
# However, the example might not be the one we intended to use, so
# we need to retrieve the actual training example.
except IntegrityError:
workflow = StudentTrainingWorkflowItem.objects.get(
workflow=self, order_num=order_num
)
return workflow.training_example
else:
return next_example return next_example
@property @property
...@@ -161,6 +180,7 @@ class StudentTrainingWorkflowItem(models.Model): ...@@ -161,6 +180,7 @@ class StudentTrainingWorkflowItem(models.Model):
class Meta: class Meta:
app_label = "assessment" app_label = "assessment"
ordering = ["workflow", "order_num"] ordering = ["workflow", "order_num"]
unique_together = ('workflow', 'order_num')
@property @property
def is_complete(self): def is_complete(self):
......
apps/openassessment/assessment/test/constants.py 0 → 100644
# -*- coding: utf-8 -*-
"""
Constants used as test data.
"""
STUDENT_ITEM = {
'student_id': u'𝓽𝓮𝓼𝓽 𝓼𝓽𝓾𝓭𝓮𝓷𝓽',
'item_id': u'𝖙𝖊𝖘𝖙 𝖎𝖙𝖊𝖒',
'course_id': u'ՇєรՇ ς๏ยгรє',
'item_type': u'openassessment'
}
ANSWER = u'ẗëṡẗ äṅṡẅëṛ'
RUBRIC_OPTIONS = [
{
"order_num": 0,
"name": u"𝒑𝒐𝒐𝒓",
"explanation": u"𝕻𝖔𝖔𝖗 𝖏𝖔𝖇!",
"points": 0,
},
{
"order_num": 1,
"name": u"𝓰𝓸𝓸𝓭",
"explanation": u"ﻭѻѻɗ ﻝѻ๒!",
"points": 1,
},
{
"order_num": 2,
"name": u"єχ¢єℓℓєηт",
"explanation": u"乇ﾒc乇ﾚﾚ乇刀ｲ ﾌo乃!",
"points": 2,
},
]
RUBRIC = {
'prompt': u"МоъЎ-ↁіск; оѓ, ГЂэ ЩЂаlэ",
'criteria': [
{
"order_num": 0,
"name": u"vøȼȺƀᵾłȺɍɏ",
"prompt": u"Ħøw vȺɍɨɇđ ɨs ŧħɇ vøȼȺƀᵾłȺɍɏ?",
"options": RUBRIC_OPTIONS
},
{
"order_num": 1,
"name": u"ﻭɼค๓๓คɼ",
"prompt": u"𝕳𝖔𝖜 𝖈𝖔𝖗𝖗𝖊𝖈𝖙 𝖎𝖘 𝖙𝖍𝖊 𝖌𝖗𝖆𝖒𝖒𝖆𝖗?",
"options": RUBRIC_OPTIONS
}
]
}
EXAMPLES = [
{
'answer': (
u"𝕿𝖍𝖊𝖗𝖊 𝖆𝖗𝖊 𝖈𝖊𝖗𝖙𝖆𝖎𝖓 𝖖𝖚𝖊𝖊𝖗 𝖙𝖎𝖒𝖊𝖘 𝖆𝖓𝖉 𝖔𝖈𝖈𝖆𝖘𝖎𝖔𝖓𝖘 𝖎𝖓 𝖙𝖍𝖎𝖘 𝖘𝖙𝖗𝖆𝖓𝖌𝖊 𝖒𝖎𝖝𝖊𝖉 𝖆𝖋𝖋𝖆𝖎𝖗 𝖜𝖊 𝖈𝖆𝖑𝖑 𝖑𝖎𝖋𝖊"
u" 𝖜𝖍𝖊𝖓 𝖆 𝖒𝖆𝖓 𝖙𝖆𝖐𝖊𝖘 𝖙𝖍𝖎𝖘 𝖜𝖍𝖔𝖑𝖊 𝖚𝖓𝖎𝖛𝖊𝖗𝖘𝖊 𝖋𝖔𝖗 𝖆 𝖛𝖆𝖘𝖙 𝖕𝖗𝖆𝖈𝖙𝖎𝖈𝖆𝖑 𝖏𝖔𝖐𝖊, 𝖙𝖍𝖔𝖚𝖌𝖍 𝖙𝖍𝖊 𝖜𝖎𝖙 𝖙𝖍𝖊𝖗𝖊𝖔𝖋"
u" 𝖍𝖊 𝖇𝖚𝖙 𝖉𝖎𝖒𝖑𝖞 𝖉𝖎𝖘𝖈𝖊𝖗𝖓𝖘, 𝖆𝖓𝖉 𝖒𝖔𝖗𝖊 𝖙𝖍𝖆𝖓 𝖘𝖚𝖘𝖕𝖊𝖈𝖙𝖘 𝖙𝖍𝖆𝖙 𝖙𝖍𝖊 𝖏𝖔𝖐𝖊 𝖎𝖘 𝖆𝖙 𝖓𝖔𝖇𝖔𝖉𝖞'𝖘 𝖊𝖝𝖕𝖊𝖓𝖘𝖊 𝖇𝖚𝖙 𝖍𝖎𝖘 𝖔𝖜𝖓."
),
'options_selected': {
u"vøȼȺƀᵾłȺɍɏ": u"𝓰𝓸𝓸𝓭",
u"ﻭɼค๓๓คɼ": u"𝒑𝒐𝒐𝒓",
}
},
{
'answer': u"Tőṕ-héávӳ ẃáś thé śhíṕ áś á díńńéŕĺéśś śtúdéńt ẃíth áĺĺ Áŕíśtőtĺé íń híś héád.",
'options_selected': {
u"vøȼȺƀᵾłȺɍɏ": u"𝒑𝒐𝒐𝒓",
u"ﻭɼค๓๓คɼ": u"єχ¢єℓℓєηт",
}
},
]
apps/openassessment/assessment/test/test_models.py
...@@ -3,10 +3,14 @@ ...@@ -3,10 +3,14 @@
Tests for assessment models. Tests for assessment models.
""" """
import mock
from django.db import IntegrityError
from openassessment.test_utils import CacheResetTest from openassessment.test_utils import CacheResetTest
from submissions import api as sub_api
from openassessment.assessment.models import ( from openassessment.assessment.models import (
Rubric, Criterion, CriterionOption, InvalidOptionSelection, Rubric, Criterion, CriterionOption, InvalidOptionSelection,
AssessmentFeedback, AssessmentFeedbackOption, AssessmentFeedback, AssessmentFeedbackOption,
StudentTrainingWorkflow
) )
......
apps/openassessment/assessment/test/test_student_training.py → apps/openassessment/assessment/test/test_student_training_api.py
...@@ -7,6 +7,7 @@ from django.db import DatabaseError ...@@ -7,6 +7,7 @@ from django.db import DatabaseError
import ddt import ddt
from mock import patch from mock import patch
from openassessment.test_utils import CacheResetTest from openassessment.test_utils import CacheResetTest
from .constants import STUDENT_ITEM, ANSWER, RUBRIC, EXAMPLES
from submissions import api as sub_api from submissions import api as sub_api
from openassessment.assessment.api import student_training as training_api from openassessment.assessment.api import student_training as training_api
from openassessment.assessment.errors import StudentTrainingRequestError, StudentTrainingInternalError from openassessment.assessment.errors import StudentTrainingRequestError, StudentTrainingInternalError
...@@ -20,80 +21,11 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -20,80 +21,11 @@ class StudentTrainingAssessmentTest(CacheResetTest):
""" """
longMessage = True longMessage = True
STUDENT_ITEM = {
'student_id': u'𝓽𝓮𝓼𝓽 𝓼𝓽𝓾𝓭𝓮𝓷𝓽',
'item_id': u'𝖙𝖊𝖘𝖙 𝖎𝖙𝖊𝖒',
'course_id': u'ՇєรՇ ς๏ยгรє',
'item_type': u'openassessment'
}
ANSWER = u'ẗëṡẗ äṅṡẅëṛ'
RUBRIC_OPTIONS = [
{
"order_num": 0,
"name": u"𝒑𝒐𝒐𝒓",
"explanation": u"𝕻𝖔𝖔𝖗 𝖏𝖔𝖇!",
"points": 0,
},
{
"order_num": 1,
"name": u"𝓰𝓸𝓸𝓭",
"explanation": u"ﻭѻѻɗ ﻝѻ๒!",
"points": 1,
},
{
"order_num": 2,
"name": u"єχ¢єℓℓєηт",
"explanation": u"乇ﾒc乇ﾚﾚ乇刀ｲ ﾌo乃!",
"points": 2,
},
]
RUBRIC = {
'prompt': u"МоъЎ-ↁіск; оѓ, ГЂэ ЩЂаlэ",
'criteria': [
{
"order_num": 0,
"name": u"vøȼȺƀᵾłȺɍɏ",
"prompt": u"Ħøw vȺɍɨɇđ ɨs ŧħɇ vøȼȺƀᵾłȺɍɏ?",
"options": RUBRIC_OPTIONS
},
{
"order_num": 1,
"name": u"ﻭɼค๓๓คɼ",
"prompt": u"𝕳𝖔𝖜 𝖈𝖔𝖗𝖗𝖊𝖈𝖙 𝖎𝖘 𝖙𝖍𝖊 𝖌𝖗𝖆𝖒𝖒𝖆𝖗?",
"options": RUBRIC_OPTIONS
}
]
}
EXAMPLES = [
{
'answer': (
u"𝕿𝖍𝖊𝖗𝖊 𝖆𝖗𝖊 𝖈𝖊𝖗𝖙𝖆𝖎𝖓 𝖖𝖚𝖊𝖊𝖗 𝖙𝖎𝖒𝖊𝖘 𝖆𝖓𝖉 𝖔𝖈𝖈𝖆𝖘𝖎𝖔𝖓𝖘 𝖎𝖓 𝖙𝖍𝖎𝖘 𝖘𝖙𝖗𝖆𝖓𝖌𝖊 𝖒𝖎𝖝𝖊𝖉 𝖆𝖋𝖋𝖆𝖎𝖗 𝖜𝖊 𝖈𝖆𝖑𝖑 𝖑𝖎𝖋𝖊"
u" 𝖜𝖍𝖊𝖓 𝖆 𝖒𝖆𝖓 𝖙𝖆𝖐𝖊𝖘 𝖙𝖍𝖎𝖘 𝖜𝖍𝖔𝖑𝖊 𝖚𝖓𝖎𝖛𝖊𝖗𝖘𝖊 𝖋𝖔𝖗 𝖆 𝖛𝖆𝖘𝖙 𝖕𝖗𝖆𝖈𝖙𝖎𝖈𝖆𝖑 𝖏𝖔𝖐𝖊, 𝖙𝖍𝖔𝖚𝖌𝖍 𝖙𝖍𝖊 𝖜𝖎𝖙 𝖙𝖍𝖊𝖗𝖊𝖔𝖋"
u" 𝖍𝖊 𝖇𝖚𝖙 𝖉𝖎𝖒𝖑𝖞 𝖉𝖎𝖘𝖈𝖊𝖗𝖓𝖘, 𝖆𝖓𝖉 𝖒𝖔𝖗𝖊 𝖙𝖍𝖆𝖓 𝖘𝖚𝖘𝖕𝖊𝖈𝖙𝖘 𝖙𝖍𝖆𝖙 𝖙𝖍𝖊 𝖏𝖔𝖐𝖊 𝖎𝖘 𝖆𝖙 𝖓𝖔𝖇𝖔𝖉𝖞'𝖘 𝖊𝖝𝖕𝖊𝖓𝖘𝖊 𝖇𝖚𝖙 𝖍𝖎𝖘 𝖔𝖜𝖓."
),
'options_selected': {
u"vøȼȺƀᵾłȺɍɏ": u"𝓰𝓸𝓸𝓭",
u"ﻭɼค๓๓คɼ": u"𝒑𝒐𝒐𝒓",
}
},
{
'answer': u"Tőṕ-héávӳ ẃáś thé śhíṕ áś á díńńéŕĺéśś śtúdéńt ẃíth áĺĺ Áŕíśtőtĺé íń híś héád.",
'options_selected': {
u"vøȼȺƀᵾłȺɍɏ": u"𝒑𝒐𝒐𝒓",
u"ﻭɼค๓๓คɼ": u"єχ¢єℓℓєηт",
}
},
]
def setUp(self): def setUp(self):
""" """
Create a submission. Create a submission.
""" """
submission = sub_api.create_submission(self.STUDENT_ITEM, self.ANSWER) submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid'] self.submission_uuid = submission['uuid']
def test_training_workflow(self): def test_training_workflow(self):
...@@ -101,18 +33,18 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -101,18 +33,18 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self._assert_workflow_status(self.submission_uuid, 0, 2) self._assert_workflow_status(self.submission_uuid, 0, 2)
# Get a training example # Get a training example
self._assert_get_example(self.submission_uuid, 0, self.EXAMPLES, self.RUBRIC) self._assert_get_example(self.submission_uuid, 0, EXAMPLES, RUBRIC)
# Assess the training example the same way the instructor did # Assess the training example the same way the instructor did
corrections = training_api.assess_training_example( corrections = training_api.assess_training_example(
self.submission_uuid, self.submission_uuid,
self.EXAMPLES[0]['options_selected'] EXAMPLES[0]['options_selected']
) )
self.assertEqual(corrections, dict()) self.assertEqual(corrections, dict())
self._assert_workflow_status(self.submission_uuid, 1, 2) self._assert_workflow_status(self.submission_uuid, 1, 2)
# Get another training example to assess # Get another training example to assess
self._assert_get_example(self.submission_uuid, 1, self.EXAMPLES, self.RUBRIC) self._assert_get_example(self.submission_uuid, 1, EXAMPLES, RUBRIC)
# Give the example different scores than the instructor gave # Give the example different scores than the instructor gave
incorrect_assessment = { incorrect_assessment = {
...@@ -124,12 +56,12 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -124,12 +56,12 @@ class StudentTrainingAssessmentTest(CacheResetTest):
) )
# Expect that we get corrected and stay on the current example # Expect that we get corrected and stay on the current example
self.assertItemsEqual(corrections, self.EXAMPLES[1]['options_selected']) self.assertItemsEqual(corrections, EXAMPLES[1]['options_selected'])
self._assert_workflow_status(self.submission_uuid, 1, 2) self._assert_workflow_status(self.submission_uuid, 1, 2)
# Try again, and this time assess the same way as the instructor # Try again, and this time assess the same way as the instructor
corrections = training_api.assess_training_example( corrections = training_api.assess_training_example(
self.submission_uuid, self.EXAMPLES[1]['options_selected'] self.submission_uuid, EXAMPLES[1]['options_selected']
) )
self.assertEqual(corrections, dict()) self.assertEqual(corrections, dict())
...@@ -139,10 +71,10 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -139,10 +71,10 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_assess_without_update(self): def test_assess_without_update(self):
# Assess the first training example the same way the instructor did # Assess the first training example the same way the instructor did
# but do NOT update the workflow # but do NOT update the workflow
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
corrections = training_api.assess_training_example( corrections = training_api.assess_training_example(
self.submission_uuid, self.submission_uuid,
self.EXAMPLES[0]['options_selected'], EXAMPLES[0]['options_selected'],
update_workflow=False update_workflow=False
) )
...@@ -152,11 +84,11 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -152,11 +84,11 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_get_same_example(self): def test_get_same_example(self):
# Retrieve a training example # Retrieve a training example
retrieved = training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) retrieved = training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
# If we retrieve an example without completing the current example, # If we retrieve an example without completing the current example,
# we should get the same one. # we should get the same one.
next_retrieved = training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) next_retrieved = training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
self.assertEqual(retrieved, next_retrieved) self.assertEqual(retrieved, next_retrieved)
def test_get_training_example_num_queries(self): def test_get_training_example_num_queries(self):
...@@ -164,33 +96,33 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -164,33 +96,33 @@ class StudentTrainingAssessmentTest(CacheResetTest):
# Run through the training example once using a different submission # Run through the training example once using a different submission
# Training examples and rubrics will be cached and shared for other # Training examples and rubrics will be cached and shared for other
# students working on the same problem. # students working on the same problem.
self._warm_cache(self.RUBRIC, self.EXAMPLES) self._warm_cache(RUBRIC, EXAMPLES)
# First training example # First training example
# This will need to create the student training workflow and the first item # This will need to create the student training workflow and the first item
# NOTE: we *could* cache the rubric model to reduce the number of queries here, # NOTE: we *could* cache the rubric model to reduce the number of queries here,
# but we're selecting it by content hash, which is indexed and should be plenty fast. # but we're selecting it by content hash, which is indexed and should be plenty fast.
with self.assertNumQueries(6): with self.assertNumQueries(6):
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
# Without assessing the first training example, try to retrieve a training example. # Without assessing the first training example, try to retrieve a training example.
# This should return the same example as before, so we won't need to create # This should return the same example as before, so we won't need to create
# any workflows or workflow items. # any workflows or workflow items.
with self.assertNumQueries(3): with self.assertNumQueries(3):
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
# Assess the current training example # Assess the current training example
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected']) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
# Retrieve the next training example, which requires us to create # Retrieve the next training example, which requires us to create
# a new workflow item (but not a new workflow). # a new workflow item (but not a new workflow).
with self.assertNumQueries(4): with self.assertNumQueries(4):
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
def test_submitter_is_finished_num_queries(self): def test_submitter_is_finished_num_queries(self):
# Complete the first training example # Complete the first training example
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected']) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
# Check whether we've completed the requirements # Check whether we've completed the requirements
requirements = {'num_required': 2} requirements = {'num_required': 2}
...@@ -199,8 +131,8 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -199,8 +131,8 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_get_num_completed_num_queries(self): def test_get_num_completed_num_queries(self):
# Complete the first training example # Complete the first training example
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected']) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
# Check the number completed # Check the number completed
with self.assertNumQueries(2): with self.assertNumQueries(2):
...@@ -208,10 +140,10 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -208,10 +140,10 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_assess_training_example_num_queries(self): def test_assess_training_example_num_queries(self):
# Populate the cache with training examples and rubrics # Populate the cache with training examples and rubrics
self._warm_cache(self.RUBRIC, self.EXAMPLES) self._warm_cache(RUBRIC, EXAMPLES)
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
with self.assertNumQueries(4): with self.assertNumQueries(4):
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected']) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
@ddt.file_data('data/validate_training_examples.json') @ddt.file_data('data/validate_training_examples.json')
def test_validate_training_examples(self, data): def test_validate_training_examples(self, data):
...@@ -230,8 +162,8 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -230,8 +162,8 @@ class StudentTrainingAssessmentTest(CacheResetTest):
self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, requirements)) self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, requirements))
def test_get_training_example_none_available(self): def test_get_training_example_none_available(self):
for example in self.EXAMPLES: for example in EXAMPLES:
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
training_api.assess_training_example(self.submission_uuid, example['options_selected']) training_api.assess_training_example(self.submission_uuid, example['options_selected'])
# Now we should be complete # Now we should be complete
...@@ -239,19 +171,19 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -239,19 +171,19 @@ class StudentTrainingAssessmentTest(CacheResetTest):
# ... and if we try to get another example, we should get None # ... and if we try to get another example, we should get None
self.assertIs( self.assertIs(
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES), training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES),
None None
) )
def test_assess_training_example_completed_workflow(self): def test_assess_training_example_completed_workflow(self):
for example in self.EXAMPLES: for example in EXAMPLES:
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
training_api.assess_training_example(self.submission_uuid, example['options_selected']) training_api.assess_training_example(self.submission_uuid, example['options_selected'])
# Try to assess again, and expect an error # Try to assess again, and expect an error
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.assess_training_example( training_api.assess_training_example(
self.submission_uuid, self.EXAMPLES[0]['options_selected'] self.submission_uuid, EXAMPLES[0]['options_selected']
) )
def test_assess_training_example_no_workflow(self): def test_assess_training_example_no_workflow(self):
...@@ -260,7 +192,7 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -260,7 +192,7 @@ class StudentTrainingAssessmentTest(CacheResetTest):
# then we should get a request error. # then we should get a request error.
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.assess_training_example( training_api.assess_training_example(
self.submission_uuid, self.EXAMPLES[0]['options_selected'] self.submission_uuid, EXAMPLES[0]['options_selected']
) )
def test_get_num_completed_no_workflow(self): def test_get_num_completed_no_workflow(self):
...@@ -269,15 +201,15 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -269,15 +201,15 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_get_training_example_invalid_rubric(self): def test_get_training_example_invalid_rubric(self):
# Rubric is missing a very important key! # Rubric is missing a very important key!
invalid_rubric = copy.deepcopy(self.RUBRIC) invalid_rubric = copy.deepcopy(RUBRIC)
del invalid_rubric['criteria'] del invalid_rubric['criteria']
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.get_training_example(self.submission_uuid, invalid_rubric, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, invalid_rubric, EXAMPLES)
def test_get_training_example_no_submission(self): def test_get_training_example_no_submission(self):
with self.assertRaises(StudentTrainingRequestError): with self.assertRaises(StudentTrainingRequestError):
training_api.get_training_example("no_such_submission", self.RUBRIC, self.EXAMPLES) training_api.get_training_example("no_such_submission", RUBRIC, EXAMPLES)
@patch.object(StudentTrainingWorkflow.objects, 'get') @patch.object(StudentTrainingWorkflow.objects, 'get')
def test_get_num_completed_database_error(self, mock_db): def test_get_num_completed_database_error(self, mock_db):
...@@ -289,14 +221,14 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -289,14 +221,14 @@ class StudentTrainingAssessmentTest(CacheResetTest):
def test_get_training_example_database_error(self, mock_db): def test_get_training_example_database_error(self, mock_db):
mock_db.side_effect = DatabaseError("Kaboom!") mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError): with self.assertRaises(StudentTrainingInternalError):
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
@patch.object(StudentTrainingWorkflow.objects, 'get') @patch.object(StudentTrainingWorkflow.objects, 'get')
def test_assess_training_example_database_error(self, mock_db): def test_assess_training_example_database_error(self, mock_db):
training_api.get_training_example(self.submission_uuid, self.RUBRIC, self.EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
mock_db.side_effect = DatabaseError("Kaboom!") mock_db.side_effect = DatabaseError("Kaboom!")
with self.assertRaises(StudentTrainingInternalError): with self.assertRaises(StudentTrainingInternalError):
training_api.assess_training_example(self.submission_uuid, self.EXAMPLES[0]['options_selected']) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
@ddt.data({}, {'num_required': 'not an integer!'}) @ddt.data({}, {'num_required': 'not an integer!'})
def test_submitter_is_finished_invalid_requirements(self, requirements): def test_submitter_is_finished_invalid_requirements(self, requirements):
...@@ -388,7 +320,7 @@ class StudentTrainingAssessmentTest(CacheResetTest): ...@@ -388,7 +320,7 @@ class StudentTrainingAssessmentTest(CacheResetTest):
None None
""" """
pre_submission = sub_api.create_submission(self.STUDENT_ITEM, self.ANSWER) pre_submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
for example in examples: for example in examples:
training_api.get_training_example(pre_submission['uuid'], rubric, examples) training_api.get_training_example(pre_submission['uuid'], rubric, examples)
training_api.assess_training_example(pre_submission['uuid'], example['options_selected']) training_api.assess_training_example(pre_submission['uuid'], example['options_selected'])
apps/openassessment/assessment/test/test_student_training_models.py 0 → 100644
"""
Tests for student training models.
"""
import mock
from django.db import IntegrityError
from submissions import api as sub_api
from openassessment.test_utils import CacheResetTest
from openassessment.assessment.models import (
StudentTrainingWorkflow, StudentTrainingWorkflowItem
)
from .constants import STUDENT_ITEM, ANSWER, EXAMPLES
class StudentTrainingWorkflowTest(CacheResetTest):
"""
Tests for the student training workflow model.
"""
@mock.patch.object(StudentTrainingWorkflow.objects, 'get')
@mock.patch.object(StudentTrainingWorkflow.objects, 'create')
def test_create_workflow_integrity_error(self, mock_create, mock_get):
# Simulate a race condition in which someone creates a workflow
# after we check if it exists. This will violate the database uniqueness
# constraints, so we need to handle this case gracefully.
mock_create.side_effect = IntegrityError
# The first time we check, we should see that no workflow exists.
# The second time, we should get the workflow created by someone else
mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
mock_get.side_effect = [
StudentTrainingWorkflow.DoesNotExist,
mock_workflow
]
# Expect that we retry and retrieve the workflow that someone else created
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.get_or_create_workflow(submission['uuid'])
self.assertEqual(workflow, mock_workflow)
@mock.patch.object(StudentTrainingWorkflowItem.objects, 'get')
@mock.patch.object(StudentTrainingWorkflowItem.objects, 'create')
def test_create_workflow_item_integrity_error(self, mock_create, mock_get):
# Create a submission and workflow
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.get_or_create_workflow(submission['uuid'])
# Simulate a race condition in which someone creates a workflow item
# after we check if it exists.
mock_workflow_item = mock.MagicMock(StudentTrainingWorkflowItem)
mock_create.side_effect = IntegrityError
mock_get.return_value = mock_workflow_item
# Expect that we retry and retrieve the workflow item created by someone else
self.assertEqual(workflow.next_training_example(EXAMPLES), mock_workflow_item.training_example)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment