Skip to content

E
edx-ora2
Commit 9802ff82 by gradyward
Options

Added in regrading functionality 

Also added automatic grading after training completes

Many code review edits
parent d9f28950
Showing with 908 additions and 40 deletions
apps/openassessment/assessment/admin.py
......@@ -128,9 +128,8 @@ class AIGradingWorkflowAdmin(admin.ModelAdmin):
class AITrainingWorkflowAdmin(admin.ModelAdmin):
list_display = ('uuid',)
# TODO -- update to include student/item/course id
search_fields = ('uuid',)
readonly_fields = ('uuid',)
search_fields = ('uuid', 'course_id', 'item_id',)
readonly_fields = ('uuid', 'course_id', 'item_id',)
class AIClassifierInline(admin.TabularInline):
......
apps/openassessment/assessment/api/ai.py
......@@ -5,12 +5,11 @@ import logging
from django.db import DatabaseError
from submissions import api as sub_api
from openassessment.assessment.serializers import (
deserialize_training_examples, InvalidTrainingExample, InvalidRubric,
full_assessment_dict
deserialize_training_examples, InvalidTrainingExample, InvalidRubric, full_assessment_dict
)
from openassessment.assessment.errors import (
AITrainingRequestError, AITrainingInternalError,
AIGradingRequestError, AIGradingInternalError
AIGradingRequestError, AIGradingInternalError, AIError
)
from openassessment.assessment.models import (
Assessment, AITrainingWorkflow, AIGradingWorkflow,
......@@ -206,10 +205,12 @@ def get_latest_assessment(submission_uuid):
return None
def train_classifiers(rubric_dict, examples, algorithm_id):
def train_classifiers(rubric_dict, examples, course_id, item_id, algorithm_id):
"""
Schedule a task to train classifiers.
All training examples must match the rubric!
After training of classifiers completes successfully, all AIGradingWorkflows that are incomplete will be
automatically rescheduled to complete.
Args:
rubric_dict (dict): The rubric used to assess the classifiers.
......@@ -225,6 +226,7 @@ def train_classifiers(rubric_dict, examples, algorithm_id):
Raises:
AITrainingRequestError
AITrainingInternalError
AIGradingInternalError
Example usage:
>>> train_classifiers(rubric, examples, 'ease')
......@@ -240,7 +242,7 @@ def train_classifiers(rubric_dict, examples, algorithm_id):
# Create the workflow model
try:
workflow = AITrainingWorkflow.start_workflow(examples, algorithm_id)
workflow = AITrainingWorkflow.start_workflow(examples, course_id, item_id, algorithm_id)
except NoTrainingExamples as ex:
raise AITrainingRequestError(ex)
except:
......@@ -258,31 +260,70 @@ def train_classifiers(rubric_dict, examples, algorithm_id):
u"Scheduled training task for the AI training workflow with UUID {workflow_uuid} "
u"(algorithm ID = {algorithm_id})"
).format(workflow_uuid=workflow.uuid, algorithm_id=algorithm_id))
except:
except (AITrainingInternalError, AITrainingRequestError):
msg = (
u"An unexpected error occurred while scheduling "
u"the task for training workflow with UUID {}"
).format(workflow.uuid)
logger.exception(msg)
raise AITrainingInternalError(msg)
except AIGradingInternalError:
# If we have an error that is coming from the rescheduled grading after successful completion:
msg = (
u"An unexpected error occurred while scheduling incomplete grading workflows after "
u"the training task was completed successfully. The course_id and item_id for the failed "
u"grading workflows are course_id={cid}, item_id={iid}."
).format(cid=course_id, iid=item_id)
logger.exception(msg)
raise AIGradingInternalError(msg)
# Return the workflow UUID
return workflow.uuid
def reschedule_unfinished_tasks(course_id=None, item_id=None, task_type=None):
def reschedule_unfinished_tasks(course_id=None, item_id=None, task_type=u"grade"):
"""
Check for unfinished tasks (both grading and training) and reschedule them.
Optionally restrict by course/item ID and task type.
Optionally restrict by course/item ID and task type. Default use case is to
only reschedule the unfinished grade tasks. Applied use case (with button in
staff mixin) is to call without argument, and to reschedule grades only.
Kwargs:
course_id (unicode): Restrict to unfinished tasks in a particular course.
item_id (unicode): Restrict to unfinished tasks for a particular item in a course.
NOTE: if you specify the item ID, you must also specify the course ID.
task_type (unicode): Either "grade" or "train". Restrict to unfinished tasks of this type.
if task_type is specified as None, both training and grading will be rescheduled, in that order.
Raises:
AIGradingInternalError
AITrainingInternalError
AIError
"""
pass
if course_id is None or item_id is None:
msg = u"Rescheduling tasks was not possible because the course_id / item_id was not assigned."
logger.exception(msg)
raise AIError
# Reschedules all of the training tasks
if task_type == u"train" or task_type is None:
try:
training_tasks.reschedule_training_tasks.apply_async(args=[course_id, item_id])
except Exception as ex:
msg = (
u"Rescheduling training tasks for course {cid} and item {iid} failed with exception: {ex}"
).format(cid=course_id, iid=item_id, ex=ex)
logger.exception(msg)
raise AITrainingInternalError(ex)
# Reschedules all of the grading tasks
if task_type == u"grade" or task_type is None:
try:
grading_tasks.reschedule_grading_tasks.apply_async(args=[course_id, item_id])
except Exception as ex:
msg = (
u"Rescheduling grading tasks for course {cid} and item {iid} failed with exception: {ex}"
).format(cid=course_id, iid=item_id, ex=ex)
logger.exception(msg)
raise AIGradingInternalError(ex)
apps/openassessment/assessment/api/ai_worker.py
......@@ -30,6 +30,8 @@ def get_grading_task_params(grading_workflow_uuid):
dict with keys:
* essay_text (unicode): The text of the essay submission.
* classifier_set (dict): Maps criterion names to serialized classifiers.
* course_id (unicode): The course ID that the training task is associated with.
* item_id (unicode): Identifies the item that the AI will be training to grade.
* algorithm_id (unicode): ID of the algorithm used to perform training.
Raises:
......@@ -66,6 +68,8 @@ def get_grading_task_params(grading_workflow_uuid):
return {
'essay_text': workflow.essay_text,
'classifier_set': classifier_set.classifiers_dict,
'course_id': workflow.course_id,
'item_id': workflow.item_id,
'algorithm_id': workflow.algorithm_id,
}
except Exception as ex:
......@@ -142,6 +146,8 @@ def get_training_task_params(training_workflow_uuid):
Returns:
dict with keys:
* training_examples (list of dict): The examples used to train the classifiers.
* course_id (unicode): The course ID that the training task is associated with.
* item_id (unicode): Identifies the item that the AI will be training to grade.
* algorithm_id (unicode): The ID of the algorithm to use for training.
Raises:
......@@ -194,7 +200,9 @@ def get_training_task_params(training_workflow_uuid):
return {
'training_examples': returned_examples,
'algorithm_id': workflow.algorithm_id
'algorithm_id': workflow.algorithm_id,
'course_id': workflow.course_id,
'item_id': workflow.item_id
}
except AITrainingWorkflow.DoesNotExist:
msg = (
......
apps/openassessment/assessment/migrations/0014_auto__add_field_aitrainingworkflow_item_id__add_field_aitrainingworkfl.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'AITrainingWorkflow.item_id'
db.add_column('assessment_aitrainingworkflow', 'item_id',
self.gf('django.db.models.fields.CharField')(default='', max_length=128, db_index=True),
keep_default=False)
# Adding field 'AITrainingWorkflow.course_id'
db.add_column('assessment_aitrainingworkflow', 'course_id',
self.gf('django.db.models.fields.CharField')(default='', max_length=40, db_index=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'AITrainingWorkflow.item_id'
db.delete_column('assessment_aitrainingworkflow', 'item_id')
# Deleting field 'AITrainingWorkflow.course_id'
db.delete_column('assessment_aitrainingworkflow', 'course_id')
models = {
'assessment.aiclassifier': {
'Meta': {'object_name': 'AIClassifier'},
'classifier_data': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'classifiers'", 'to': "orm['assessment.AIClassifierSet']"}),
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Criterion']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.aiclassifierset': {
'Meta': {'ordering': "['-created_at']", 'object_name': 'AIClassifierSet'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"})
},
'assessment.aigradingworkflow': {
'Meta': {'object_name': 'AIGradingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.Assessment']"}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'essay_text': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.Rubric']"}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.aitrainingworkflow': {
'Meta': {'object_name': 'AITrainingWorkflow'},
'algorithm_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'classifier_set': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['assessment.AIClassifierSet']"}),
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'scheduled_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'training_examples': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'+'", 'symmetrical': 'False', 'to': "orm['assessment.TrainingExample']"}),
'uuid': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '36', 'blank': 'True'})
},
'assessment.assessment': {
'Meta': {'ordering': "['-scored_at', '-id']", 'object_name': 'Assessment'},
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"}),
'score_type': ('django.db.models.fields.CharField', [], {'max_length': '2'}),
'scored_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'scorer_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedback': {
'Meta': {'object_name': 'AssessmentFeedback'},
'assessments': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.Assessment']"}),
'feedback_text': ('django.db.models.fields.TextField', [], {'default': "''", 'max_length': '10000'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options': ('django.db.models.fields.related.ManyToManyField', [], {'default': 'None', 'related_name': "'assessment_feedback'", 'symmetrical': 'False', 'to': "orm['assessment.AssessmentFeedbackOption']"}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.assessmentfeedbackoption': {
'Meta': {'object_name': 'AssessmentFeedbackOption'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'text': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'assessment.assessmentpart': {
'Meta': {'object_name': 'AssessmentPart'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'parts'", 'to': "orm['assessment.Assessment']"}),
'feedback': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'option': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['assessment.CriterionOption']"})
},
'assessment.criterion': {
'Meta': {'ordering': "['rubric', 'order_num']", 'object_name': 'Criterion'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'prompt': ('django.db.models.fields.TextField', [], {'max_length': '10000'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'criteria'", 'to': "orm['assessment.Rubric']"})
},
'assessment.criterionoption': {
'Meta': {'ordering': "['criterion', 'order_num']", 'object_name': 'CriterionOption'},
'criterion': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'options'", 'to': "orm['assessment.Criterion']"}),
'explanation': ('django.db.models.fields.TextField', [], {'max_length': '10000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'points': ('django.db.models.fields.PositiveIntegerField', [], {})
},
'assessment.peerworkflow': {
'Meta': {'ordering': "['created_at', 'id']", 'object_name': 'PeerWorkflow'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'created_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'grading_completed_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.peerworkflowitem': {
'Meta': {'ordering': "['started_at', 'id']", 'object_name': 'PeerWorkflowItem'},
'assessment': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Assessment']", 'null': 'True'}),
'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded_by'", 'to': "orm['assessment.PeerWorkflow']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'scored': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'scorer': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'graded'", 'to': "orm['assessment.PeerWorkflow']"}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'})
},
'assessment.rubric': {
'Meta': {'object_name': 'Rubric'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'assessment.studenttrainingworkflow': {
'Meta': {'object_name': 'StudentTrainingWorkflow'},
'course_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'item_id': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}),
'student_id': ('django.db.models.fields.CharField', [], {'max_length': '40', 'db_index': 'True'}),
'submission_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'db_index': 'True'})
},
'assessment.studenttrainingworkflowitem': {
'Meta': {'ordering': "['workflow', 'order_num']", 'unique_together': "(('workflow', 'order_num'),)", 'object_name': 'StudentTrainingWorkflowItem'},
'completed_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'order_num': ('django.db.models.fields.PositiveIntegerField', [], {}),
'started_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'training_example': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.TrainingExample']"}),
'workflow': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'to': "orm['assessment.StudentTrainingWorkflow']"})
},
'assessment.trainingexample': {
'Meta': {'object_name': 'TrainingExample'},
'content_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'options_selected': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['assessment.CriterionOption']", 'symmetrical': 'False'}),
'raw_answer': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'rubric': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['assessment.Rubric']"})
}
}
complete_apps = ['assessment']
\ No newline at end of file
apps/openassessment/assessment/models/ai.py
......@@ -3,19 +3,24 @@ Database models for AI assessment.
"""
from uuid import uuid4
import json
import logging
from django.conf import settings
from django.core.files.base import ContentFile
from django.db import models, transaction
from django.db import models, transaction, DatabaseError
from django.utils.timezone import now
from django.core.exceptions import ObjectDoesNotExist
from django_extensions.db.fields import UUIDField
from submissions import api as sub_api
from openassessment.assessment.serializers import rubric_from_dict
from openassessment.assessment.errors.ai import AIError
from .base import Rubric, Criterion, Assessment, AssessmentPart
from .training import TrainingExample
AI_ASSESSMENT_TYPE = "AI"
logger = logging.getLogger(__name__)
class IncompleteClassifierSet(Exception):
"""
......@@ -251,6 +256,13 @@ class AIWorkflow(models.Model):
# Unique identifier used to track this workflow
uuid = UUIDField(version=1, db_index=True)
# Course Entity and Item Discriminator
# Though these items are duplicated in the database tables for the submissions app,
# and every workflow has a reference to a submission entry, this is okay because
# submissions are immutable.
course_id = models.CharField(max_length=40, db_index=True)
item_id = models.CharField(max_length=128, db_index=True)
# Timestamps
# The task is *scheduled* as soon as a client asks the API to
# train classifiers.
......@@ -296,6 +308,42 @@ class AIWorkflow(models.Model):
self.completed_at = now()
self.save()
@classmethod
def get_incomplete_workflows(cls, course_id, item_id):
"""
Gets all incomplete grading workflows for a given course and item.
Args:
course_id (unicode): Uniquely identifies the course
item_id (unicode): The discriminator for the item we are looking for
Yields:
All incomplete workflows for this item, as a delayed "stream"
Raises:
DatabaseError
cls.DoesNotExist
"""
# Finds all of the uuid's for workflows contained within the query
grade_workflow_uuids = [
wflow['uuid'] for wflow in cls.objects.filter(
course_id=course_id, item_id=item_id, completed_at__isnull=True
).values('uuid')
]
# Continues to generate output until all workflows in the queryset have been output
for workflow_uuid in grade_workflow_uuids:
# Returns the grading workflow associated with the uuid stored in the initial query
try:
grading_workflow = cls.objects.get(uuid=workflow_uuid)
yield grading_workflow
except (cls.DoesNotExist, ObjectDoesNotExist, DatabaseError) as ex:
msg = u"No workflow with uuid '{}' could be found within the system.".format(workflow_uuid)
logger.exception(msg)
raise AIError(ex)
class AITrainingWorkflow(AIWorkflow):
"""
......@@ -318,12 +366,14 @@ class AITrainingWorkflow(AIWorkflow):
@classmethod
@transaction.commit_on_success
def start_workflow(cls, examples, algorithm_id):
def start_workflow(cls, examples, course_id, item_id, algorithm_id):
"""
Start a workflow to track a training task.
Args:
examples (list of TrainingExample): The training examples used to create the classifiers.
course_id (unicode): The ID for the course that the training workflow is associated with.
item_id (unicode): The ID for the item that the training workflow is training to assess.
algorithm_id (unicode): The ID of the algorithm to use for training.
Returns:
......@@ -336,7 +386,7 @@ class AITrainingWorkflow(AIWorkflow):
if len(examples) == 0:
raise NoTrainingExamples()
workflow = AITrainingWorkflow.objects.create(algorithm_id=algorithm_id)
workflow = AITrainingWorkflow.objects.create(algorithm_id=algorithm_id, item_id=item_id, course_id=course_id)
workflow.training_examples.add(*examples)
workflow.save()
return workflow
......@@ -422,8 +472,6 @@ class AIGradingWorkflow(AIWorkflow):
# associated with one submission, it's safe to duplicate
# this information here from the submissions models.
student_id = models.CharField(max_length=40, db_index=True)
item_id = models.CharField(max_length=128, db_index=True)
course_id = models.CharField(max_length=40, db_index=True)
@classmethod
@transaction.commit_on_success
......
apps/openassessment/assessment/tasks.py
......@@ -3,5 +3,5 @@ Celery looks for tasks in this module,
so import the tasks we want the workers to implement.
"""
# pylint:disable=W0611
from .worker.training import train_classifiers
from .worker.grading import grade_essay
from .worker.training import train_classifiers, reschedule_training_tasks
from .worker.grading import grade_essay, reschedule_grading_tasks
\ No newline at end of file
apps/openassessment/assessment/test/test_ai.py
......@@ -9,14 +9,16 @@ from django.test.utils import override_settings
from openassessment.test_utils import CacheResetTest
from submissions import api as sub_api
from openassessment.assessment.api import ai as ai_api
from openassessment.assessment.models import (
AITrainingWorkflow, AIGradingWorkflow, AIClassifierSet, Assessment
)
from openassessment.assessment.worker.algorithm import AIAlgorithm
from openassessment.assessment.models import AITrainingWorkflow, AIGradingWorkflow, AIClassifierSet
from openassessment.assessment.worker.algorithm import AIAlgorithm, AIAlgorithmError
from openassessment.assessment.serializers import rubric_from_dict
from openassessment.assessment.errors import (
AITrainingRequestError, AITrainingInternalError,
AIGradingRequestError, AIGradingInternalError
AITrainingRequestError, AITrainingInternalError, AIGradingRequestError, AIGradingInternalError, AIError
)
from openassessment.assessment.test.constants import RUBRIC, EXAMPLES, STUDENT_ITEM, ANSWER
......@@ -56,6 +58,8 @@ class StubAIAlgorithm(AIAlgorithm):
ALGORITHM_ID = "test-stub"
COURSE_ID = STUDENT_ITEM.get('course_id')
ITEM_ID = STUDENT_ITEM.get('item_id')
AI_ALGORITHMS = {
ALGORITHM_ID: '{module}.StubAIAlgorithm'.format(module=__name__),
......@@ -94,7 +98,7 @@ class AITrainingTest(CacheResetTest):
# Schedule a training task
# Because Celery is configured in "always eager" mode,
# expect the task to be executed synchronously.
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
# Retrieve the classifier set from the database
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
......@@ -135,12 +139,12 @@ class AITrainingTest(CacheResetTest):
# Expect a request error
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, mutated_examples, ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, mutated_examples, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_train_classifiers_no_examples(self):
# Empty list of training examples
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, [], ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, [], COURSE_ID, ITEM_ID, ALGORITHM_ID)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch.object(AITrainingWorkflow.objects, 'create')
......@@ -148,15 +152,15 @@ class AITrainingTest(CacheResetTest):
# Simulate a database error when creating the training workflow
mock_create.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
@mock.patch('openassessment.assessment.api.ai.training_tasks')
def test_schedule_training_error(self, mock_training_tasks):
# Simulate an exception raised when scheduling a training task
mock_training_tasks.train_classifiers.apply_async.side_effect = Exception("KABOOM!")
mock_training_tasks.train_classifiers.apply_async.side_effect = AITrainingRequestError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, ALGORITHM_ID)
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
class AIGradingTest(CacheResetTest):
......@@ -263,4 +267,233 @@ class AIUntrainedGradingTest:
# Test that no score has been created, and get_score returns None.
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
score = ai_api.get_score(self.submission_uuid, {})
self.assertIsNone(score)
\ No newline at end of file
self.assertIsNone(score)
class AIReschedulingTest(CacheResetTest):
"""
Tests AI rescheduling.
Tests in both orders, and tests all error conditions that can arise as a result of calling rescheduling
"""
def setUp(self):
"""
Sets up each test so that it will have unfinished tasks of both types
"""
# 1) Schedule Grading, have the scheduling succeeed but the grading fail because no classifiers exist
for i in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
# 2) Schedule Training, have it INTENTIONALLY fail. Now we are a point where both parts need to be rescheduled
patched_method = 'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async'
with mock.patch(patched_method) as mock_train_classifiers:
mock_train_classifiers.side_effect = AITrainingInternalError('Training Classifiers Failed for some Reason.')
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
self._assert_complete(training_done=False, grading_done=False)
def _assert_complete(self, training_done=None, grading_done=None):
"""
Asserts that the Training and Grading are of a given completion status
Serves as an assertion for a number of unit tests.
Args:
training_done (bool): whether the user expects there to be unfinished training workflows
grading_done (bool): whether the user expects there to be unfinished grading workflows
"""
incomplete_training_workflows = AITrainingWorkflow.get_incomplete_workflows(course_id=COURSE_ID,item_id=ITEM_ID)
incomplete_grading_workflows = AIGradingWorkflow.get_incomplete_workflows(course_id=COURSE_ID, item_id=ITEM_ID)
if training_done is not None:
self.assertEqual(self._is_empty_generator(incomplete_training_workflows), training_done)
if grading_done is not None:
self.assertEqual(self._is_empty_generator(incomplete_grading_workflows), grading_done)
def _is_empty_generator(self, gen):
"""
Tests whether a given generator has any more output.
Consumes a unit of output in test.
Args:
gen (generator): A generator to test if empty
Returns:
(bool): whether or not the generator contained output before testing
"""
try:
next(gen)
return False
except StopIteration:
return True
def _call_reschedule_safe(self, task_type=u"grade"):
"""
A method which will reject an exception thrown by the unfinished task API.
This method is necessary because when we set our celery workers to propogate all errors upward
(as we now do in our unit testing suite), that also means that when a task fails X times (say
a grading task fails because classifiers are not defined) that exception will be retruned from
the call of the grade_essay (even though asynchronous), and peroclate up. This method is used
to agknowledge the fact that we expect there to be an error, and allow us to call reschedule
unfinished tasks without catching that error directly.
Args:
task_type (unicode): describes what tasks we should reschedule
"""
try:
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=task_type)
except Exception as ex:
# This exception is being raised because of a timeout.
pass
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_grading_success(self):
# Rescheduling grading only, expect no successes
self._call_reschedule_safe(task_type=u"grade")
# Neither training nor grading should be complete.
self._assert_complete(grading_done=False, training_done=False)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_training_success(self):
# Reschedule training, expect all successes
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=u"train")
# Both training and grading should be complete.
self._assert_complete(grading_done=True, training_done=True)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_training_success(self):
# Reschedule everything, expect all successes
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=None)
# Both training and grading should be complete.
self._assert_complete(grading_done=True, training_done=True)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_database_failure(self):
# 3) Mock in a DB error.
patched_method = 'openassessment.assessment.worker.grading.AIGradingWorkflow.objects.filter'
with mock.patch(patched_method) as mock_filter:
mock_filter.side_effect = DatabaseError('DB ERROR: KABOOM')
with self.assertRaises(AIGradingInternalError):
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=None)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_grade_failure(self):
# Mock in consistent failure to grade an essay
with mock.patch('openassessment.assessment.worker.grading.grade_essay.apply_async') as mock_grade_essay:
mock_grade_essay.side_effect = AIGradingInternalError("Sorry, no grade for you.")
with self.assertRaises(AIGradingInternalError):
# Try to reschedule all unfinished tasks, expect an internal grading error
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=None)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_train_failure(self):
# Mock in consistent failure to grade an essay
with mock.patch('openassessment.assessment.worker.training.train_classifiers.apply_async') as mock_train:
mock_train.side_effect = AITrainingInternalError("OH MY! THE TRAINing DERAILED!")
with self.assertRaises(AITrainingInternalError):
# Try to reschedule all unfinished tasks, expect an internal tra error
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=u"train")
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_non_valid_args(self):
with self.assertRaises(AIError):
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, task_type=u"train")
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_reschedule_all_large(self):
"""
Specifically tests the querying mechanisms (python generator functions), and ensures that our methodology
holds up for querysets with 125+ entries
"""
# Creates 125 more grades (for a total of 135)
for i in range(0, 125):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
# Both training and grading should not be complete.
self._assert_complete(grading_done=False, training_done=False)
# Reschedule both
ai_api.reschedule_unfinished_tasks(course_id=COURSE_ID, item_id=ITEM_ID, task_type=None)
# Check that both training and grading are now complete
self._assert_complete(grading_done=True, training_done=True)
class AIAutomaticGradingTest(CacheResetTest):
def _assert_complete(self, training_done=None, grading_done=None):
"""
Asserts that the Training and Grading are of a given completion status
Serves as an assertion for a number of unit tests.
Args:
training_done (bool): whether the user expects there to be unfinished training workflows
grading_done (bool): whether the user expects there to be unfinished grading workflows
"""
incomplete_training_workflows = AITrainingWorkflow.get_incomplete_workflows(course_id=COURSE_ID,item_id=ITEM_ID)
incomplete_grading_workflows = AIGradingWorkflow.get_incomplete_workflows(course_id=COURSE_ID, item_id=ITEM_ID)
if training_done is not None:
self.assertEqual(self._is_empty_generator(incomplete_training_workflows), training_done)
if grading_done is not None:
self.assertEqual(self._is_empty_generator(incomplete_grading_workflows), grading_done)
def _is_empty_generator(self, gen):
"""
Tests whether a given generator has any more output.
Consumes a unit of output in test.
Args:
gen (generator): A generator to test if empty
Returns:
(bool): whether or not the generator contained output before testing
"""
try:
next(gen)
return False
except StopIteration:
return True
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_automatic_grade(self):
# Create some submissions which will not succeed. No classifiers yet exist.
for i in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
# Check that there are unresolved grading workflows
self._assert_complete(training_done=True, grading_done=False)
# Create and train a classifier set. This should set off automatic grading.
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
# Check to make sure that all work is done.
self._assert_complete(training_done=True, grading_done=True)
@override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
def test_automatic_grade_error(self):
# Create some submissions which will not succeed. No classifiers yet exist.
for i in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
ai_api.submit(self.submission_uuid, RUBRIC, ALGORITHM_ID)
# Check that there are unresolved grading workflows
self._assert_complete(training_done=True, grading_done=False)
patched_method = 'openassessment.assessment.worker.training.reschedule_grading_tasks.apply_async'
with mock.patch(patched_method) as mocked_reschedule_grading:
mocked_reschedule_grading.side_effect = AIGradingInternalError("Kablewey.")
with self.assertRaises(AIGradingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
apps/openassessment/assessment/test/test_ai_worker.py
......@@ -48,12 +48,33 @@ class AIWorkerTrainingTest(CacheResetTest):
Tests for the AI API calls a worker would make when
completing a training task.
"""
COURSE_ID = u"sämplë ċöürsë"
ITEM_ID = u"12231"
ALGORITHM_ID = "test-algorithm"
# Classifier data
# Since this is controlled by the AI algorithm implementation,
# we could put anything here as long as it's JSON-serializable.
CLASSIFIERS = {
u"vøȼȺƀᵾłȺɍɏ": {
'name': u'𝒕𝒆𝒔𝒕 𝒄𝒍𝒂𝒔𝒔𝒊𝒇𝒊𝒆𝒓',
'data': u'Öḧ ḷëẗ ẗḧë ṡüṅ ḅëäẗ ḋöẅṅ üṗöṅ ṁÿ ḟäċë, ṡẗäṛṡ ẗö ḟïḷḷ ṁÿ ḋṛëäṁ"'
},
u"ﻭɼค๓๓คɼ": {
'name': u'𝒕𝒆𝒔𝒕 𝒄𝒍𝒂𝒔𝒔𝒊𝒇𝒊𝒆𝒓',
'data': u"І ам а тѓаvэlэѓ оf ъотЂ тімэ аиↁ ѕрасэ, то ъэ шЂэѓэ І Ђаvэ ъээи"
}
}
def setUp(self):
"""
Create a training workflow in the database.
"""
examples = deserialize_training_examples(EXAMPLES, RUBRIC)
workflow = AITrainingWorkflow.start_workflow(examples, ALGORITHM_ID)
workflow = AITrainingWorkflow.start_workflow(examples, self.COURSE_ID, self.ITEM_ID, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def test_get_training_task_params(self):
......@@ -204,7 +225,9 @@ class AIWorkerGradingTest(CacheResetTest):
expected_params = {
'essay_text': ANSWER,
'classifier_set': CLASSIFIERS,
'algorithm_id': ALGORITHM_ID
'algorithm_id': ALGORITHM_ID,
'course_id': STUDENT_ITEM.get('course_id'),
'item_id': STUDENT_ITEM.get('item_id')
}
self.assertItemsEqual(params, expected_params)
......
apps/openassessment/assessment/test/test_worker.py
......@@ -90,12 +90,28 @@ class AITrainingTaskTest(CeleryTaskTest):
Tests for the training task executed asynchronously by Celery workers.
"""
COURSE_ID = u"10923"
ITEM_ID = u"12231"
ALGORITHM_ID = u"test-stub"
ERROR_STUB_ALGORITHM_ID = u"error-stub"
UNDEFINED_CLASS_ALGORITHM_ID = u"undefined_class"
UNDEFINED_MODULE_ALGORITHM_ID = u"undefined_module"
AI_ALGORITHMS = {
ALGORITHM_ID: '{module}.StubAIAlgorithm'.format(module=__name__),
ERROR_STUB_ALGORITHM_ID: '{module}.ErrorStubAIAlgorithm'.format(module=__name__),
UNDEFINED_CLASS_ALGORITHM_ID: '{module}.NotDefinedAIAlgorithm'.format(module=__name__),
UNDEFINED_MODULE_ALGORITHM_ID: 'openassessment.not.valid.NotDefinedAIAlgorithm'
}
def setUp(self):
"""
Create a training workflow in the database.
"""
examples = deserialize_training_examples(EXAMPLES, RUBRIC)
workflow = AITrainingWorkflow.start_workflow(examples, ALGORITHM_ID)
workflow = AITrainingWorkflow.start_workflow(examples, self.COURSE_ID, self.ITEM_ID, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def test_unknown_algorithm(self):
......
apps/openassessment/assessment/worker/grading.py
......@@ -3,10 +3,12 @@ Asynchronous tasks for grading essays using text classifiers.
"""
from celery import task
from django.db import DatabaseError
from celery.utils.log import get_task_logger
from openassessment.assessment.api import ai_worker as ai_worker_api
from openassessment.assessment.errors import AIError
from openassessment.assessment.errors import AIError, AIGradingInternalError, AIGradingRequestError
from .algorithm import AIAlgorithm, AIAlgorithmError
from openassessment.assessment.models.ai import AIClassifierSet, AIGradingWorkflow
MAX_RETRIES = 2
......@@ -87,3 +89,98 @@ def grade_essay(workflow_uuid):
).format(uuid=workflow_uuid, scores=scores_by_criterion)
logger.exception(msg)
raise grade_essay.retry()
@task(max_retries=MAX_RETRIES) # pylint: disable=E1102
def reschedule_grading_tasks(course_id, item_id):
"""
Reschedules all incomplete grading workflows with the specified parameters.
Args:
course_id (unicode): The course item that we will be rerunning the rescheduling on.
item_id (unicode): The item that the rescheduling will be running on
"""
# Finds all incomplete grading workflows
grading_workflows = AIGradingWorkflow.get_incomplete_workflows(course_id, item_id)
# Notes whether or not one or more operations failed. If they did, the process of rescheduling will be retried.
failures = 0
# A dictionary mapping tuples of (rubric, algorithm_id) to completed classifier sets. Used to avoid repeated
# queries which will return the same value. This loop implements a memoization of the the query.
maintained_classifiers = {}
# Try to grade all incomplete grading workflows
for workflow in grading_workflows:
# We will always go through the process of finding the most recent set of classifiers for an
# incomplete grading workflow. The rationale for this is that if we are ever rescheduling
# grading, we likely had classifiers which were not working. This way, we always take the last
# completed set.
# Note that this solution will lead to failure if "Train Classifiers" and "Refinish Grading Tasks"
# are called in rapid succession. This is part of the reason this button is in the admin view.
# Tries to find a set of classifiers that are already defined in our maintained_classifiers based on a
# description of the workflow in the form of a tuple (rubric, algorithm_id)
workflow_description = (workflow.rubric, workflow.algorithm_id)
found_classifiers = maintained_classifiers.get(workflow_description)
# If no set of classifiers is found, we perform the query to try to find them. We take the most recent
# and add it to our dictionary of maintained classifiers for future reference.
if found_classifiers is None:
try:
classifier_set_candidates = AIClassifierSet.objects.filter(
rubric=workflow.rubric, algorithm_id=workflow.algorithm_id
).order_by('-created_at')[:1]
found_classifiers = classifier_set_candidates[0]
maintained_classifiers[workflow_description] = found_classifiers
except IndexError:
msg = u"No classifiers yet exist for essay with uuid='{}'".format(workflow.uuid)
logger.log(msg)
except DatabaseError as ex:
msg = (
u"A Database error occurred while trying to assign classifiers to an essay with uuid='{id}'"
).format(id=workflow.uuid)
logger.exception(msg)
if found_classifiers is not None:
workflow.classifier_set = found_classifiers
try:
workflow.save()
logger.info(
(
u"Classifiers were successfully assigned to grading workflow with uuid={}"
).format(workflow.uuid)
)
except DatabaseError as ex:
msg = (
u"A Database error occurred while trying to save classifiers to an essay with uuid='{id}'"
).format(id=workflow.uuid)
logger.exception(msg)
# Now we should (unless we had an exception above) have a classifier set.
# Try to schedule the grading
try:
grade_essay.apply_async(args=[workflow.uuid])
logger.info(
u"Rescheduling of grading was successful for grading workflow with uuid='{}'".format(workflow.uuid)
)
except (AIGradingInternalError, AIGradingRequestError, AIError) as ex:
msg = (
u"An error occurred while try to grade essay with uuid='{id}': {ex}"
).format(id=workflow.uuid, ex=ex)
logger.exception(msg)
failures += 1
# If one or more of these failed, we want to retry rescheduling. Note that this retry is executed in such a way
# that if it fails, an AIGradingInternalError will be raised with the number of failures on the last attempt (i.e.
# the total number of workflows matching these critera that still have left to be graded).
if failures > 0:
try:
raise AIGradingInternalError(
u"In an attempt to reschedule grading workflows, there were {} failures.".format(failures)
)
except AIGradingInternalError as ex:
raise reschedule_grading_tasks.retry()
\ No newline at end of file
apps/openassessment/assessment/worker/training.py
......@@ -7,6 +7,9 @@ from celery.utils.log import get_task_logger
from openassessment.assessment.api import ai_worker as ai_worker_api
from openassessment.assessment.errors import AIError
from .algorithm import AIAlgorithm, AIAlgorithmError
from .grading import reschedule_grading_tasks
from openassessment.assessment.errors.ai import AIGradingInternalError
from openassessment.assessment.models.ai import AITrainingWorkflow
MAX_RETRIES = 2
......@@ -58,6 +61,8 @@ def train_classifiers(workflow_uuid):
params = ai_worker_api.get_training_task_params(workflow_uuid)
examples = params['training_examples']
algorithm_id = params['algorithm_id']
course_id = params['course_id']
item_id = params['item_id']
except (AIError, KeyError):
msg = (
u"An error occurred while retrieving AI training "
......@@ -121,6 +126,45 @@ def train_classifiers(workflow_uuid):
logger.exception(msg)
raise train_classifiers.retry()
# Upon successful completion of the creation of classifiers, we will try to automatically schedule any
# grading tasks for the same item.
try:
reschedule_grading_tasks.apply_async(args=[course_id, item_id])
except AIGradingInternalError as ex:
msg = (
u"An error occured while trying to regrade all ungraded assignments"
u"after classifiers were trained successfully: {}"
).format(ex)
logger.exception(msg)
# Here we don't retry, because they will already retry once in the grading task.
raise
@task(max_retries=MAX_RETRIES) #pylint: disable E=1102
def reschedule_training_tasks(course_id, item_id):
"""
Reschedules all incomplete training tasks
Args:
course_id (unicode): The course that we are going to search for unfinished training workflows
item_id (unicode): The specific item within that course that we will reschedule unfinished workflows for
"""
# Run a query to find the incomplete training workflows
training_workflows = AITrainingWorkflow.get_incomplete_workflows(course_id, item_id)
# Tries to train every workflow that has not completed.
for target_workflow in training_workflows:
try:
train_classifiers.apply_async(args=[target_workflow.uuid])
logger.info(
u"Rescheduling of training was successful for workflow with uuid{}".format(target_workflow.uuid)
)
except Exception as ex:
msg = (
u"An unexpected error occurred while scheduling the task for training workflow with UUID {}"
).format(target_workflow.uuid)
logger.exception(msg)
raise reschedule_training_tasks.retry()
def _examples_by_criterion(examples):
"""
......
apps/openassessment/templates/openassessmentblock/staff_debug/staff_debug.html
......@@ -85,6 +85,13 @@
</div>
{% endif %}
{% if display_reschedule_unfinished_tasks %}
<div class="staff-info__status ui-staff__content__section">
<a aria-role="button" href="" id="reschedule_unfinished_tasks" class="action--submit"><span class="copy">{% trans "Reschedule All Unfinished Example Based Assessment Grading Tasks" %}</span></a>
<div id="reschedule_unfinished_tasks_message"></div>
</div>
{% endif %}
<div class="staff-info__student ui-staff__content__section">
<div class="wrapper--input" class="staff-info__student__form">
<form id="openassessment_student_info_form">
......
apps/openassessment/xblock/staff_info_mixin.py
......@@ -6,7 +6,7 @@ import copy
from django.utils.translation import ugettext as _
from xblock.core import XBlock
from openassessment.assessment.errors.ai import AIError
from openassessment.assessment.errors.ai import AIError, AIGradingInternalError, AITrainingInternalError
from openassessment.xblock.resolve_dates import DISTANT_PAST, DISTANT_FUTURE
from openassessment.xblock.data_conversion import create_rubric_dict, convert_training_examples_list_to_dict
from submissions import api as submission_api
......@@ -52,6 +52,11 @@ class StaffInfoMixin(object):
assessment = self.get_assessment_module('example-based-assessment')
context['display_schedule_training'] = self.is_admin and assessment
# Show the reschedule tasks button if the user is an administrator and
# is not in studio preview mode and there exists example based assessment
# as part of the problem definition.
context['display_reschedule_unfinished_tasks'] = self.is_admin and assessment and not self.in_studio_preview
# We need to display the new-style locations in the course staff
# info, even if we're using old-style locations internally,
# so course staff can use the locations to delete student state.
......@@ -82,12 +87,16 @@ class StaffInfoMixin(object):
}
assessment = self.get_assessment_module('example-based-assessment')
student_item_dict = self.get_student_item_dict()
if assessment:
examples = assessment["examples"]
try:
workflow_uuid = ai_api.train_classifiers(
create_rubric_dict(self.prompt, self.rubric_criteria),
convert_training_examples_list_to_dict(examples),
student_item_dict.get('course_id'),
student_item_dict.get('item_id'),
assessment["algorithm_id"]
)
return {
......@@ -179,3 +188,49 @@ class StaffInfoMixin(object):
path = 'openassessmentblock/staff_debug/student_info.html'
return path, context
@XBlock.json_handler
def reschedule_unfinished_tasks(self, data, suffix=''):
"""
Wrapper which invokes the API call for rescheduling grading tasks.
Checks that the requester is an administrator that is not in studio-preview mode,
and that the api-call returns without error. If it returns with an error, (any
exception), the appropriate JSON serializable dictionary with success conditions
is passed back.
Args:
data (not used)
suffix (not used)
Return:
Json serilaizable dict with the following elements:
'success': (bool) Indicates whether or not the tasks were rescheduled successfully
'msg': The response to the server (could be error message or success message)
"""
# Verifies permissions after the push of the button is made
if not self.is_admin or self.in_studio_preview:
return {
'success': False,
'msg': _(u"You do not have permission to reschedule tasks.")
}
# Identifies the course and item that will need to be re-run
student_item_dict = self.get_student_item_dict()
course_id = student_item_dict.get('course_id')
item_id = student_item_dict.get('item_id')
try:
# Note that we only want to recschdule grading tasks, but maintain the potential functionallity
# within the API to also reschedule training tasks.
ai_api.reschedule_unfinished_tasks(course_id=course_id, item_id=item_id, task_type=u"grade")
return {
'success': True,
'msg': _(u"All AI tasks associated with this item have been rescheduled successfully.")
}
except (AIGradingInternalError, AITrainingInternalError, AIError) as ex:
return {
'success': False,
'msg': _(u"An error occurred while rescheduling tasks: {}".format(ex))
}
\ No newline at end of file
apps/openassessment/xblock/static/js/openassessment.min.js
if(typeof OpenAssessment=="undefined"||!OpenAssessment){OpenAssessment={}}if(typeof window.gettext==="undefined"){window.gettext=function(text){return text}}OpenAssessment.BaseView=function(runtime,element,server){this.runtime=runtime;this.element=element;this.server=server;this.responseView=new OpenAssessment.ResponseView(this.element,this.server,this);this.trainingView=new OpenAssessment.StudentTrainingView(this.element,this.server,this);this.selfView=new OpenAssessment.SelfView(this.element,this.server,this);this.peerView=new OpenAssessment.PeerView(this.element,this.server,this);this.gradeView=new OpenAssessment.GradeView(this.element,this.server,this);this.messageView=new OpenAssessment.MessageView(this.element,this.server,this);this.staffInfoView=new OpenAssessment.StaffInfoView(this.element,this.server,this)};OpenAssessment.BaseView.prototype={scrollToTop:function(){if($.scrollTo instanceof Function){$(window).scrollTo($("#openassessment__steps"),800,{offset:-50})}},setUpCollapseExpand:function(parentSel,onExpand){parentSel.find(".ui-toggle-visibility__control").click(function(eventData){var sel=$(eventData.target).closest(".ui-toggle-visibility");if(sel.hasClass("is--collapsed")&&onExpand!==undefined){onExpand()}sel.toggleClass("is--collapsed")})},load:function(){this.responseView.load();this.loadAssessmentModules();this.staffInfoView.load()},loadAssessmentModules:function(){this.trainingView.load();this.peerView.load();this.selfView.load();this.gradeView.load()},loadMessageView:function(){this.messageView.load()},toggleActionError:function(type,msg){var element=this.element;var container=null;if(type=="save"){container=".response__submission__actions"}else if(type=="submit"||type=="peer"||type=="self"||type=="student-training"){container=".step__actions"}else if(type=="feedback_assess"){container=".submission__feedback__actions"}if(container===null){if(msg!==null){console.log(msg)}}else{var msgHtml=msg===null?"":msg;$(container+" .message__content",element).html("<p>"+msgHtml+"</p>");$(container,element).toggleClass("has--error",msg!==null)}},showLoadError:function(step){var container="#openassessment__"+step;$(container).toggleClass("has--error",true);$(container+" .step__status__value i").removeClass().addClass("ico icon-warning-sign");$(container+" .step__status__value .copy").html(gettext("Unable to Load"))}};function OpenAssessmentBlock(runtime,element){$(function($){var server=new OpenAssessment.Server(runtime,element);var view=new OpenAssessment.BaseView(runtime,element,server);view.load()})}OpenAssessment.StudioView=function(runtime,element,server){this.runtime=runtime;this.server=server;this.codeBox=CodeMirror.fromTextArea($(element).find(".openassessment-editor").first().get(0),{mode:"xml",lineNumbers:true,lineWrapping:true});var view=this;$(element).find(".openassessment-save-button").click(function(eventData){view.save()});$(element).find(".openassessment-cancel-button").click(function(eventData){view.cancel()})};OpenAssessment.StudioView.prototype={load:function(){var view=this;this.server.loadXml().done(function(xml){view.codeBox.setValue(xml)}).fail(function(msg){view.showError(msg)})},save:function(){var view=this;this.server.checkReleased().done(function(isReleased){if(isReleased){view.confirmPostReleaseUpdate($.proxy(view.updateXml,view))}else{view.updateXml()}}).fail(function(errMsg){view.showError(msg)})},confirmPostReleaseUpdate:function(onConfirm){var msg=gettext("This problem has already been released. Any changes will apply only to future assessments.");if(confirm(msg)){onConfirm()}},updateXml:function(){this.runtime.notify("save",{state:"start"});var xml=this.codeBox.getValue();var view=this;this.server.updateXml(xml).done(function(){view.runtime.notify("save",{state:"end"});view.load()}).fail(function(msg){view.showError(msg)})},cancel:function(){this.runtime.notify("cancel",{})},showError:function(errorMsg){this.runtime.notify("error",{msg:errorMsg})}};function OpenAssessmentEditor(runtime,element){$(function($){var server=new OpenAssessment.Server(runtime,element);var view=new OpenAssessment.StudioView(runtime,element,server);view.load()})}OpenAssessment.GradeView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView};OpenAssessment.GradeView.prototype={load:function(){var view=this;var baseView=this.baseView;this.server.render("grade").done(function(html){$("#openassessment__grade",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){baseView.showLoadError("grade",errMsg)})},installHandlers:function(){var sel=$("#openassessment__grade",this.element);this.baseView.setUpCollapseExpand(sel);var view=this;sel.find("#feedback__submit").click(function(eventObject){eventObject.preventDefault();view.submitFeedbackOnAssessment()})},feedbackText:function(text){if(typeof text==="undefined"){return $("#feedback__remarks__value",this.element).val()}else{$("#feedback__remarks__value",this.element).val(text)}},feedbackOptions:function(options){var view=this;if(typeof options==="undefined"){return $.map($(".feedback__overall__value:checked",view.element),function(element,index){return $(element).val()})}else{$(".feedback__overall__value",this.element).prop("checked",false);$.each(options,function(index,opt){$("#feedback__overall__value--"+opt,view.element).prop("checked",true)})}},setHidden:function(sel,hidden){sel.toggleClass("is--hidden",hidden);sel.attr("aria-hidden",hidden?"true":"false")},isHidden:function(sel){return sel.hasClass("is--hidden")&&sel.attr("aria-hidden")=="true"},feedbackState:function(newState){var containerSel=$(".submission__feedback__content",this.element);var instructionsSel=containerSel.find(".submission__feedback__instructions");var fieldsSel=containerSel.find(".submission__feedback__fields");var actionsSel=containerSel.find(".submission__feedback__actions");var transitionSel=containerSel.find(".transition__status");var messageSel=containerSel.find(".message--complete");if(typeof newState==="undefined"){var isSubmitting=containerSel.hasClass("is--transitioning")&&containerSel.hasClass("is--submitting")&&!this.isHidden(transitionSel)&&this.isHidden(messageSel)&&this.isHidden(instructionsSel)&&this.isHidden(fieldsSel)&&this.isHidden(actionsSel);var hasSubmitted=containerSel.hasClass("is--submitted")&&this.isHidden(transitionSel)&&!this.isHidden(messageSel)&&this.isHidden(instructionsSel)&&this.isHidden(fieldsSel)&&this.isHidden(actionsSel);var isOpen=!containerSel.hasClass("is--submitted")&&!containerSel.hasClass("is--transitioning")&&!containerSel.hasClass("is--submitting")&&this.isHidden(transitionSel)&&this.isHidden(messageSel)&&!this.isHidden(instructionsSel)&&!this.isHidden(fieldsSel)&&!this.isHidden(actionsSel);if(isOpen){return"open"}else if(isSubmitting){return"submitting"}else if(hasSubmitted){return"submitted"}else{throw"Invalid feedback state"}}else{if(newState=="open"){containerSel.toggleClass("is--transitioning",false);containerSel.toggleClass("is--submitting",false);containerSel.toggleClass("is--submitted",false);this.setHidden(instructionsSel,false);this.setHidden(fieldsSel,false);this.setHidden(actionsSel,false);this.setHidden(transitionSel,true);this.setHidden(messageSel,true)}else if(newState=="submitting"){containerSel.toggleClass("is--transitioning",true);containerSel.toggleClass("is--submitting",true);containerSel.toggleClass("is--submitted",false);this.setHidden(instructionsSel,true);this.setHidden(fieldsSel,true);this.setHidden(actionsSel,true);this.setHidden(transitionSel,false);this.setHidden(messageSel,true)}else if(newState=="submitted"){containerSel.toggleClass("is--transitioning",false);containerSel.toggleClass("is--submitting",false);containerSel.toggleClass("is--submitted",true);this.setHidden(instructionsSel,true);this.setHidden(fieldsSel,true);this.setHidden(actionsSel,true);this.setHidden(transitionSel,true);this.setHidden(messageSel,false)}}},submitFeedbackOnAssessment:function(){var view=this;var baseView=this.baseView;$("#feedback__submit",this.element).toggleClass("is--disabled",true);view.feedbackState("submitting");this.server.submitFeedbackOnAssessment(this.feedbackText(),this.feedbackOptions()).done(function(){view.feedbackState("submitted")}).fail(function(errMsg){baseView.toggleActionError("feedback_assess",errMsg)})}};OpenAssessment.MessageView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView};OpenAssessment.MessageView.prototype={load:function(){var view=this;var baseView=this.baseView;this.server.render("message").done(function(html){$("#openassessment__message",view.element).replaceWith(html)}).fail(function(errMsg){baseView.showLoadError("message",errMsg)})}};OpenAssessment.PeerView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.rubric=null};OpenAssessment.PeerView.prototype={load:function(){var view=this;this.server.render("peer_assessment").done(function(html){$("#openassessment__peer-assessment",view.element).replaceWith(html);view.installHandlers(false)}).fail(function(errMsg){view.baseView.showLoadError("peer-assessment")});view.baseView.loadMessageView()},loadContinuedAssessment:function(){var view=this;this.server.renderContinuedPeer().done(function(html){$("#openassessment__peer-assessment",view.element).replaceWith(html);view.installHandlers(true)}).fail(function(errMsg){view.baseView.showLoadError("peer-assessment")})},installHandlers:function(isContinuedAssessment){var sel=$("#openassessment__peer-assessment",this.element);var view=this;this.baseView.setUpCollapseExpand(sel,$.proxy(view.loadContinuedAssessment,view));var rubricSelector=$("#peer-assessment--001__assessment",this.element);if(rubricSelector.size()>0){var rubricElement=rubricSelector.get(0);this.rubric=new OpenAssessment.Rubric(rubricElement)}if(this.rubric!==null){this.rubric.canSubmitCallback($.proxy(view.peerSubmitEnabled,view))}sel.find("#peer-assessment--001__assessment__submit").click(function(eventObject){eventObject.preventDefault();if(!isContinuedAssessment){view.peerAssess()}else{view.continuedPeerAssess()}})},peerSubmitEnabled:function(enabled){var button=$("#peer-assessment--001__assessment__submit",this.element);if(typeof enabled==="undefined"){return!button.hasClass("is--disabled")}else{button.toggleClass("is--disabled",!enabled)}},peerAssess:function(){var view=this;var baseView=view.baseView;this.peerAssessRequest(function(){view.load();baseView.loadAssessmentModules();baseView.scrollToTop()})},continuedPeerAssess:function(){var view=this;var gradeView=this.baseView.gradeView;var baseView=view.baseView;view.peerAssessRequest(function(){view.loadContinuedAssessment();gradeView.load();baseView.scrollToTop()})},peerAssessRequest:function(successFunction){var view=this;view.baseView.toggleActionError("peer",null);view.peerSubmitEnabled(false);this.server.peerAssess(this.rubric.optionsSelected(),this.rubric.criterionFeedback(),this.overallFeedback()).done(successFunction).fail(function(errMsg){view.baseView.toggleActionError("peer",errMsg);view.peerSubmitEnabled(true)})},overallFeedback:function(overallFeedback){var selector="#assessment__rubric__question--feedback__value";if(typeof overallFeedback==="undefined"){return $(selector,this.element).val()}else{$(selector,this.element).val(overallFeedback)}}};OpenAssessment.ResponseView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.savedResponse="";this.lastChangeTime=Date.now();this.errorOnLastSave=false;this.autoSaveTimerId=null};OpenAssessment.ResponseView.prototype={AUTO_SAVE_POLL_INTERVAL:2e3,AUTO_SAVE_WAIT:3e4,load:function(){var view=this;this.server.render("submission").done(function(html){$("#openassessment__response",view.element).replaceWith(html);view.installHandlers();view.setAutoSaveEnabled(true)}).fail(function(errMsg){view.baseView.showLoadError("response")})},installHandlers:function(){var sel=$("#openassessment__response",this.element);var view=this;this.baseView.setUpCollapseExpand(sel);this.savedResponse=this.response();var handleChange=function(eventData){view.handleResponseChanged()};sel.find("#submission__answer__value").on("change keyup drop paste",handleChange);sel.find("#step--response__submit").click(function(eventObject){eventObject.preventDefault();view.submit()});sel.find("#submission__save").click(function(eventObject){eventObject.preventDefault();view.save()})},setAutoSaveEnabled:function(enabled){if(enabled){if(this.autoSaveTimerId===null){this.autoSaveTimerId=setInterval($.proxy(this.autoSave,this),this.AUTO_SAVE_POLL_INTERVAL)}}else{if(this.autoSaveTimerId!==null){clearInterval(this.autoSaveTimerId)}}},submitEnabled:function(enabled){var sel=$("#step--response__submit",this.element);if(typeof enabled==="undefined"){return!sel.hasClass("is--disabled")}else{sel.toggleClass("is--disabled",!enabled)}},saveEnabled:function(enabled){var sel=$("#submission__save",this.element);if(typeof enabled==="undefined"){return!sel.hasClass("is--disabled")}else{sel.toggleClass("is--disabled",!enabled)}},saveStatus:function(msg){var sel=$("#response__save_status h3",this.element);if(typeof msg==="undefined"){return sel.text()}else{var label=gettext("Status of Your Response");sel.html('<span class="sr">'+label+":"+"</span>\n"+msg)}},unsavedWarningEnabled:function(enabled){if(typeof enabled==="undefined"){return window.onbeforeunload!==null}else{if(enabled){window.onbeforeunload=function(){return"If you leave this page without saving or submitting your response, "+"you'll lose any work you've done on the response."}}else{window.onbeforeunload=null}}},response:function(text){var sel=$("#submission__answer__value",this.element);if(typeof text==="undefined"){return sel.val()}else{sel.val(text)}},responseChanged:function(){var currentResponse=$.trim(this.response());var savedResponse=$.trim(this.savedResponse);return savedResponse!==currentResponse},autoSave:function(){var timeSinceLastChange=Date.now()-this.lastChangeTime;if(this.responseChanged()&&timeSinceLastChange>this.AUTO_SAVE_WAIT&&!this.errorOnLastSave){this.save()}},handleResponseChanged:function(){var isBlank=$.trim(this.response())!=="";this.submitEnabled(isBlank);if(this.responseChanged()){this.saveEnabled(isBlank);this.saveStatus(gettext("This response has not been saved."));this.unsavedWarningEnabled(true)}this.lastChangeTime=Date.now()},save:function(){this.errorOnLastSave=false;this.saveStatus(gettext("Saving..."));this.baseView.toggleActionError("save",null);this.unsavedWarningEnabled(false);var view=this;var savedResponse=this.response();this.server.save(savedResponse).done(function(){view.savedResponse=savedResponse;var currentResponse=view.response();view.submitEnabled(currentResponse!=="");if(currentResponse==savedResponse){view.saveEnabled(false);view.saveStatus(gettext("This response has been saved but not submitted."))}}).fail(function(errMsg){view.saveStatus(gettext("Error"));view.baseView.toggleActionError("save",errMsg);view.errorOnLastSave=true})},submit:function(){this.submitEnabled(false);var view=this;var baseView=this.baseView;this.confirmSubmission().pipe(function(){var submission=$("#submission__answer__value",view.element).val();baseView.toggleActionError("response",null);return view.server.submit(submission)}).done($.proxy(view.moveToNextStep,view)).fail(function(errCode,errMsg){if(errCode=="ENOMULTI"){view.moveToNextStep()}else{if(errMsg){baseView.toggleActionError("submit",errMsg)}view.submitEnabled(true)}})},moveToNextStep:function(){this.load();this.baseView.loadAssessmentModules();this.unsavedWarningEnabled(false)},confirmSubmission:function(){var msg="You're about to submit your response for this assignment. "+"After you submit this response, you can't change it or submit a new response.";return $.Deferred(function(defer){if(confirm(msg)){defer.resolve()}else{defer.reject()}})}};OpenAssessment.Rubric=function(element){this.element=element};OpenAssessment.Rubric.prototype={criterionFeedback:function(criterionFeedback){var selector="textarea.answer__value";var feedback={};$(selector,this.element).each(function(index,sel){if(typeof criterionFeedback!=="undefined"){$(sel).val(criterionFeedback[sel.name]);feedback[sel.name]=criterionFeedback[sel.name]}else{feedback[sel.name]=$(sel).val()}});return feedback},optionsSelected:function(optionsSelected){var selector="input[type=radio]";if(typeof optionsSelected==="undefined"){var options={};$(selector+":checked",this.element).each(function(index,sel){options[sel.name]=sel.value});return options}else{$(selector,this.element).prop("checked",false);$(selector,this.element).each(function(index,sel){if(optionsSelected.hasOwnProperty(sel.name)){if(sel.value==optionsSelected[sel.name]){$(sel).prop("checked",true)}}})}},canSubmitCallback:function(callback){$(this.element).change(function(){var numChecked=$("input[type=radio]:checked",this).length;var numAvailable=$(".field--radio.assessment__rubric__question",this).length;var canSubmit=numChecked==numAvailable;callback(canSubmit)})},showCorrections:function(corrections){var selector="input[type=radio]";var hasErrors=false;$(selector,this.element).each(function(index,sel){var listItem=$(sel).parents(".assessment__rubric__question");if(corrections.hasOwnProperty(sel.name)){hasErrors=true;listItem.find(".message--incorrect").removeClass("is--hidden");listItem.find(".message--correct").addClass("is--hidden")}else{listItem.find(".message--correct").removeClass("is--hidden");listItem.find(".message--incorrect").addClass("is--hidden")}});return hasErrors}};OpenAssessment.SelfView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.rubric=null};OpenAssessment.SelfView.prototype={load:function(){var view=this;this.server.render("self_assessment").done(function(html){$("#openassessment__self-assessment",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){view.showLoadError("self-assessment")})},installHandlers:function(){var view=this;var sel=$("#openassessment__self-assessment",view.element);this.baseView.setUpCollapseExpand(sel);var rubricSelector=$("#self-assessment--001__assessment",this.element);if(rubricSelector.size()>0){var rubricElement=rubricSelector.get(0);this.rubric=new OpenAssessment.Rubric(rubricElement)}if(this.rubric!==null){this.rubric.canSubmitCallback($.proxy(this.selfSubmitEnabled,this))}sel.find("#self-assessment--001__assessment__submit").click(function(eventObject){eventObject.preventDefault();view.selfAssess()})},selfSubmitEnabled:function(enabled){var button=$("#self-assessment--001__assessment__submit",this.element);if(typeof enabled==="undefined"){return!button.hasClass("is--disabled")}else{button.toggleClass("is--disabled",!enabled)}},selfAssess:function(){var view=this;var baseView=this.baseView;baseView.toggleActionError("self",null);view.selfSubmitEnabled(false);var options=this.rubric.optionsSelected();this.server.selfAssess(options).done(function(){baseView.loadAssessmentModules();baseView.scrollToTop()}).fail(function(errMsg){baseView.toggleActionError("self",errMsg);view.selfSubmitEnabled(true)})}};OpenAssessment.Server=function(runtime,element){this.runtime=runtime;this.element=element};OpenAssessment.Server.prototype={url:function(handler){return this.runtime.handlerUrl(this.element,handler)},render:function(component){var url=this.url("render_"+component);return $.Deferred(function(defer){$.ajax({url:url,type:"POST",dataType:"html"}).done(function(data){defer.resolveWith(this,[data])}).fail(function(data){defer.rejectWith(this,[gettext("This section could not be loaded.")])})}).promise()},renderContinuedPeer:function(){var url=this.url("render_peer_assessment");return $.Deferred(function(defer){$.ajax({url:url,type:"POST",dataType:"html",data:{continue_grading:true}}).done(function(data){defer.resolveWith(this,[data])}).fail(function(data){defer.rejectWith(this,[gettext("This section could not be loaded.")])})}).promise()},studentInfo:function(student_id){var url=this.url("render_student_info");return $.Deferred(function(defer){$.ajax({url:url,type:"POST",dataType:"html",data:{student_id:student_id}}).done(function(data){defer.resolveWith(this,[data])}).fail(function(data){defer.rejectWith(this,[gettext("This section could not be loaded.")])})}).promise()},submit:function(submission){var url=this.url("submit");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:JSON.stringify({submission:submission})}).done(function(data){var success=data[0];if(success){var studentId=data[1];var attemptNum=data[2];defer.resolveWith(this,[studentId,attemptNum])}else{var errorNum=data[1];var errorMsg=data[2];defer.rejectWith(this,[errorNum,errorMsg])}}).fail(function(data){defer.rejectWith(this,["AJAX",gettext("This response could not be submitted.")])})}).promise()},save:function(submission){var url=this.url("save_submission");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:JSON.stringify({submission:submission})}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This response could not be saved.")])})}).promise()},submitFeedbackOnAssessment:function(text,options){var url=this.url("submit_feedback");var payload=JSON.stringify({feedback_text:text,feedback_options:options});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This feedback could not be submitted.")])})}).promise()},peerAssess:function(optionsSelected,criterionFeedback,overallFeedback){var url=this.url("peer_assess");var payload=JSON.stringify({options_selected:optionsSelected,criterion_feedback:criterionFeedback,overall_feedback:overallFeedback});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})}).promise()},selfAssess:function(optionsSelected){var url=this.url("self_assess");var payload=JSON.stringify({options_selected:optionsSelected});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})})},trainingAssess:function(optionsSelected){var url=this.url("training_assess");var payload=JSON.stringify({options_selected:optionsSelected});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolveWith(this,[data.corrections])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})})},scheduleTraining:function(){var url=this.url("schedule_training");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:'""'}).done(function(data){if(data.success){defer.resolveWith(this,[data.msg])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})})},loadXml:function(){var url=this.url("xml");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:'""'}).done(function(data){if(data.success){defer.resolveWith(this,[data.xml])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This problem could not be loaded.")])})}).promise()},updateXml:function(xml){var url=this.url("update_xml");var payload=JSON.stringify({xml:xml});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This problem could not be saved.")])})}).promise()},checkReleased:function(){var url=this.url("check_released");var payload='""';return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolveWith(this,[data.is_released])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("The server could not be contacted.")])})}).promise()}};if(typeof OpenAssessment=="undefined"||!OpenAssessment){OpenAssessment={}}if(typeof window.gettext==="undefined"){window.gettext=function(text){return text}}OpenAssessment.StaffInfoView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView};OpenAssessment.StaffInfoView.prototype={load:function(){var view=this;if($("#openassessment__staff-info",view.element).length>0){this.server.render("staff_info").done(function(html){$("#openassessment__staff-info",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){view.baseView.showLoadError("staff_info")})}},loadStudentInfo:function(){var view=this;var sel=$("#openassessment__staff-info",this.element);var student_id=sel.find("#openassessment__student_id").val();this.server.studentInfo(student_id).done(function(html){$("#openassessment__student-info",view.element).replaceWith(html)}).fail(function(errMsg){view.showLoadError("student_info")})},installHandlers:function(){var sel=$("#openassessment__staff-info",this.element);var view=this;if(sel.length<=0){return}this.baseView.setUpCollapseExpand(sel,function(){});sel.find("#openassessment_student_info_form").submit(function(eventObject){eventObject.preventDefault();view.loadStudentInfo()});sel.find("#submit_student_id").click(function(eventObject){eventObject.preventDefault();view.loadStudentInfo()});sel.find("#schedule_training").click(function(eventObject){eventObject.preventDefault();view.scheduleTraining()})},scheduleTraining:function(){var view=this;this.server.scheduleTraining().done(function(msg){$("#schedule_training_message",this.element).text(msg)}).fail(function(errMsg){$("#schedule_training_message",this.element).text(errMsg)})}};OpenAssessment.StudentTrainingView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.rubric=null};OpenAssessment.StudentTrainingView.prototype={load:function(){var view=this;this.server.render("student_training").done(function(html){$("#openassessment__student-training",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){view.baseView.showLoadError("student-training")})},installHandlers:function(){var sel=$("#openassessment__student-training",this.element);var view=this;this.baseView.setUpCollapseExpand(sel);var rubricSelector=$("#student-training--001__assessment",this.element);if(rubricSelector.size()>0){var rubricElement=rubricSelector.get(0);this.rubric=new OpenAssessment.Rubric(rubricElement)}if(this.rubric!==null){this.rubric.canSubmitCallback($.proxy(this.assessButtonEnabled,this))}sel.find("#student-training--001__assessment__submit").click(function(eventObject){eventObject.preventDefault();view.assess()})},assess:function(){this.assessButtonEnabled(false);var options={};if(this.rubric!==null){options=this.rubric.optionsSelected()}var view=this;var baseView=this.baseView;this.server.trainingAssess(options).done(function(corrections){var incorrect=$("#openassessment__student-training--incorrect",this.element);var instructions=$("#openassessment__student-training--instructions",this.element);if(!view.rubric.showCorrections(corrections)){baseView.loadAssessmentModules();incorrect.addClass("is--hidden");instructions.removeClass("is--hidden")}else{instructions.addClass("is--hidden");incorrect.removeClass("is--hidden")}baseView.scrollToTop()}).fail(function(errMsg){baseView.toggleActionError("student-training",errMsg);view.assessButtonEnabled(true)})},assessButtonEnabled:function(isEnabled){var button=$("#student-training--001__assessment__submit",this.element);if(typeof isEnabled==="undefined"){return!button.hasClass("is--disabled")}else{button.toggleClass("is--disabled",!isEnabled)}}};
\ No newline at end of file
if(typeof OpenAssessment=="undefined"||!OpenAssessment){OpenAssessment={}}if(typeof window.gettext==="undefined"){window.gettext=function(text){return text}}OpenAssessment.BaseView=function(runtime,element,server){this.runtime=runtime;this.element=element;this.server=server;this.responseView=new OpenAssessment.ResponseView(this.element,this.server,this);this.trainingView=new OpenAssessment.StudentTrainingView(this.element,this.server,this);this.selfView=new OpenAssessment.SelfView(this.element,this.server,this);this.peerView=new OpenAssessment.PeerView(this.element,this.server,this);this.gradeView=new OpenAssessment.GradeView(this.element,this.server,this);this.messageView=new OpenAssessment.MessageView(this.element,this.server,this);this.staffInfoView=new OpenAssessment.StaffInfoView(this.element,this.server,this)};OpenAssessment.BaseView.prototype={scrollToTop:function(){if($.scrollTo instanceof Function){$(window).scrollTo($("#openassessment__steps"),800,{offset:-50})}},setUpCollapseExpand:function(parentSel,onExpand){parentSel.find(".ui-toggle-visibility__control").click(function(eventData){var sel=$(eventData.target).closest(".ui-toggle-visibility");if(sel.hasClass("is--collapsed")&&onExpand!==undefined){onExpand()}sel.toggleClass("is--collapsed")})},load:function(){this.responseView.load();this.loadAssessmentModules();this.staffInfoView.load()},loadAssessmentModules:function(){this.trainingView.load();this.peerView.load();this.selfView.load();this.gradeView.load()},loadMessageView:function(){this.messageView.load()},toggleActionError:function(type,msg){var element=this.element;var container=null;if(type=="save"){container=".response__submission__actions"}else if(type=="submit"||type=="peer"||type=="self"||type=="student-training"){container=".step__actions"}else if(type=="feedback_assess"){container=".submission__feedback__actions"}if(container===null){if(msg!==null){console.log(msg)}}else{var msgHtml=msg===null?"":msg;$(container+" .message__content",element).html("<p>"+msgHtml+"</p>");$(container,element).toggleClass("has--error",msg!==null)}},showLoadError:function(step){var container="#openassessment__"+step;$(container).toggleClass("has--error",true);$(container+" .step__status__value i").removeClass().addClass("ico icon-warning-sign");$(container+" .step__status__value .copy").html(gettext("Unable to Load"))}};function OpenAssessmentBlock(runtime,element){$(function($){var server=new OpenAssessment.Server(runtime,element);var view=new OpenAssessment.BaseView(runtime,element,server);view.load()})}OpenAssessment.StudioView=function(runtime,element,server){this.runtime=runtime;this.server=server;this.codeBox=CodeMirror.fromTextArea($(element).find(".openassessment-editor").first().get(0),{mode:"xml",lineNumbers:true,lineWrapping:true});var view=this;$(element).find(".openassessment-save-button").click(function(eventData){view.save()});$(element).find(".openassessment-cancel-button").click(function(eventData){view.cancel()})};OpenAssessment.StudioView.prototype={load:function(){var view=this;this.server.loadXml().done(function(xml){view.codeBox.setValue(xml)}).fail(function(msg){view.showError(msg)})},save:function(){var view=this;this.server.checkReleased().done(function(isReleased){if(isReleased){view.confirmPostReleaseUpdate($.proxy(view.updateXml,view))}else{view.updateXml()}}).fail(function(errMsg){view.showError(msg)})},confirmPostReleaseUpdate:function(onConfirm){var msg=gettext("This problem has already been released. Any changes will apply only to future assessments.");if(confirm(msg)){onConfirm()}},updateXml:function(){this.runtime.notify("save",{state:"start"});var xml=this.codeBox.getValue();var view=this;this.server.updateXml(xml).done(function(){view.runtime.notify("save",{state:"end"});view.load()}).fail(function(msg){view.showError(msg)})},cancel:function(){this.runtime.notify("cancel",{})},showError:function(errorMsg){this.runtime.notify("error",{msg:errorMsg})}};function OpenAssessmentEditor(runtime,element){$(function($){var server=new OpenAssessment.Server(runtime,element);var view=new OpenAssessment.StudioView(runtime,element,server);view.load()})}OpenAssessment.GradeView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView};OpenAssessment.GradeView.prototype={load:function(){var view=this;var baseView=this.baseView;this.server.render("grade").done(function(html){$("#openassessment__grade",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){baseView.showLoadError("grade",errMsg)})},installHandlers:function(){var sel=$("#openassessment__grade",this.element);this.baseView.setUpCollapseExpand(sel);var view=this;sel.find("#feedback__submit").click(function(eventObject){eventObject.preventDefault();view.submitFeedbackOnAssessment()})},feedbackText:function(text){if(typeof text==="undefined"){return $("#feedback__remarks__value",this.element).val()}else{$("#feedback__remarks__value",this.element).val(text)}},feedbackOptions:function(options){var view=this;if(typeof options==="undefined"){return $.map($(".feedback__overall__value:checked",view.element),function(element,index){return $(element).val()})}else{$(".feedback__overall__value",this.element).prop("checked",false);$.each(options,function(index,opt){$("#feedback__overall__value--"+opt,view.element).prop("checked",true)})}},setHidden:function(sel,hidden){sel.toggleClass("is--hidden",hidden);sel.attr("aria-hidden",hidden?"true":"false")},isHidden:function(sel){return sel.hasClass("is--hidden")&&sel.attr("aria-hidden")=="true"},feedbackState:function(newState){var containerSel=$(".submission__feedback__content",this.element);var instructionsSel=containerSel.find(".submission__feedback__instructions");var fieldsSel=containerSel.find(".submission__feedback__fields");var actionsSel=containerSel.find(".submission__feedback__actions");var transitionSel=containerSel.find(".transition__status");var messageSel=containerSel.find(".message--complete");if(typeof newState==="undefined"){var isSubmitting=containerSel.hasClass("is--transitioning")&&containerSel.hasClass("is--submitting")&&!this.isHidden(transitionSel)&&this.isHidden(messageSel)&&this.isHidden(instructionsSel)&&this.isHidden(fieldsSel)&&this.isHidden(actionsSel);var hasSubmitted=containerSel.hasClass("is--submitted")&&this.isHidden(transitionSel)&&!this.isHidden(messageSel)&&this.isHidden(instructionsSel)&&this.isHidden(fieldsSel)&&this.isHidden(actionsSel);var isOpen=!containerSel.hasClass("is--submitted")&&!containerSel.hasClass("is--transitioning")&&!containerSel.hasClass("is--submitting")&&this.isHidden(transitionSel)&&this.isHidden(messageSel)&&!this.isHidden(instructionsSel)&&!this.isHidden(fieldsSel)&&!this.isHidden(actionsSel);if(isOpen){return"open"}else if(isSubmitting){return"submitting"}else if(hasSubmitted){return"submitted"}else{throw"Invalid feedback state"}}else{if(newState=="open"){containerSel.toggleClass("is--transitioning",false);containerSel.toggleClass("is--submitting",false);containerSel.toggleClass("is--submitted",false);this.setHidden(instructionsSel,false);this.setHidden(fieldsSel,false);this.setHidden(actionsSel,false);this.setHidden(transitionSel,true);this.setHidden(messageSel,true)}else if(newState=="submitting"){containerSel.toggleClass("is--transitioning",true);containerSel.toggleClass("is--submitting",true);containerSel.toggleClass("is--submitted",false);this.setHidden(instructionsSel,true);this.setHidden(fieldsSel,true);this.setHidden(actionsSel,true);this.setHidden(transitionSel,false);this.setHidden(messageSel,true)}else if(newState=="submitted"){containerSel.toggleClass("is--transitioning",false);containerSel.toggleClass("is--submitting",false);containerSel.toggleClass("is--submitted",true);this.setHidden(instructionsSel,true);this.setHidden(fieldsSel,true);this.setHidden(actionsSel,true);this.setHidden(transitionSel,true);this.setHidden(messageSel,false)}}},submitFeedbackOnAssessment:function(){var view=this;var baseView=this.baseView;$("#feedback__submit",this.element).toggleClass("is--disabled",true);view.feedbackState("submitting");this.server.submitFeedbackOnAssessment(this.feedbackText(),this.feedbackOptions()).done(function(){view.feedbackState("submitted")}).fail(function(errMsg){baseView.toggleActionError("feedback_assess",errMsg)})}};OpenAssessment.MessageView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView};OpenAssessment.MessageView.prototype={load:function(){var view=this;var baseView=this.baseView;this.server.render("message").done(function(html){$("#openassessment__message",view.element).replaceWith(html)}).fail(function(errMsg){baseView.showLoadError("message",errMsg)})}};OpenAssessment.PeerView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.rubric=null};OpenAssessment.PeerView.prototype={load:function(){var view=this;this.server.render("peer_assessment").done(function(html){$("#openassessment__peer-assessment",view.element).replaceWith(html);view.installHandlers(false)}).fail(function(errMsg){view.baseView.showLoadError("peer-assessment")});view.baseView.loadMessageView()},loadContinuedAssessment:function(){var view=this;this.server.renderContinuedPeer().done(function(html){$("#openassessment__peer-assessment",view.element).replaceWith(html);view.installHandlers(true)}).fail(function(errMsg){view.baseView.showLoadError("peer-assessment")})},installHandlers:function(isContinuedAssessment){var sel=$("#openassessment__peer-assessment",this.element);var view=this;this.baseView.setUpCollapseExpand(sel,$.proxy(view.loadContinuedAssessment,view));var rubricSelector=$("#peer-assessment--001__assessment",this.element);if(rubricSelector.size()>0){var rubricElement=rubricSelector.get(0);this.rubric=new OpenAssessment.Rubric(rubricElement)}if(this.rubric!==null){this.rubric.canSubmitCallback($.proxy(view.peerSubmitEnabled,view))}sel.find("#peer-assessment--001__assessment__submit").click(function(eventObject){eventObject.preventDefault();if(!isContinuedAssessment){view.peerAssess()}else{view.continuedPeerAssess()}})},peerSubmitEnabled:function(enabled){var button=$("#peer-assessment--001__assessment__submit",this.element);if(typeof enabled==="undefined"){return!button.hasClass("is--disabled")}else{button.toggleClass("is--disabled",!enabled)}},peerAssess:function(){var view=this;var baseView=view.baseView;this.peerAssessRequest(function(){view.load();baseView.loadAssessmentModules();baseView.scrollToTop()})},continuedPeerAssess:function(){var view=this;var gradeView=this.baseView.gradeView;var baseView=view.baseView;view.peerAssessRequest(function(){view.loadContinuedAssessment();gradeView.load();baseView.scrollToTop()})},peerAssessRequest:function(successFunction){var view=this;view.baseView.toggleActionError("peer",null);view.peerSubmitEnabled(false);this.server.peerAssess(this.rubric.optionsSelected(),this.rubric.criterionFeedback(),this.overallFeedback()).done(successFunction).fail(function(errMsg){view.baseView.toggleActionError("peer",errMsg);view.peerSubmitEnabled(true)})},overallFeedback:function(overallFeedback){var selector="#assessment__rubric__question--feedback__value";if(typeof overallFeedback==="undefined"){return $(selector,this.element).val()}else{$(selector,this.element).val(overallFeedback)}}};OpenAssessment.ResponseView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.savedResponse="";this.lastChangeTime=Date.now();this.errorOnLastSave=false;this.autoSaveTimerId=null};OpenAssessment.ResponseView.prototype={AUTO_SAVE_POLL_INTERVAL:2e3,AUTO_SAVE_WAIT:3e4,load:function(){var view=this;this.server.render("submission").done(function(html){$("#openassessment__response",view.element).replaceWith(html);view.installHandlers();view.setAutoSaveEnabled(true)}).fail(function(errMsg){view.baseView.showLoadError("response")})},installHandlers:function(){var sel=$("#openassessment__response",this.element);var view=this;this.baseView.setUpCollapseExpand(sel);this.savedResponse=this.response();var handleChange=function(eventData){view.handleResponseChanged()};sel.find("#submission__answer__value").on("change keyup drop paste",handleChange);sel.find("#step--response__submit").click(function(eventObject){eventObject.preventDefault();view.submit()});sel.find("#submission__save").click(function(eventObject){eventObject.preventDefault();view.save()})},setAutoSaveEnabled:function(enabled){if(enabled){if(this.autoSaveTimerId===null){this.autoSaveTimerId=setInterval($.proxy(this.autoSave,this),this.AUTO_SAVE_POLL_INTERVAL)}}else{if(this.autoSaveTimerId!==null){clearInterval(this.autoSaveTimerId)}}},submitEnabled:function(enabled){var sel=$("#step--response__submit",this.element);if(typeof enabled==="undefined"){return!sel.hasClass("is--disabled")}else{sel.toggleClass("is--disabled",!enabled)}},saveEnabled:function(enabled){var sel=$("#submission__save",this.element);if(typeof enabled==="undefined"){return!sel.hasClass("is--disabled")}else{sel.toggleClass("is--disabled",!enabled)}},saveStatus:function(msg){var sel=$("#response__save_status h3",this.element);if(typeof msg==="undefined"){return sel.text()}else{var label=gettext("Status of Your Response");sel.html('<span class="sr">'+label+":"+"</span>\n"+msg)}},unsavedWarningEnabled:function(enabled){if(typeof enabled==="undefined"){return window.onbeforeunload!==null}else{if(enabled){window.onbeforeunload=function(){return"If you leave this page without saving or submitting your response, "+"you'll lose any work you've done on the response."}}else{window.onbeforeunload=null}}},response:function(text){var sel=$("#submission__answer__value",this.element);if(typeof text==="undefined"){return sel.val()}else{sel.val(text)}},responseChanged:function(){var currentResponse=$.trim(this.response());var savedResponse=$.trim(this.savedResponse);return savedResponse!==currentResponse},autoSave:function(){var timeSinceLastChange=Date.now()-this.lastChangeTime;if(this.responseChanged()&&timeSinceLastChange>this.AUTO_SAVE_WAIT&&!this.errorOnLastSave){this.save()}},handleResponseChanged:function(){var isBlank=$.trim(this.response())!=="";this.submitEnabled(isBlank);if(this.responseChanged()){this.saveEnabled(isBlank);this.saveStatus(gettext("This response has not been saved."));this.unsavedWarningEnabled(true)}this.lastChangeTime=Date.now()},save:function(){this.errorOnLastSave=false;this.saveStatus(gettext("Saving..."));this.baseView.toggleActionError("save",null);this.unsavedWarningEnabled(false);var view=this;var savedResponse=this.response();this.server.save(savedResponse).done(function(){view.savedResponse=savedResponse;var currentResponse=view.response();view.submitEnabled(currentResponse!=="");if(currentResponse==savedResponse){view.saveEnabled(false);view.saveStatus(gettext("This response has been saved but not submitted."))}}).fail(function(errMsg){view.saveStatus(gettext("Error"));view.baseView.toggleActionError("save",errMsg);view.errorOnLastSave=true})},submit:function(){this.submitEnabled(false);var view=this;var baseView=this.baseView;this.confirmSubmission().pipe(function(){var submission=$("#submission__answer__value",view.element).val();baseView.toggleActionError("response",null);return view.server.submit(submission)}).done($.proxy(view.moveToNextStep,view)).fail(function(errCode,errMsg){if(errCode=="ENOMULTI"){view.moveToNextStep()}else{if(errMsg){baseView.toggleActionError("submit",errMsg)}view.submitEnabled(true)}})},moveToNextStep:function(){this.load();this.baseView.loadAssessmentModules();this.unsavedWarningEnabled(false)},confirmSubmission:function(){var msg="You're about to submit your response for this assignment. "+"After you submit this response, you can't change it or submit a new response.";return $.Deferred(function(defer){if(confirm(msg)){defer.resolve()}else{defer.reject()}})}};OpenAssessment.Rubric=function(element){this.element=element};OpenAssessment.Rubric.prototype={criterionFeedback:function(criterionFeedback){var selector="textarea.answer__value";var feedback={};$(selector,this.element).each(function(index,sel){if(typeof criterionFeedback!=="undefined"){$(sel).val(criterionFeedback[sel.name]);feedback[sel.name]=criterionFeedback[sel.name]}else{feedback[sel.name]=$(sel).val()}});return feedback},optionsSelected:function(optionsSelected){var selector="input[type=radio]";if(typeof optionsSelected==="undefined"){var options={};$(selector+":checked",this.element).each(function(index,sel){options[sel.name]=sel.value});return options}else{$(selector,this.element).prop("checked",false);$(selector,this.element).each(function(index,sel){if(optionsSelected.hasOwnProperty(sel.name)){if(sel.value==optionsSelected[sel.name]){$(sel).prop("checked",true)}}})}},canSubmitCallback:function(callback){$(this.element).change(function(){var numChecked=$("input[type=radio]:checked",this).length;var numAvailable=$(".field--radio.assessment__rubric__question",this).length;var canSubmit=numChecked==numAvailable;callback(canSubmit)})},showCorrections:function(corrections){var selector="input[type=radio]";var hasErrors=false;$(selector,this.element).each(function(index,sel){var listItem=$(sel).parents(".assessment__rubric__question");if(corrections.hasOwnProperty(sel.name)){hasErrors=true;listItem.find(".message--incorrect").removeClass("is--hidden");listItem.find(".message--correct").addClass("is--hidden")}else{listItem.find(".message--correct").removeClass("is--hidden");listItem.find(".message--incorrect").addClass("is--hidden")}});return hasErrors}};OpenAssessment.SelfView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.rubric=null};OpenAssessment.SelfView.prototype={load:function(){var view=this;this.server.render("self_assessment").done(function(html){$("#openassessment__self-assessment",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){view.showLoadError("self-assessment")})},installHandlers:function(){var view=this;var sel=$("#openassessment__self-assessment",view.element);this.baseView.setUpCollapseExpand(sel);var rubricSelector=$("#self-assessment--001__assessment",this.element);if(rubricSelector.size()>0){var rubricElement=rubricSelector.get(0);this.rubric=new OpenAssessment.Rubric(rubricElement)}if(this.rubric!==null){this.rubric.canSubmitCallback($.proxy(this.selfSubmitEnabled,this))}sel.find("#self-assessment--001__assessment__submit").click(function(eventObject){eventObject.preventDefault();view.selfAssess()})},selfSubmitEnabled:function(enabled){var button=$("#self-assessment--001__assessment__submit",this.element);if(typeof enabled==="undefined"){return!button.hasClass("is--disabled")}else{button.toggleClass("is--disabled",!enabled)}},selfAssess:function(){var view=this;var baseView=this.baseView;baseView.toggleActionError("self",null);view.selfSubmitEnabled(false);var options=this.rubric.optionsSelected();this.server.selfAssess(options).done(function(){baseView.loadAssessmentModules();baseView.scrollToTop()}).fail(function(errMsg){baseView.toggleActionError("self",errMsg);view.selfSubmitEnabled(true)})}};OpenAssessment.Server=function(runtime,element){this.runtime=runtime;this.element=element};OpenAssessment.Server.prototype={url:function(handler){return this.runtime.handlerUrl(this.element,handler)},render:function(component){var url=this.url("render_"+component);return $.Deferred(function(defer){$.ajax({url:url,type:"POST",dataType:"html"}).done(function(data){defer.resolveWith(this,[data])}).fail(function(data){defer.rejectWith(this,[gettext("This section could not be loaded.")])})}).promise()},renderContinuedPeer:function(){var url=this.url("render_peer_assessment");return $.Deferred(function(defer){$.ajax({url:url,type:"POST",dataType:"html",data:{continue_grading:true}}).done(function(data){defer.resolveWith(this,[data])}).fail(function(data){defer.rejectWith(this,[gettext("This section could not be loaded.")])})}).promise()},studentInfo:function(student_id){var url=this.url("render_student_info");return $.Deferred(function(defer){$.ajax({url:url,type:"POST",dataType:"html",data:{student_id:student_id}}).done(function(data){defer.resolveWith(this,[data])}).fail(function(data){defer.rejectWith(this,[gettext("This section could not be loaded.")])})}).promise()},submit:function(submission){var url=this.url("submit");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:JSON.stringify({submission:submission})}).done(function(data){var success=data[0];if(success){var studentId=data[1];var attemptNum=data[2];defer.resolveWith(this,[studentId,attemptNum])}else{var errorNum=data[1];var errorMsg=data[2];defer.rejectWith(this,[errorNum,errorMsg])}}).fail(function(data){defer.rejectWith(this,["AJAX",gettext("This response could not be submitted.")])})}).promise()},save:function(submission){var url=this.url("save_submission");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:JSON.stringify({submission:submission})}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This response could not be saved.")])})}).promise()},submitFeedbackOnAssessment:function(text,options){var url=this.url("submit_feedback");var payload=JSON.stringify({feedback_text:text,feedback_options:options});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This feedback could not be submitted.")])})}).promise()},peerAssess:function(optionsSelected,criterionFeedback,overallFeedback){var url=this.url("peer_assess");var payload=JSON.stringify({options_selected:optionsSelected,criterion_feedback:criterionFeedback,overall_feedback:overallFeedback});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})}).promise()},selfAssess:function(optionsSelected){var url=this.url("self_assess");var payload=JSON.stringify({options_selected:optionsSelected});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})})},trainingAssess:function(optionsSelected){var url=this.url("training_assess");var payload=JSON.stringify({options_selected:optionsSelected});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolveWith(this,[data.corrections])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})})},scheduleTraining:function(){var url=this.url("schedule_training");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:'""'}).done(function(data){if(data.success){defer.resolveWith(this,[data.msg])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This assessment could not be submitted.")])})})},loadXml:function(){var url=this.url("xml");return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:'""'}).done(function(data){if(data.success){defer.resolveWith(this,[data.xml])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This problem could not be loaded.")])})}).promise()},updateXml:function(xml){var url=this.url("update_xml");var payload=JSON.stringify({xml:xml});return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolve()}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("This problem could not be saved.")])})}).promise()},checkReleased:function(){var url=this.url("check_released");var payload='""';return $.Deferred(function(defer){$.ajax({type:"POST",url:url,data:payload}).done(function(data){if(data.success){defer.resolveWith(this,[data.is_released])}else{defer.rejectWith(this,[data.msg])}}).fail(function(data){defer.rejectWith(this,[gettext("The server could not be contacted.")])})}).promise()}};if(typeof OpenAssessment=="undefined"||!OpenAssessment){OpenAssessment={}}if(typeof window.gettext==="undefined"){window.gettext=function(text){return text}}OpenAssessment.StaffInfoView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView};OpenAssessment.StaffInfoView.prototype={load:function(){var view=this;if($("#openassessment__staff-info",view.element).length>0){this.server.render("staff_info").done(function(html){$("#openassessment__staff-info",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){view.baseView.showLoadError("staff_info")})}},loadStudentInfo:function(){var view=this;var sel=$("#openassessment__staff-info",this.element);var student_id=sel.find("#openassessment__student_id").val();this.server.studentInfo(student_id).done(function(html){$("#openassessment__student-info",view.element).replaceWith(html)}).fail(function(errMsg){view.showLoadError("student_info")})},installHandlers:function(){var sel=$("#openassessment__staff-info",this.element);var view=this;if(sel.length<=0){return}this.baseView.setUpCollapseExpand(sel,function(){});sel.find("#openassessment_student_info_form").submit(function(eventObject){eventObject.preventDefault();view.loadStudentInfo()});sel.find("#submit_student_id").click(function(eventObject){eventObject.preventDefault();view.loadStudentInfo()});sel.find("#schedule_training").click(function(eventObject){eventObject.preventDefault();view.scheduleTraining()});sel.find("#reschedule_unfinished_tasks").click(function(eventObject){eventObject.preventDefault();view.rescheduleUnfinishedTasks()})},scheduleTraining:function(){var view=this;this.server.scheduleTraining().done(function(msg){$("#schedule_training_message",this.element).text(msg)}).fail(function(errMsg){$("#schedule_training_message",this.element).text(errMsg)})},rescheduleUnfinishedTasks:function(){var view=this;this.server.rescheduleUnfinishedTasks().done(function(msg){$("#reschedule_unfinished_tasks_message",this.element).text(msg)}).fail(function(errMsg){$("#reschedule_unfinished_tasks_message",this.element).text(errMsg)})}};OpenAssessment.StudentTrainingView=function(element,server,baseView){this.element=element;this.server=server;this.baseView=baseView;this.rubric=null};OpenAssessment.StudentTrainingView.prototype={load:function(){var view=this;this.server.render("student_training").done(function(html){$("#openassessment__student-training",view.element).replaceWith(html);view.installHandlers()}).fail(function(errMsg){view.baseView.showLoadError("student-training")})},installHandlers:function(){var sel=$("#openassessment__student-training",this.element);var view=this;this.baseView.setUpCollapseExpand(sel);var rubricSelector=$("#student-training--001__assessment",this.element);if(rubricSelector.size()>0){var rubricElement=rubricSelector.get(0);this.rubric=new OpenAssessment.Rubric(rubricElement)}if(this.rubric!==null){this.rubric.canSubmitCallback($.proxy(this.assessButtonEnabled,this))}sel.find("#student-training--001__assessment__submit").click(function(eventObject){eventObject.preventDefault();view.assess()})},assess:function(){this.assessButtonEnabled(false);var options={};if(this.rubric!==null){options=this.rubric.optionsSelected()}var view=this;var baseView=this.baseView;this.server.trainingAssess(options).done(function(corrections){var incorrect=$("#openassessment__student-training--incorrect",this.element);var instructions=$("#openassessment__student-training--instructions",this.element);if(!view.rubric.showCorrections(corrections)){baseView.loadAssessmentModules();incorrect.addClass("is--hidden");instructions.removeClass("is--hidden")}else{instructions.addClass("is--hidden");incorrect.removeClass("is--hidden")}baseView.scrollToTop()}).fail(function(errMsg){baseView.toggleActionError("student-training",errMsg);view.assessButtonEnabled(true)})},assessButtonEnabled:function(isEnabled){var button=$("#student-training--001__assessment__submit",this.element);if(typeof isEnabled==="undefined"){return!button.hasClass("is--disabled")}else{button.toggleClass("is--disabled",!isEnabled)}}};
\ No newline at end of file
apps/openassessment/xblock/static/js/spec/oa_staff_info.js
......@@ -27,6 +27,15 @@ describe("OpenAssessment.StaffInfoView", function() {
}).promise();
};
this.rescheduleUnfinishedTasks = function() {
var server = this;
return $.Deferred(function(defer) {
defer.resolveWith(server, [server.data]);
}).promise();
};
this.data = {};
};
// Stub base view
......@@ -93,6 +102,7 @@ describe("OpenAssessment.StaffInfoView", function() {
expect(server.scheduleTraining).toHaveBeenCalled();
});
it("Loads staff info if the page contains a course staff section", function() {
// Load the fixture for the container page that DOES include a course staff section
loadFixtures('oa_base_course_staff.html');
......@@ -105,4 +115,45 @@ describe("OpenAssessment.StaffInfoView", function() {
loadFixtures('oa_base.html');
assertStaffInfoAjaxCall(false);
});
it("reschedules training of AI tasks", function() {
server.data = {
"success": true,
"workflow_uuid": "abc123",
"msg": "Great success."
};
var el = $("#openassessment-base").get(0);
var view = new OpenAssessment.StaffInfoView(el, server, baseView);
view.load();
spyOn(server, 'rescheduleUnfinishedTasks').andCallThrough();
// Test the Rescheduling
view.rescheduleUnfinishedTasks();
// Expect that the server was instructed to reschedule Unifinished Taks
expect(server.rescheduleUnfinishedTasks).toHaveBeenCalled();
});
it("reschedules training of AI tasks", function() {
server.data = {
"success": false,
"workflow_uuid": "abc123",
"errMsg": "Stupendous Failure."
};
var el = $("#openassessment-base").get(0);
var view = new OpenAssessment.StaffInfoView(el, server, baseView);
view.load();
spyOn(server, 'rescheduleUnfinishedTasks').andCallThrough();
// Test the Rescheduling
view.rescheduleUnfinishedTasks();
// Expect that the server was instructed to reschedule Unifinished Taks
expect(server.rescheduleUnfinishedTasks).toHaveBeenCalled();
});
});
apps/openassessment/xblock/static/js/src/oa_staff_info.js
......@@ -95,6 +95,14 @@ OpenAssessment.StaffInfoView.prototype = {
view.scheduleTraining();
}
);
// Install a click handler for rescheduling unfinished AI tasks for this problem
sel.find('#reschedule_unfinished_tasks').click(
function(eventObject) {
eventObject.preventDefault();
view.rescheduleUnfinishedTasks();
}
);
},
/**
......@@ -111,5 +119,22 @@ OpenAssessment.StaffInfoView.prototype = {
).fail(function(errMsg) {
$('#schedule_training_message', this.element).text(errMsg)
});
},
/**
Begins the process of rescheduling all unfinished grading tasks. This incdludes
checking if the classifiers have been created, and grading any unfinished
student submissions.
**/
rescheduleUnfinishedTasks: function() {
var view = this;
this.server.rescheduleUnfinishedTasks().done(
function(msg) {
$('#reschedule_unfinished_tasks_message', this.element).text(msg)
}
).fail(function(errMsg) {
$('#reschedule_unfinished_tasks_message', this.element).text(errMsg)
});
}
};
apps/openassessment/xblock/test/test_staff_info.py
......@@ -8,7 +8,7 @@ from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
from openassessment.assessment.api import ai as ai_api
from openassessment.workflow import api as workflow_api
from openassessment.assessment.errors.ai import AIError
from openassessment.assessment.errors.ai import AIError, AIGradingInternalError, AITrainingInternalError
from submissions import api as sub_api
from openassessment.xblock.test.base import scenario, XBlockHandlerTestCase
# Test dependency on Stub AI Algorithm configuration
......@@ -355,6 +355,51 @@ class TestCourseStaff(XBlockHandlerTestCase):
self.assertFalse(response['success'])
self.assertTrue('error' in response['msg'])
@scenario('data/example_based_assessment.xml', user_id='Bob')
def test_display_reschedule_unfinished_grading_tasks(self, xblock):
xblock.rubric_assessments.append(EXAMPLE_BASED_ASSESSMENT)
xblock.xmodule_runtime = self._create_mock_runtime(
xblock.scope_ids.usage_id, True, True, "Bob"
)
path, context = xblock.get_staff_path_and_context()
self.assertEquals('openassessmentblock/staff_debug/staff_debug.html', path)
self.assertTrue(context['display_reschedule_unfinished_tasks'])
@scenario('data/example_based_assessment.xml', user_id='Bob')
def test_reschedule_unfinished_grading_tasks_no_permissions(self, xblock):
xblock.xmodule_runtime = self._create_mock_runtime(
xblock.scope_ids.usage_id, True, False, "Bob"
)
response = self.request(xblock, 'reschedule_unfinished_tasks', json.dumps({}), response_format='json')
self.assertFalse(response['success'])
self.assertTrue('permission' in response['msg'])
@patch.object(ai_api, "reschedule_unfinished_tasks")
@scenario('data/example_based_assessment.xml', user_id='Bob')
def test_reschedule_unfinished_grading_tasks_success(self, xblock, mock_api):
mock_api.side_effect = Mock()
xblock.xmodule_runtime = self._create_mock_runtime(
xblock.scope_ids.usage_id, True, True, "Bob"
)
response = self.request(xblock, 'reschedule_unfinished_tasks', json.dumps({}), response_format='json')
self.assertTrue(response['success'])
self.assertTrue(u'All' in response['msg'])
mock_api.assert_called_with(
course_id=unicode(STUDENT_ITEM.get('course_id')), item_id=unicode(xblock.scope_ids.usage_id),
task_type=u"grade"
)
@patch.object(ai_api, "reschedule_unfinished_tasks")
@scenario('data/example_based_assessment.xml', user_id='Bob')
def test_reschedule_unfinished_grading_tasks_error(self, xblock, mock_api):
mock_api.side_effect = AIGradingInternalError("Oh Noooo!")
xblock.xmodule_runtime = self._create_mock_runtime(
xblock.scope_ids.usage_id, True, True, "Bob"
)
response = self.request(xblock, 'reschedule_unfinished_tasks', json.dumps({}), response_format='json')
self.assertFalse(response['success'])
self.assertTrue('error' in response['msg'])
@scenario('data/peer_only_scenario.xml', user_id='Bob')
def test_no_example_based_assessment(self, xblock):
xblock.xmodule_runtime = self._create_mock_runtime(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment