Merge pull request #374 from edx/will/simulate-grading-error-cmd

Will/simulate grading error cmd

apps/openassessment/management/commands/simulate_ai_grading_error.py

+# -*- coding: utf-8 -*-
+"""
+Simulate failure of the worker AI grading tasks.
+When the workers fail to successfully complete AI grading,
+the AI grading workflow in the database will never be marked complete.
+
+To simulate the error condition, therefore, we create incomplete
+AI grading workflows without scheduling a grading task.
+
+To recover, a staff member can reschedule incomplete grading tasks.
+
+"""
+from django.core.management.base import BaseCommand, CommandError
+from submissions import api as sub_api
+from openassessment.assessment.models import AIGradingWorkflow, AIClassifierSet
+from openassessment.assessment.serializers import rubric_from_dict
+
+
+class Command(BaseCommand):
+    """
+    Create submissions and AI incomplete grading workflows.
+    """
+
+    help = (
+        u"Simulate failure of the worker AI grading tasks "
+        u"by creating incomplete AI grading workflows in the database."
+    )
+
+    args = '<COURSE_ID> <PROBLEM_ID> <NUM_SUBMISSIONS>'
+
+    RUBRIC_OPTIONS = [
+        {
+            "order_num": 0,
+            "name": u"poor",
+            "explanation": u"Poor job!",
+            "points": 0,
+        },
+        {
+            "order_num": 1,
+            "name": u"good",
+            "explanation": u"Good job!",
+            "points": 1,
+        }
+    ]
+
+    RUBRIC = {
+        'prompt': u"Test prompt",
+        'criteria': [
+            {
+                "order_num": 0,
+                "name": u"vocabulary",
+                "prompt": u"Vocabulary",
+                "options": RUBRIC_OPTIONS
+            },
+            {
+                "order_num": 1,
+                "name": u"grammar",
+                "prompt": u"Grammar",
+                "options": RUBRIC_OPTIONS
+            }
+        ]
+    }
+
+    # Since we're not actually running an AI scoring algorithm,
+    # we can use dummy data for the classifier, as long as it's
+    # JSON-serializable.
+    CLASSIFIERS = {
+        u'vocabulary': {},
+        u'grammar': {}
+    }
+
+    ALGORITHM_ID = u'fake'
+    STUDENT_ID = u'test_student'
+    ANSWER = {'answer': 'test answer'}
+
+    def handle(self, *args, **options):
+        """
+        Execute the command.
+
+        Args:
+            course_id (unicode): The ID of the course to create submissions/workflows in.
+            item_id (unicode): The ID of the problem in the course.
+            num_submissions (int): The number of submissions/workflows to create.
+
+        Raises:
+            CommandError
+
+        """
+        if len(args) < 3:
+            raise CommandError(u"Usage: simulate_ai_grading_error {}".format(self.args))
+
+        # Parse arguments
+        course_id = args[0].decode('utf-8')
+        item_id = args[1].decode('utf-8')
+        num_submissions = int(args[2])
+
+        # Create the rubric model
+        rubric = rubric_from_dict(self.RUBRIC)
+
+        # Create the classifier set
+        classifier_set = AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS, rubric, self.ALGORITHM_ID
+        )
+
+        # Create submissions and grading workflows
+        for num in range(num_submissions):
+            student_item = {
+                'course_id': course_id,
+                'item_id': item_id,
+                'item_type': 'openassessment',
+                'student_id': "{base}_{num}".format(base=self.STUDENT_ID, num=num)
+            }
+            submission = sub_api.create_submission(student_item, self.ANSWER)
+            workflow = AIGradingWorkflow.start_workflow(
+                submission['uuid'], self.RUBRIC, self.ALGORITHM_ID
+            )
+            workflow.classifier_set = classifier_set
+            workflow.save()
+            print u"{num}: Created incomplete grading workflow with UUID {uuid}".format(
+                num=num, uuid=workflow.uuid
+            )

apps/openassessment/management/commands/upload_oa_data.py

@@ -52,7 +52,7 @@ class Command(BaseCommand):
         Execute the command.
 
         Args:
-            course_id (unicode): The ID of the course to create submissions for.
+            course_id (unicode): The ID of the course to use.
             s3_bucket_name (unicode): The name of the S3 bucket to upload to.
 
         Raises:

apps/openassessment/management/tests/test_simulate_ai_grading_error.py

+# -*- coding: utf-8 -*-
+"""
+Tests for the simulate AI grading error management command.
+"""
+
+from openassessment.test_utils import CacheResetTest
+from openassessment.management.commands import simulate_ai_grading_error
+from openassessment.assessment.models import AIGradingWorkflow
+
+
+class SimulateAIGradingErrorTest(CacheResetTest):
+    """
+    Tests for the simulate AI grading error management command.
+    """
+
+    COURSE_ID = u"TɘꙅT ↄoUᴙꙅɘ"
+    ITEM_ID = u"𝖙𝖊𝖘𝖙 𝖎𝖙𝖊𝖒"
+    NUM_SUBMISSIONS = 20
+
+
+    def test_simulate_ai_grading_error(self):
+        # Run the command
+        cmd = simulate_ai_grading_error.Command()
+        cmd.handle(
+            self.COURSE_ID.encode('utf-8'),
+            self.ITEM_ID.encode('utf-8'),
+            self.NUM_SUBMISSIONS
+        )
+
+        # Check that the correct number of incomplete workflows
+        # were created.  These workflows should still have
+        # a classifier set, though, because otherwise they
+        # wouldn't have been scheduled for grading
+        # (that is, the submissions were made before classifier
+        # training completed).
+        num_errors = AIGradingWorkflow.objects.filter(
+            classifier_set__isnull=False,
+            completed_at__isnull=True
+        ).count()
+        self.assertEqual(self.NUM_SUBMISSIONS, num_errors)

apps/openassessment/management/tests/test_upload_oa_data.py

@@ -4,15 +4,15 @@ Tests for management command that uploads submission/assessment data.
 """
 from StringIO import StringIO
 import tarfile
-from django.test import TestCase
 import boto
 import moto
+from openassessment.test_utils import CacheResetTest
 from openassessment.management.commands import upload_oa_data
 from openassessment.workflow import api as workflow_api
 from submissions import api as sub_api
 
 
-class UploadDataTest(TestCase):
+class UploadDataTest(CacheResetTest):
     """
     Test the upload management command.  Archiving and upload are in-scope,
     but the contents of the generated CSV files are tested elsewhere.