Define AI grading API.

apps/openassessment/assessment/api/ai-worker.py

+"""
+Public interface for AI training and grading, used by workers.
+"""
+
+
+def get_submission(grading_workflow_uuid):
+    """
+    Retrieve the submission associated with a particular grading workflow.
+
+    Args:
+        grading_workflow_uuid (str): The UUID of the grading workflow.
+
+    Returns:
+        submission (JSON-serializable): submission from the student.
+
+    Raises:
+        AIGradingRequestError
+        AIGradingInternalError
+
+    """
+    pass
+
+
+def get_classifier_set(grading_workflow_uuid):
+    """
+    Retrieve the classifier set associated with a particular grading workflow.
+
+    Args:
+        grading_workflow_uuid (str): The UUID of the grading workflow.
+
+    Returns:
+        dict: Maps criterion names to serialized classifiers.
+            (binary classifiers are base-64 encoded).
+
+    Raises:
+        AIGradingRequestError
+        AIGradingInternalError
+
+    """
+    pass
+
+
+
+def create_assessment(grading_workflow_uuid, assessment):
+    """
+    Create an AI assessment (complete the AI grading task).
+
+    Args:
+        grading_workflow_uuid (str): The UUID of the grading workflow.
+        assessment (dict): The serialized assessment.
+
+    Returns:
+        None
+
+    Raises:
+        AIGradingRequestError
+        AIGradingInternalError
+
+    """
+    pass
+
+
+def get_algorithm_id(training_workflow_uuid):
+    """
+    Retrieve the ID of the algorithm to use.
+
+    Args:
+        training_workflow_uuid (str): The UUID of the training workflow.
+
+    Returns:
+        unicode: The algorithm ID associated with the training task.
+
+    Raises:
+        AITrainingRequestError
+        AITrainingInternalError
+
+    """
+    pass
+
+
+def get_training_examples(training_workflow_uuid):
+    """
+    Retrieve the training examples associated with a training task.
+
+    Args:
+        training_workflow_uuid (str): The UUID of the training workflow.
+
+    Returns:
+        list of dict: Serialized training examples.
+
+    Raises:
+        AITrainingRequestError
+        AITrainingInternalError
+
+    """
+    pass
+
+
+def create_classifiers(training_workflow_uuid, classifier_set):
+    """
+    Upload trained classifiers and mark the workflow complete.
+
+    If grading tasks were submitted before any classifiers were trained,
+    this call will automatically reschedule those tasks.
+
+    Args:
+        training_workflow_uuid (str): The UUID of the training workflow.
+        classifier_set (dict): Mapping of criterion names to serialized classifiers.
+            (binary classifiers should be base-64 encoded).
+
+    Returns:
+        None
+
+    Raises:
+        AITrainingRequestError
+        AITrainingInternalError
+
+    """
+    pass

apps/openassessment/assessment/api/ai.py

+"""
+Public interface for AI training and grading, used by students/course authors.
+"""
+
+
+def submit(submission_uuid, rubric):
+    """
+    Submit a response for AI assessment.
+    This will:
+        (a) create a workflow (database record) to track the grading task
+        (b) if classifiers exist for the rubric, schedule an asynchronous grading task.
+
+    Args:
+        submission_uuid (str): The UUID of the submission to assess.
+        rubric (dict): Serialized rubric model.
+
+    Returns:
+        grading_workflow_uuid (str): The UUID of the grading workflow.
+            Usually the caller of `submit()` won't need this (since the workers
+            are parameterized by grading workflow UUID), but it's
+            useful for testing.
+
+    Raises:
+        AIGradingRequestError
+        AIGradingInternalError
+
+    """
+    pass
+
+
+def get_latest_assessment(submission_uuid):
+    """
+    Retrieve the latest AI assessment for a submission.
+
+    Args:
+        submission_uuid (str): The UUID of the submission being assessed.
+
+    Returns:
+        dict: The serialized assessment model
+
+    Raises:
+        AIGradingRequestError
+        AIGradingInternalError
+
+    """
+    pass
+
+
+def train_classifiers(rubric, examples, algorithm_id):
+    """
+    Schedule a task to train classifiers.
+    All training examples must match the rubric!
+
+    Args:
+        rubric (dict): The rubric used to assess the classifiers.
+        examples (list of dict): Serialized training examples.
+        algorithm_id (unicode): The ID of the algorithm used to train the classifiers.
+
+    Returns:
+        training_workflow_uuid (str): The UUID of the training workflow.
+            Usually the caller will not need this (since the workers
+            are parametrized by training workflow UUID), but it's
+            useful for testing.
+
+    Raises:
+        AITrainingRequestError
+        AITrainingInternalError
+
+    """
+    pass
+
+
+def reschedule_unfinished_tasks(course_id=None, item_id=None, task_type=None):
+    """
+    Check for unfinished tasks (both grading and training) and reschedule them.
+    Optionally restrict by course/item ID and task type.
+
+    Kwargs:
+        course_id (unicode): Restrict to unfinished tasks in a particular course.
+        item_id (unicode): Restrict to unfinished tasks for a particular item in a course.
+            NOTE: if you specify the item ID, you must also specify the course ID.
+        task_type (unicode): Either "grade" or "train".  Restrict to unfinished tasks of this type.
+
+    Raises:
+        AIRequestError
+        AIInternalError
+
+    """
+    pass

apps/openassessment/assessment/errors/ai.py

+"""
+Errors related to AI assessment.
+"""
+
+
+class AIError(Exception):
+    """
+    A general error occurred while using the AI assessment API.
+    """
+    pass
+
+
+class AITrainingRequestError(AIError):
+    """
+    There was a problem with the request sent to the AI assessment API.
+    """
+    pass
+
+
+class AITrainingInternalError(AIError):
+    """
+    An unexpected error occurred while using the AI assessment API.
+    """
+    pass
+
+
+class AIGradingRequestError(AIError):
+    """
+    There was a problem with the request sent to the AI assessment API.
+    """
+    pass
+
+
+class AIGradingInternalError(AIError):
+    """
+    An unexpected error occurred while using the AI assessment API.
+    """
+    pass

doc/source/architecture/ai_grading.rst

@@ -272,12 +272,15 @@ Data Model
 
     a. Rubric (Foreign Key)
     b. Created at (timestamp)
+    c. Training example set (many-to-many)
+    d. Hash of rubric and training examples (varchar): Useful for quickly finding existing classifiers (e.g. for Studio authors trying out the demo problem).
 
 5. **Classifier**
 
     a. ClassifierSet (Foreign Key)
     b. URL for trained classifier (varchar)
     c. Algorithm ID (varchar)
+    d. Criterion (Foreign Key)
 
 6. **Assessment** (same as current implementation)