Merge remote-tracking branch 'master' into authoring

Merged in AI Grading changes

Conflicts:
	openassessment/xblock/static/js/openassessment.min.js
	openassessment/xblock/static/js/src/oa_edit.js
	openassessment/xblock/xml.py

.gitignore

@@ -30,6 +30,7 @@ pip-log.txt
 .tox
 nosetests.xml
 htmlcov
+coverage.xml
 
 # Mr Developer
 .mr.developer.cfg
@@ -53,7 +54,11 @@ coverage
 
 # tim-specific
 ora2db
+storage/*
 openassessment/xblock/static/js/fixtures/*.html
 
 # logging
-logs/*.log*
+logs/*/*.log*
+
+# Vagrant
+.vagrant

.travis.yml

 language: python
 python:
   - "2.7"
-before_script:
-  - export DISPLAY=:99.0
-  - sh -e /etc/init.d/xvfb start
+before_install:
+  - sudo apt-get update
+  - sudo xargs -a apt-packages.txt apt-get install --fix-missing
+install:
   - "pip install coveralls"
+  - "./scripts/install-wheels.sh"
+  - travis_retry ./scripts/download-nltk-data.sh
+before_script:
+  - "export DISPLAY=:99.0"
+  - "sh -e /etc/init.d/xvfb start"
 script:
   - "./scripts/test.sh"
   - "./scripts/i18n.sh"
-  - "python setup.py install"
 after_success:
   coveralls

.tx/config

+[main]
+host = https://www.transifex.com
+
+[edx-platform.openassessment]
+file_filter = conf/locale/<lang>/LC_MESSAGES/django.po
+source_file = conf/locale/en/LC_MESSAGES/django.po
+source_lang = en
+type = PO
+
+[edx-platform.openassessment-js]
+file_filter = conf/locale/<lang>/LC_MESSAGES/djangojs.po
+source_file = conf/locale/en/LC_MESSAGES/djangojs.po
+source_lang = en
+type = PO

README.rst

@@ -71,11 +71,11 @@ To run the Python and Javascript unit test suites:
 
     ./scripts/test.sh
 
-To limit Python tests to a particular Django app:
+To limit Python tests to a particular module:
 
 .. code:: bash
 
-    ./scripts/test-python.sh openassessment.xblock
+    ./scripts/test-python.sh openassessment/xblock/test/test_openassessment.py
 
 To run just the JavaScript tests:
 
@@ -103,7 +103,7 @@ Check for quality violations:
 
 .. code:: bash
 
-    pylint apps
+    pylint openassessment
 
 Disable quality violations on a line or file:
 
@@ -112,6 +112,48 @@ Disable quality violations on a line or file:
     # pylint: disable=W0123,E4567
 
 
+Vagrant
+=======
+
+This repository includes a Vagrant configuration file, which is useful for testing
+ORA2 in an environment that is closer to production:
+
+* Uses `gunicorn <http://gunicorn.org/>`_ to serve the workbench application.
+  Unlike Django ``runserver``, gunicorn will process requests in parallel.
+
+* Uses `mysql <http://www.mysql.com/>`_ as the database, which (unlike
+  `sqlite <http://www.sqlite.org/>`_) allows for simultaneous writes.
+
+* Serves static files using `nginx <http://wiki.nginx.org/Main>`_ instead
+  of Django `staticfiles <https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/>`_.
+
+* Runs multiple `celery workers <http://celery.readthedocs.org/en/latest/>`_.
+
+* Uses `memcached <http://memcached.org/>`_.
+
+* Installs `EASE <https://github.com/edx/ease>`_ for AI grading, including
+  its many requirements.
+
+To use the Vagrant VM:
+
+1) `Install Vagrant <https://docs.vagrantup.com/v2/installation/>`_.
+2) ``vagrant up`` to start and provision the Vagrant VM.
+3) Visit `http://192.168.44.10 <http://192.168.44.10>`_
+4) You should see the workbench index page load.
+
+After making a change to the code in the ``edx-ora2`` directory,
+you must restart the services on the Vagrant VM:
+
+1) ``vagrant ssh`` to ssh into the Vagrant VM.
+2) ``./update.sh`` to restart the services, run database migrations, and collect static assets.
+3) Visit `http://192.168.44.10 <http://192.168.44.10>`_
+
+By default, the Vagrant VM also includes a monitoring tool for Celery tasks called `Flower <https://github.com/mher/flower>`_.
+To use the tool, visit: `http://192.168.44.10:5555 <http://192.168.44.10:5555>`_
+
+The log files from the Vagrant VM are located in ``edx-ora2/logs/vagrant``, which is shared with the host machine.
+
+
 i18n
 ====
 

Vagrantfile

+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+$script = <<END
+set -e
+
+echo "Updating apt packages..."
+apt-get update -y
+
+echo "Installing basic system requirements..."
+apt-get install -y curl git vim libxml2-dev libxslt1-dev memcached nginx
+
+echo "Installing mysql server..."
+DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server-5.5
+echo "CREATE DATABASE IF NOT EXISTS workbench DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" | mysql -u root
+
+echo "Installing Python system requirements..."
+apt-get install -y python2.7 python2.7-dev python-pip python-software-properties python-mysqldb libmysqlclient-dev
+pip install virtualenv
+
+echo "Installing FireFox and xvfb (for JavaScript tests)..."
+add-apt-repository "ppa:ubuntu-mozilla-security/ppa"
+apt-get install -y firefox dbus-x11 xvfb
+
+cat /home/vagrant/edx-ora2/vagrant/xvfb.conf > /etc/init/xvfb.conf
+start xvfb || true
+
+echo "Installing RabbitMQ..."
+add-apt-repository "deb http://www.rabbitmq.com/debian/ testing main"
+cd /tmp && wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc && apt-key add rabbitmq-signing-key-public.asc
+apt-get update -y
+apt-get install -y rabbitmq-server
+
+echo "Installing NodeJS..."
+add-apt-repository ppa:chris-lea/node.js
+apt-get update -y
+apt-get install -y nodejs
+
+# Stop all Python upstart jobs
+sudo stop workbench || true
+sudo stop celery || true
+sudo stop flower || true
+
+su vagrant <<EOF
+    set -e
+
+    echo "Creating a virtualenv..."
+    mkdir -p /home/vagrant/.virtualenvs
+    virtualenv /home/vagrant/.virtualenvs/edx-ora2
+    source /home/vagrant/.virtualenvs/edx-ora2/bin/activate
+
+    echo "Configuring login script..."
+    cat /home/vagrant/edx-ora2/vagrant/bash_profile > /home/vagrant/.bash_profile
+
+    echo "Installing EASE..."
+    if [ ! -d /home/vagrant/ease ]; then
+        git clone https://github.com/edx/ease.git /home/vagrant/ease
+    fi
+    cat /home/vagrant/ease/apt-packages.txt | xargs sudo apt-get -y install
+    cd /home/vagrant/ease && pip install -r pre-requirements.txt
+    cd /home/vagrant/ease && python setup.py install
+
+    echo "Downloading NLTK corpus..."
+    cd /home/vagrant/ease && ./download-nltk-corpus.sh
+
+    echo "Installing gunicorn..."
+    pip install gunicorn
+
+    echo "Instally Python MySQL library..."
+    pip install MySQL-python
+
+    echo "Installing celery flower..."
+    pip install flower
+
+    echo "Install edx-ora2..."
+    cd /home/vagrant/edx-ora2 && ./scripts/install.sh
+
+    echo "Update the database..."
+    cd /home/vagrant/edx-ora2 && python manage.py syncdb --migrate --noinput --settings settings.vagrant
+
+    echo "Collect static assets..."
+    mkdir -p /home/vagrant/static
+    cd /home/vagrant/edx-ora2 && python manage.py collectstatic --noinput --settings settings.vagrant
+
+    echo "Creating the update script..."
+    cp /home/vagrant/edx-ora2/vagrant/update.sh /home/vagrant/update.sh
+
+EOF
+
+echo "Creating upstart script for workbench..."
+cat /home/vagrant/edx-ora2/vagrant/workbench_upstart.conf > /etc/init/workbench.conf
+start workbench || true
+
+echo "Create upstart script for Celery workers..."
+cat /home/vagrant/edx-ora2/vagrant/celery_upstart.conf > /etc/init/celery.conf
+start celery || true
+
+echo "Create upstart script for Celery flower..."
+cat /home/vagrant/edx-ora2/vagrant/flower_upstart.conf > /etc/init/flower.conf
+start flower || true
+
+echo "Configure nginx"
+cat /home/vagrant/edx-ora2/vagrant/nginx.conf > /etc/nginx/sites-enabled/workbench.conf
+
+echo "Restart nginx"
+sudo service nginx stop || true
+sudo service nginx start || true
+
+END
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  config.vm.box = "precise64"
+  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+
+  config.vm.network "private_network", ip: "192.168.44.10"
+  config.vm.synced_folder ".", "/home/vagrant/edx-ora2"
+
+
+  config.vm.provider :virtualbox do |vb|
+    # Increase memory and CPU
+    vb.customize ["modifyvm", :id, "--memory", "2048"]
+    vb.customize ["modifyvm", :id, "--cpus", "2"]
+
+    # Allow DNS to work for Ubuntu 12.10 host
+    # http://askubuntu.com/questions/238040/how-do-i-fix-name-service-for-vagrant-client
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+  end
+
+  config.vm.provision "shell", inline: $script
+end

apt-packages.txt

+aspell
+gcc
+g++
+gfortran
+libblas3gf
+libblas-dev
+liblapack3gf
+liblapack-dev
+libatlas-base-dev
+libxml2-dev
+libxslt1-dev

docs/en_us/course_authors/source/PeerAssessment.rst

@@ -8,8 +8,6 @@ Open Response Assessments
 Introduction to Open Response Assessments
 *****************************************
 
-.. note:: **The open response assessments feature is in limited release.** If you're at an edX consortium university and you plan to include open response assessments in a MOOC, you'll work with your edX project manager (PM) to enable the open response assessments feature and create assignments.
-
 Open response assessments allow instructors to assign questions that may not have definite answers. Students submit a response to the question, and then that student and the student's peers compare the response to a rubric that you create.
 
 Open response assessments include peer assessments and self assessments. In peer assessments, students compare their peers' responses to a rubric that you create. In self assessments, students compare their own responses to the rubric.
@@ -252,9 +250,7 @@ Note, again, that final scores are calculated by criteria, not by individual ass
 Create a Peer Assessment Problem
 ********************************
 
-.. warning:: Peer assessments are in limited release and are only available in a few courses. To enable the peer assessment feature in your course, contact your edX program manager. After the feature has been enabled, you can create peer assessments by following the steps below.
-
-To create a peer assessment problem, you'll edit the XML code in a Problem component, similar to creating other problems. The following image shows what a peer assessment component looks like when you edit it in Studio, as well as the way that students see that peer assessment in the courseware.
+To create a peer assessment problem, you'll edit XML code in a Problem component, similar to the way you create other problems. The following image shows what a peer assessment component looks like when you edit it in Studio, as well as the way that students see that peer assessment in the courseware.
 
 .. image:: /Images/PA_XML_LMS_All.png
    :alt: Image of a peer assessment in Studio and LMS views
@@ -264,13 +260,16 @@ Creating a peer assessment is a multi-step process:
 
 * :ref:`PA Create Component`
 * :ref:`PA Specify Name and Assessment Types`
+* :ref:`PA Add a Student Training Assessment`
 * :ref:`PA Add Due Dates`
 * :ref:`PA Add Question`
 * :ref:`PA Add Rubric`
+* :ref:`PA Provide Comment Options`
 * :ref:`PA Test Problem`
 
 Each of these steps is covered in detail below.
 
+
 .. _PA Create Component:
 
 ============================
@@ -278,7 +277,7 @@ Step 1. Create the Component
 ============================
 
 #. In Studio, open the unit where you want to create the assessment.
-#. Under **Add New Component**, click **Advanced**, and then click **Peer Assessment**.
+#. Under **Add New Component**, click **Problem**, click the **Advanced** tab, and then click **Peer Assessment**.
 #. In the Problem component that appears, click **Edit**.
 
 When the component editor opens, you can see sample code that includes the following. You'll replace this sample content with the content for your problem:
@@ -395,7 +394,7 @@ For more information, see :ref:`PA Student Training Assessments`.
 .. _PA Add Due Dates:
 
 ==========================================
-Step 3. Add Start and Due Dates (optional)
+Step 4. Add Start and Due Dates (optional)
 ==========================================
 
 Setting start and due dates is optional. If you don't specify dates, the deadline for all student responses and assessments is the due date that you set for the subsection that contains the peer assessment. If you do specify dates, those dates take precedence over the subsection due date.
@@ -444,7 +443,7 @@ In this example, the problem is set at the subsection level to open on February
 .. _PA Add Question:
 
 ============================
-Step 4. Add the Question
+Step 5. Add the Question
 ============================
 
 The following image shows a question in the component editor along with the way the question appears to students.
@@ -488,7 +487,7 @@ To remove the prompt from the Peer Assessment component, open the component edit
 .. _PA Add Rubric:
 
 ============================
-Step 5. Add the Rubric
+Step 6. Add the Rubric
 ============================
 
 To add the rubric, you'll create your criteria and options in XML. The following image shows a highlighted criterion and its options in the component editor, followed by the way the criterion and options appear to students.
@@ -561,7 +560,7 @@ You can use the following code as a template:
 .. _PA Provide Comment Options:
 
 =============================================
-Step 6 (optional). Provide Comment Options
+Step 7. Provide Comment Options (optional)
 =============================================
 
 After students fill out the rubric, they can provide additional comments for the responses they've assessed. By default, students see a field for comments below the rubric.
@@ -639,10 +638,8 @@ If you want to provide comment fields below additional criteria, add the ``feedb
 .. _PA Test Problem:
 
 ============================
-Step 7. Test the Problem
+Step 8. Test the Problem
 ============================
 
 To test your assignment, set up the assignment in a test course, and ask a group of beta users to submit responses and grade each other. The beta testers can then let you know if they found the question and the rubric easy to understand or if they found any problems with the assignment.
 
-.. _PA Access Status:
-

docs/en_us/developers/requirements.txt

 # Packages we need in order to build the docs, separated out so that rtfd.org
 # can install them.
-
+path.py
 sphinx
 sphinxcontrib-napoleon
 markupsafe
+
+# need to install requirements for application, in order to import Python code
+-r ../../../requirements/base.txt

docs/en_us/developers/source/api.rst

@@ -27,6 +27,12 @@ Self Assessment
 .. automodule:: openassessment.assessment.api.self
    :members:
 
+Example-Based Assessment (AI)
+*****************************
+
+.. automodule:: openassessment.assessment.api.ai
+   :members:
+
 Student Training
 ****************
 

docs/en_us/developers/source/architecture/ai_grading.rst

@@ -240,28 +240,23 @@ Data Model
 1. **GradingWorkflow**
 
     a. Submission UUID (varchar)
-    b. Rubric UUID (varchar)
-    c. ClassifierSet (Foreign Key, Nullable)
-    d. Assessment (Foreign Key, Nullable)
-    e. Scheduled at (timestamp): The time the task was placed on the queue.
-    f. Started at (timestamp): The time the task was picked up by the worker.
+    b. ClassifierSet (Foreign Key, Nullable)
+    c. Assessment (Foreign Key, Nullable)
+    d. Rubric (Foreign Key): Used to search for classifier sets if none are available when the workflow is started.
+    e. Algorithm ID (varchar): Used to search for classifier sets if none are available when the workflow is started.
+    f. Scheduled at (timestamp): The time the task was placed on the queue.
     g. Completed at (timestamp): The time the task was completed.  If set, the task is considered complete.
-    h. Course ID (varchar): The ID of the course associated with the submission.  Useful for rescheduling
-       failed grading tasks in a particular course.
-    i. Item ID (varchar): The ID of the item (problem) associated with the submission.  Useful for rescheduling
-       failed grading tasks in a particular item in a course.
-    j. Worker version (varchar): Identifier for the code running on the worker when the task was started.  Useful for error tracking.
+    h. Course ID (varchar): The ID of the course associated with the submission.  Useful for rescheduling failed grading tasks in a particular course.
+    i. Item ID (varchar): The ID of the item (problem) associated with the submission.  Useful for rescheduling failed grading tasks in a particular item in a course.
+
 
 2. **TrainingWorkflow**
 
     a. Algorithm ID (varchar)
-    b. Rubric UUID (varchar)
-    c. Many-to-many relation with **TrainingExample**.  We can re-use examples for multiple workflows.
-    d. ClassifierSet (Foreign Key)
-    e. Scheduled at (timestamp): The time the task was placed on the queue.
-    f. Started at (timestamp): The time the task was picked up by the worker.
-    g. Completed at (timestamp): The time the task was completed.  If set, the task is considered complete.
-    h. Worker version (varchar): Identifier for the code running on the worker when the task was started.  Useful for error tracking.
+    b. Many-to-many relation with **TrainingExample**.  We can re-use examples for multiple workflows.
+    c. ClassifierSet (Foreign Key)
+    d. Scheduled at (timestamp): The time the task was placed on the queue.
+    e. Completed at (timestamp): The time the task was completed.  If set, the task is considered complete.
 
 3. **TrainingExample**
 
@@ -272,12 +267,13 @@ Data Model
 
     a. Rubric (Foreign Key)
     b. Created at (timestamp)
+    c. Algorithm ID (varchar)
 
 5. **Classifier**
 
     a. ClassifierSet (Foreign Key)
     b. URL for trained classifier (varchar)
-    c. Algorithm ID (varchar)
+    c. Criterion (Foreign Key)
 
 6. **Assessment** (same as current implementation)
 

docs/en_us/developers/source/architecture/workflow.rst

@@ -49,6 +49,8 @@ Isolation of Assessment types
       a non `None` value has been returned by this function for a given
       `submission_uuid`, repeated calls to this function should return the same
       thing.
+    `on_start(submission_uuid)`
+      Notification to the API that the student has started the assessment step.
 
    In the long run, it could be that `OpenAssessmentBlock` becomes a wrapper
    that talks to child XBlocks via this kind of API, and that each child would
@@ -96,3 +98,15 @@ Handling Problem Definition Change
    2. If the sequence of steps changes, we look at the new steps and advance to
       the first step that the user has not completed (`is_submitter_done()`
       returns `False`).
+
+Django settings
+   Assessments in the workflow are configurable using Django settings.
+   This encapsulates the workflow API from the assessment modules.
+
+   The two settings are:
+
+   * `ORA2_ASSESSMENTS`: a `dict` mapping assessment names to the Python module path
+     of the corresponding assessment API.
+   * `ORA2_ASSESSMENT_SCORE_PRIORITY`: a `list` of assessment names that determine
+     which assessment type is used to generate a student's score.
+

docs/en_us/developers/source/conf.py

@@ -12,6 +12,7 @@
 # serve to show the default.
 
 import sys, os
+from path import path
 
 #import sphinx_rtd_theme
 
@@ -22,12 +23,27 @@ import sys, os
 
 # -- General configuration -----------------------------------------------------
 
+
+# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+if not on_rtd: # only import and set the theme if we're building docs locally
+  import sphinx_rtd_theme
+  html_theme = 'sphinx_rtd_theme'
+  html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
-root = os.path.abspath('../../../..')
-sys.path.append(root)
+root = path('../../../..').abspath()
+sys.path.insert(0, root)
+sys.path.append(root / "openassessment/assessment")
+sys.path.append(root / "openassessment/management")
+sys.path.append(root / "openassessment/templates")
+sys.path.append(root / "openassessment/workflow")
+sys.path.append(root / "openassessment/xblock")
+sys.path.append(root / "openassessment/templates")
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.base")
 
+
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon', 'sphinx.ext.coverage', 'sphinx.ext.ifconfig', 'sphinx.ext.todo']
@@ -46,14 +62,6 @@ master_doc = 'index'
 
 
 
-# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd: # only import and set the theme if we're building docs locally
-  import sphinx_rtd_theme
-  html_theme = 'sphinx_rtd_theme'
-  html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
 # General information about the project.
 project = u'edx-ora2'
 copyright = u'2014, edX.org'

docs/shared/requirements.txt

-# Read the docs requirements file
-# ----------------------------------------
-# Installing modules with C dependencies on RTD can be tricky, and pip doesn't
-# have an 'ignore-errors' option, so all requirements fail. Here we keep the
-# maximal list of modules that still works.
-#
-beautifulsoup4==4.1.3
-beautifulsoup==3.2.1
-boto==2.6.0
-celery==3.0.19
-distribute>=0.6.28, <0.7
-django-celery==3.0.17
-django-countries==1.5
-django-filter==0.6.0
-django-followit==0.0.3
-django-keyedcache==1.4-6
-django-kombu==0.9.4
-django-mako==0.1.5pre
-django-masquerade==0.1.6
-django-mptt==0.5.5
-django-openid-auth==0.4
-django-robots==0.9.1
-django-sekizai==0.6.1
-django-ses==0.4.1
-django-storages==1.1.5
-django-threaded-multihost==1.4-1
-django-method-override==0.1.0
-djangorestframework==2.3.5
-django==1.4.5
-feedparser==5.1.3
-fs==0.4.0
-GitPython==0.3.2.RC1
-glob2==0.3
-lxml==3.0.1
-mako==0.7.3
-Markdown==2.2.1
-mock==1.0.1
-networkx==1.7
-nltk==2.0.4
-oauthlib==0.5.1
-paramiko==1.9.0
-path.py==3.0.1
-Pillow==1.7.8
-pip>=1.3
-polib==1.0.3
-pycrypto>=2.6
-pygments==1.5
-pymongo==2.4.1
-python-memcached==1.48
-python-openid==2.2.5
-pytz==2012h
-PyYAML==3.10
-requests==1.2.3
-Shapely==1.2.16
-sorl-thumbnail==11.12
-South==0.7.6
-sympy==0.7.1
-xmltodict==0.4.1
-
-# Metrics gathering and monitoring
-dogapi==1.2.1
-dogstatsd-python==0.2.1
-newrelic==1.13.1.31
-
-# Used for Internationalization and localization
-Babel==1.3
-transifex-client==0.9.1
-
-
--e common/lib/calc
--e common/lib/capa
--e common/lib/chem
--e common/lib/sandbox-packages
--e common/lib/symmath
--e common/lib/xmodule
--e .
-
--e git+https://github.com/edx/XBlock.git@b697bebd45deebd0f868613fab6722a0460ca0c1#egg=XBlock

openassessment/assessment/admin.py

@@ -5,7 +5,8 @@ from django.core.urlresolvers import reverse
 from django.utils import html
 
 from openassessment.assessment.models import (
-    Assessment, AssessmentFeedback, PeerWorkflow, PeerWorkflowItem, Rubric
+    Assessment, AssessmentFeedback, PeerWorkflow, PeerWorkflowItem, Rubric,
+    AIGradingWorkflow, AITrainingWorkflow, AIClassifierSet, AIClassifier
 )
 from openassessment.assessment.serializers import RubricSerializer
 
@@ -17,7 +18,8 @@ class RubricAdmin(admin.ModelAdmin):
     list_display_links = ('id', 'content_hash')
     search_fields = ('id', 'content_hash')
     readonly_fields = (
-        'id', 'content_hash', 'points_possible', 'criteria_summary', 'data'
+        'id', 'content_hash', 'structure_hash',
+        'points_possible', 'criteria_summary', 'data'
     )
 
     def criteria_summary(self, rubric_obj):
@@ -119,7 +121,32 @@ class AssessmentFeedbackAdmin(admin.ModelAdmin):
     assessments_by.allow_tags = True
 
 
+class AIGradingWorkflowAdmin(admin.ModelAdmin):
+    list_display = ('uuid', 'submission_uuid')
+    search_fields = ('uuid', 'submission_uuid', 'student_id', 'item_id', 'course_id')
+    readonly_fields = ('uuid', 'submission_uuid', 'student_id', 'item_id', 'course_id')
+
+
+class AITrainingWorkflowAdmin(admin.ModelAdmin):
+    list_display = ('uuid',)
+    search_fields = ('uuid', 'course_id', 'item_id',)
+    readonly_fields = ('uuid', 'course_id', 'item_id',)
+
+
+class AIClassifierInline(admin.TabularInline):
+    model = AIClassifier
+
+
+class AIClassifierSetAdmin(admin.ModelAdmin):
+    list_display = ('id',)
+    search_fields = ('id',)
+    inlines = [AIClassifierInline]
+
+
 admin.site.register(Rubric, RubricAdmin)
 admin.site.register(PeerWorkflow, PeerWorkflowAdmin)
 admin.site.register(Assessment, AssessmentAdmin)
 admin.site.register(AssessmentFeedback, AssessmentFeedbackAdmin)
+admin.site.register(AIGradingWorkflow, AIGradingWorkflowAdmin)
+admin.site.register(AITrainingWorkflow, AITrainingWorkflowAdmin)
+admin.site.register(AIClassifierSet, AIClassifierSetAdmin)

openassessment/assessment/api/peer.py

@@ -28,6 +28,25 @@ PEER_TYPE = "PE"
 
 
 def submitter_is_finished(submission_uuid, requirements):
+    """
+    Check whether the submitter has made the required number of assessments.
+
+    If the requirements dict is None (because we're being updated
+    asynchronously or when the workflow is first created),
+    then automatically return False.
+
+    Args:
+        submission_uuid (str): The UUID of the submission being tracked.
+        requirements (dict): Dictionary with the key "must_grade" indicating
+            the required number of submissions the student must grade.
+
+    Returns:
+        bool
+
+    """
+    if requirements is None:
+        return False
+
     try:
         workflow = PeerWorkflow.objects.get(submission_uuid=submission_uuid)
         if workflow.completed_at is not None:
@@ -41,18 +60,88 @@ def submitter_is_finished(submission_uuid, requirements):
         return False
 
 
+def assessment_is_finished(submission_uuid, requirements):
+    """
+    Check whether the submitter has received enough assessments
+    to get a score.
+
+    If the requirements dict is None (because we're being updated
+    asynchronously or when the workflow is first created),
+    then automatically return False.
+
+    Args:
+        submission_uuid (str): The UUID of the submission being tracked.
+        requirements (dict): Dictionary with the key "must_be_graded_by"
+            indicating the required number of assessments the student
+            must receive to get a score.
+
+    Returns:
+
+        bool
+    """
+    if requirements is None:
+        return False
+    return bool(get_score(submission_uuid, requirements))
+
+
+def on_start(submission_uuid):
+    """Create a new peer workflow for a student item and submission.
+
+    Creates a unique peer workflow for a student item, associated with a
+    submission.
+
+    Args:
+        submission_uuid (str): The submission associated with this workflow.
+
+    Returns:
+        None
+
+    Raises:
+        SubmissionError: There was an error retrieving the submission.
+        PeerAssessmentInternalError: Raised when there is an internal error
+            creating the Workflow.
+
+    """
+    try:
+        submission = sub_api.get_submission_and_student(submission_uuid)
+        workflow, __ = PeerWorkflow.objects.get_or_create(
+            student_id=submission['student_item']['student_id'],
+            course_id=submission['student_item']['course_id'],
+            item_id=submission['student_item']['item_id'],
+            submission_uuid=submission_uuid
+        )
+        workflow.save()
+    except IntegrityError:
+        # If we get an integrity error, it means someone else has already
+        # created a workflow for this submission, so we don't need to do anything.
+        pass
+    except DatabaseError:
+        error_message = (
+            u"An internal error occurred while creating a new peer "
+            u"workflow for submission {}"
+            .format(submission_uuid)
+        )
+        logger.exception(error_message)
+        raise PeerAssessmentInternalError(error_message)
+
+
 def get_score(submission_uuid, requirements):
     """
     Retrieve a score for a submission if requirements have been satisfied.
 
     Args:
         submission_uuid (str): The UUID of the submission.
-        requirements (dict): Description of requirements for receiving a score,
-            specific to the particular kind of submission (e.g. self or peer).
+        requirements (dict): Dictionary with the key "must_be_graded_by"
+            indicating the required number of assessments the student
+            must receive to get a score.
 
     Returns:
         dict with keys "points_earned" and "points_possible".
+
     """
+    if requirements is None:
+        return None
+
     # User hasn't completed their own submission yet
     if not submitter_is_finished(submission_uuid, requirements):
         return None
@@ -93,10 +182,6 @@ def get_score(submission_uuid, requirements):
     }
 
 
-def assessment_is_finished(submission_uuid, requirements):
-    return bool(get_score(submission_uuid, requirements))
-
-
 def create_assessment(
         scorer_submission_uuid,
         scorer_id,

openassessment/assessment/api/self.py

@@ -24,6 +24,72 @@ SELF_TYPE = "SE"
 logger = logging.getLogger("openassessment.assessment.api.self")
 
 
+def submitter_is_finished(submission_uuid, requirements):
+    """
+    Check whether a self-assessment has been completed for a submission.
+
+    Args:
+        submission_uuid (str): The unique identifier of the submission.
+        requirements (dict): Any attributes of the assessment module required
+            to determine if this assessment is complete. There are currently
+            no requirements for a self-assessment.
+    Returns:
+        True if the submitter has assessed their answer
+    Examples:
+        >>> submitter_is_finished('222bdf3d-a88e-11e3-859e-040ccee02800', {})
+        True
+    """
+    return Assessment.objects.filter(
+        score_type=SELF_TYPE, submission_uuid=submission_uuid
+    ).exists()
+
+
+def assessment_is_finished(submission_uuid, requirements):
+    """
+    Check whether a self-assessment has been completed. For self-assessment,
+    this function is synonymous with submitter_is_finished.
+
+    Args:
+        submission_uuid (str): The unique identifier of the submission.
+        requirements (dict): Any attributes of the assessment module required
+            to determine if this assessment is complete. There are currently
+            no requirements for a self-assessment.
+    Returns:
+        True if the assessment is complete.
+    Examples:
+        >>> assessment_is_finished('222bdf3d-a88e-11e3-859e-040ccee02800', {})
+        True
+    """
+    return submitter_is_finished(submission_uuid, requirements)
+
+
+def get_score(submission_uuid, requirements):
+    """
+    Get the score for this particular assessment.
+
+    Args:
+        submission_uuid (str): The unique identifier for the submission
+        requirements (dict): Not used.
+    Returns:
+        A dict of points earned and points possible for the given submission.
+        Returns None if no score can be determined yet.
+    Examples:
+        >>> get_score('222bdf3d-a88e-11e3-859e-040ccee02800', {})
+        {
+            'points_earned': 5,
+            'points_possible': 10
+        }
+    """
+    assessment = get_assessment(submission_uuid)
+    if not assessment:
+        return None
+
+    return {
+        "points_earned": assessment["points_earned"],
+        "points_possible": assessment["points_possible"]
+    }
+
+
 def create_assessment(submission_uuid, user_id, options_selected, rubric_dict, scored_at=None):
     """
     Create a self-assessment for a submission.
@@ -155,73 +221,6 @@ def get_assessment(submission_uuid):
     return serialized_assessment
 
 
-def submitter_is_finished(submission_uuid, requirements):
-    """
-    Check whether a self-assessment has been completed for a submission.
-
-    Args:
-        submission_uuid (str): The unique identifier of the submission.
-        requirements (dict): Any attributes of the assessment module required
-            to determine if this assessment is complete. There are currently
-            no requirements for a self-assessment.
-    Returns:
-        True if the submitter has assessed their answer
-    Examples:
-        >>> submitter_is_finished('222bdf3d-a88e-11e3-859e-040ccee02800', {})
-        True
-    """
-    return Assessment.objects.filter(
-        score_type=SELF_TYPE, submission_uuid=submission_uuid
-    ).exists()
-
-
-def assessment_is_finished(submission_uuid, requirements):
-    """
-    Check whether a self-assessment has been completed. For self-assessment,
-    this function is synonymous with submitter_is_finished.
-
-    Args:
-        submission_uuid (str): The unique identifier of the submission.
-        requirements (dict): Any attributes of the assessment module required
-            to determine if this assessment is complete. There are currently
-            no requirements for a self-assessment.
-    Returns:
-        True if the assessment is complete.
-    Examples:
-        >>> assessment_is_finished('222bdf3d-a88e-11e3-859e-040ccee02800', {})
-        True
-    """
-    return submitter_is_finished(submission_uuid, requirements)
-
-
-def get_score(submission_uuid, requirements):
-    """
-    Get the score for this particular assessment.
-
-    Args:
-        submission_uuid (str): The unique identifier for the submission
-        requirements (dict): Any attributes of the assessment module required
-            to determine if this assessment is complete. There are currently
-            no requirements for a self-assessment.
-    Returns:
-        A dict of points earned and points possible for the given submission.
-        Returns None if no score can be determined yet.
-    Examples:
-        >>> get_score('222bdf3d-a88e-11e3-859e-040ccee02800', {})
-        {
-            'points_earned': 5,
-            'points_possible': 10
-        }
-    """
-    assessment = get_assessment(submission_uuid)
-    if not assessment:
-        return None
-
-    return {
-        "points_earned": assessment["points_earned"],
-        "points_possible": assessment["points_possible"]
-    }
-
 
 def get_assessment_scores_by_criteria(submission_uuid):
     """Get the median score for each rubric criterion

openassessment/assessment/api/student_training.py

@@ -41,6 +41,9 @@ def submitter_is_finished(submission_uuid, requirements):   # pylint:disable=W06
         StudentTrainingRequestError
 
     """
+    if requirements is None:
+        return False
+
     try:
         num_required = int(requirements['num_required'])
     except KeyError:
@@ -56,19 +59,33 @@ def submitter_is_finished(submission_uuid, requirements):   # pylint:disable=W06
         return workflow.num_completed >= num_required
 
 
-def assessment_is_finished(submission_uuid, requirements):  # pylint:disable=W0613
+def on_start(submission_uuid):
     """
-    Since the student is not being assessed by others,
-    this always returns true.
-    """
-    return True
+    Creates a new student training workflow.
 
+    This function should be called to indicate that a submission has entered the
+    student training workflow part of the assessment process.
 
-def get_score(submission_uuid, requirements):   # pylint:disable=W0613
-    """
-    Training is either complete or incomplete; there is no score.
+    Args:
+        submission_uuid (str): The submission UUID for the student that is
+            initiating training.
+
+    Returns:
+        None
+
+    Raises:
+        StudentTrainingInternalError: Raised when an error occurs persisting the
+            Student Training Workflow
     """
-    return None
+    try:
+        StudentTrainingWorkflow.create_workflow(submission_uuid)
+    except Exception:
+        msg = (
+            u"An internal error has occurred while creating the student "
+            u"training workflow for submission UUID {}".format(submission_uuid)
+        )
+        logger.exception(msg)
+        raise StudentTrainingInternalError(msg)
 
 
 def validate_training_examples(rubric, examples):
@@ -354,34 +371,6 @@ def get_training_example(submission_uuid, rubric, examples):
         raise StudentTrainingInternalError(msg)
 
 
-def create_student_training_workflow(submission_uuid):
-    """
-    Creates a new student training workflow.
-
-    This function should be called to indicate that a submission has entered the
-    student training workflow part of the assessment process.
-
-    Args:
-        submission_uuid (str): The submission UUID for the student that is
-            initiating training.
-
-    Returns:
-        None
-
-    Raises:
-        StudentTrainingInternalError: Raised when an error occurs persisting the
-            Student Training Workflow
-    """
-    try:
-        StudentTrainingWorkflow.create_workflow(submission_uuid)
-    except Exception:
-        msg = (
-            u"An internal error has occurred while creating the student "
-            u"training workflow for submission UUID {}".format(submission_uuid)
-        )
-        logger.exception(msg)
-        raise StudentTrainingInternalError(msg)
-
 def assess_training_example(submission_uuid, options_selected, update_workflow=True):
     """
     Assess a training example and update the workflow.

openassessment/assessment/errors/__init__.py

@@ -7,3 +7,4 @@ Export errors from all modules defined in this package.
 from .peer import *
 from .self import *
 from .student_training import *
+from .ai import *

openassessment/assessment/errors/ai.py

+"""
+Errors related to AI assessment.
+"""
+
+from celery.exceptions import InvalidTaskError, NotConfigured, NotRegistered, QueueNotFound
+from socket import error as socket_error
+
+ANTICIPATED_CELERY_ERRORS = (InvalidTaskError, NotConfigured, NotRegistered, QueueNotFound, socket_error)
+
+class AIError(Exception):
+    """
+    A general error occurred while using the AI assessment API.
+    """
+    pass
+
+
+class AITrainingRequestError(AIError):
+    """
+    There was a problem with the request sent to the AI assessment API.
+    """
+    pass
+
+
+class AITrainingInternalError(AIError):
+    """
+    An unexpected error occurred while using the AI assessment API.
+    """
+    pass
+
+
+class AIGradingRequestError(AIError):
+    """
+    There was a problem with the request sent to the AI assessment API.
+    """
+    pass
+
+
+class AIGradingInternalError(AIError):
+    """
+    An unexpected error occurred while using the AI assessment API.
+    """
+    pass
+
+
+class AIReschedulingRequestError(AIError):
+    """
+    There was a problem with the request sent to the AI assessment API.
+    """
+    pass
+
+
+class AIReschedulingInternalError(AIError):
+    """
+    An unexpected error occurred while using the AI assessment API.
+    """
+    pass

openassessment/assessment/models/__init__.py

@@ -7,3 +7,4 @@ from .base import *
 from .peer import *
 from .training import *
 from .student_training import *
+from .ai import *

openassessment/assessment/models/base.py

@@ -56,9 +56,12 @@ class Rubric(models.Model):
     creating a new Rubric instead. This makes it easy to cache and do hash-based
     lookups.
     """
-    # SHA1 hash
+    # SHA1 hash, including prompts and explanations
     content_hash = models.CharField(max_length=40, unique=True, db_index=True)
 
+    # SHA1 hash of just the rubric structure (criteria / options / points)
+    structure_hash = models.CharField(max_length=40, db_index=True)
+
     class Meta:
         app_label = "assessment"
 
@@ -90,6 +93,38 @@ class Rubric(models.Model):
         canonical_form = json.dumps(rubric_dict, sort_keys=True)
         return sha1(canonical_form).hexdigest()
 
+    @staticmethod
+    def structure_hash_from_dict(rubric_dict):
+        """
+        Generate a hash of the rubric that includes only structural information:
+            * Criteria names and order
+            * Option names / points / order number
+
+        We do NOT include prompt text or option explanations.
+
+        NOTE: currently, we use the criterion and option names as unique identifiers,
+        so we include them in the structure.  In the future, we plan to assign
+        criteria/options unique IDs -- when we do that, we will need to update
+        this method and create a data migration for existing rubrics.
+        """
+        structure = [
+            {
+                "criterion_name": criterion.get('name'),
+                "criterion_order": criterion.get('order_num'),
+                "options": [
+                    {
+                        "option_name": option.get('name'),
+                        "option_points": option.get('points'),
+                        "option_order": option.get('order_num')
+                    }
+                    for option in criterion.get('options', [])
+                ]
+            }
+            for criterion in rubric_dict.get('criteria', [])
+        ]
+        canonical_form = json.dumps(structure, sort_keys=True)
+        return sha1(canonical_form).hexdigest()
+
     def options_ids(self, options_selected):
         """Given a mapping of selected options, return the option IDs.
 
@@ -170,6 +205,55 @@ class Rubric(models.Model):
 
         return option_id_set
 
+    def options_ids_for_points(self, criterion_points):
+        """
+        Given a mapping of selected point values, return the option IDs.
+        If there are multiple options with the same point value,
+        this will return the first one (lower order number).
+
+        Args:
+            criterion_points (dict): Mapping of criteria names to point values.
+
+        Returns:
+            list of option IDs
+
+        Raises:
+            InvalidOptionSelection
+
+        """
+        # Retrieve the mapping of criterion names/points to option IDs
+        # from the cache, if it's available
+        cache_key = "assessment.rubric_points_dict.{}".format(self.content_hash)
+        rubric_points_dict = cache.get(cache_key)
+
+        # Otherwise, create the dict by querying the database
+        if not rubric_points_dict:
+            rubric_options = CriterionOption.objects.filter(
+                criterion__rubric=self
+            ).select_related()
+
+            rubric_points_dict = defaultdict(dict)
+            for option in rubric_options:
+                if option.points not in rubric_points_dict[option.criterion.name]:
+                    rubric_points_dict[option.criterion.name][option.points] = option.id
+
+            # Store the dict in the cache
+            cache.set(cache_key, rubric_points_dict)
+
+        # Find the IDs for the options matching the specified point value
+        option_id_set = set()
+        for criterion_name, option_points in criterion_points.iteritems():
+            if (criterion_name in rubric_points_dict and option_points in rubric_points_dict[criterion_name]):
+                option_id = rubric_points_dict[criterion_name][option_points]
+                option_id_set.add(option_id)
+            else:
+                msg = u"{criterion} option with point value {points} not found in rubric".format(
+                    criterion=criterion_name, points=option_points
+                )
+                raise InvalidOptionSelection(msg)
+
+        return option_id_set
+
 
 class Criterion(models.Model):
     """A single aspect of a submission that needs assessment.

openassessment/assessment/serializers/base.py

@@ -92,7 +92,7 @@ class RubricSerializer(NestedModelSerializer):
 
     class Meta:
         model = Rubric
-        fields = ('id', 'content_hash', 'criteria', 'points_possible')
+        fields = ('id', 'content_hash', 'structure_hash', 'criteria', 'points_possible')
 
     def validate_criteria(self, attrs, source):
         """Make sure we have at least one Criterion in the Rubric."""
@@ -283,6 +283,7 @@ def rubric_from_dict(rubric_dict):
         rubric = Rubric.objects.get(content_hash=content_hash)
     except Rubric.DoesNotExist:
         rubric_dict["content_hash"] = content_hash
+        rubric_dict["structure_hash"] = Rubric.structure_hash_from_dict(rubric_dict)
         for crit_idx, criterion in enumerate(rubric_dict.get("criteria", {})):
             if "order_num" not in criterion:
                 criterion["order_num"] = crit_idx

openassessment/assessment/signals.py

+"""
+Signals for the workflow API.
+See https://docs.djangoproject.com/en/1.4/topics/signals
+"""
+
+import django.dispatch
+
+# Indicate that an assessment has completed
+# You can fire this signal from asynchronous processes (such as AI grading)
+# to notify receivers that an assessment is available.
+assessment_complete_signal = django.dispatch.Signal(providing_args=['submission_uuid'])    # pylint: disable=C0103

openassessment/assessment/tasks.py

+"""
+Celery looks for tasks in this module,
+so import the tasks we want the workers to implement.
+"""
+# pylint:disable=W0611
+from .worker.training import train_classifiers, reschedule_training_tasks
+from .worker.grading import grade_essay, reschedule_grading_tasks
\ No newline at end of file

openassessment/assessment/test/test_ai_algorithm.py

+# coding=utf-8
+"""
+Tests for AI algorithm implementations.
+"""
+import unittest
+import json
+import mock
+from openassessment.test_utils import CacheResetTest
+from openassessment.assessment.worker.algorithm import (
+    AIAlgorithm, FakeAIAlgorithm, EaseAIAlgorithm,
+    TrainingError, InvalidClassifier
+)
+
+
+EXAMPLES = [
+    AIAlgorithm.ExampleEssay(u"Mine's a tale that can't be told, my ƒяєє∂σм I hold dear.", 2),
+    AIAlgorithm.ExampleEssay(u"How years ago in days of old, when 𝒎𝒂𝒈𝒊𝒄 filled th air.", 1),
+    AIAlgorithm.ExampleEssay(u"Ṫ'ẅäṡ in the darkest depths of Ṁöṛḋöṛ, I met a girl so fair.", 1),
+    AIAlgorithm.ExampleEssay(u"But goﾚﾚuﾶ, and the evil one crept up and slipped away with her", 0),
+    AIAlgorithm.ExampleEssay(u"", 4),
+    AIAlgorithm.ExampleEssay(u".!?", 4),
+    AIAlgorithm.ExampleEssay(u"no punctuation", 4),
+    AIAlgorithm.ExampleEssay(u"one", 4),
+]
+
+INPUT_ESSAYS = [
+    u"Good times, 𝑩𝒂𝒅 𝑻𝒊𝒎𝒆𝒔, you know I had my share",
+    u"When my woman left home for a 𝒃𝒓𝒐𝒘𝒏 𝒆𝒚𝒆𝒅 𝒎𝒂𝒏",
+    u"Well, I still don't seem to 𝒄𝒂𝒓𝒆",
+    u"",
+    u".!?",
+    u"no punctuation",
+    u"one",
+]
+
+
+class AIAlgorithmTest(CacheResetTest):
+    """
+    Base class for testing AI algorithm implementations.
+    """
+    ALGORITHM_CLASS = None
+
+    def setUp(self):
+        self.algorithm = self.ALGORITHM_CLASS()   # pylint:disable=E1102
+
+    def _scores(self, classifier, input_essays):
+        """
+        Use the classifier to score multiple input essays.
+
+        Args:
+            input_essays (list of unicode): The essays to score.
+
+        Returns:
+            list of int: The scores
+
+        """
+        cache = {}
+        return [
+            self.algorithm.score(input_essay, classifier, cache)
+            for input_essay in input_essays
+        ]
+
+
+class FakeAIAlgorithmTest(AIAlgorithmTest):
+    """
+    Test for the fake AI algorithm implementation.
+    """
+    ALGORITHM_CLASS = FakeAIAlgorithm
+
+    def test_train_and_score(self):
+        classifier = self.algorithm.train_classifier(EXAMPLES)
+        expected_scores = [2, 0, 0, 0, 4, 2, 4]
+        scores = self._scores(classifier, INPUT_ESSAYS)
+        self.assertEqual(scores, expected_scores)
+
+    def test_score_classifier_missing_key(self):
+        with self.assertRaises(InvalidClassifier):
+            self.algorithm.score(u"Test input", {}, {})
+
+    def test_score_classifier_no_scores(self):
+        with self.assertRaises(InvalidClassifier):
+            self.algorithm.score(u"Test input", {'scores': []}, {})
+
+
+# Try to import EASE -- if we can't, then skip the tests that require it
+try:
+    import ease # pylint: disable=F0401,W0611
+    EASE_INSTALLED = True
+except ImportError:
+    EASE_INSTALLED = False
+
+
+@unittest.skipUnless(EASE_INSTALLED, "EASE library required")
+class EaseAIAlgorithmTest(AIAlgorithmTest):
+    """
+    Test for the EASE AI library wrapper.
+    """
+    ALGORITHM_CLASS = EaseAIAlgorithm
+
+    def test_train_and_score(self):
+        classifier = self.algorithm.train_classifier(EXAMPLES)
+        scores = self._scores(classifier, INPUT_ESSAYS)
+
+        # Check that we got scores in the correct range
+        valid_scores = set(example.score for example in EXAMPLES)
+        for score in scores:
+            self.assertIn(score, valid_scores)
+
+        # Check that the scores are consistent when we re-run the algorithm
+        repeat_scores = self._scores(classifier, INPUT_ESSAYS)
+        self.assertEqual(scores, repeat_scores)
+
+    def test_all_examples_have_same_score(self):
+        examples = [
+            AIAlgorithm.ExampleEssay(u"Test ëṡṡäÿ", 1),
+            AIAlgorithm.ExampleEssay(u"Another test ëṡṡäÿ", 1),
+        ]
+        # No assertion -- just verifying that this does not raise an exception
+        classifier = self.algorithm.train_classifier(examples)
+        self._scores(classifier, INPUT_ESSAYS)
+
+    def test_most_examples_have_same_score(self):
+        # All training examples have the same score except for one
+        examples = [
+            AIAlgorithm.ExampleEssay(u"Test ëṡṡäÿ", 1),
+            AIAlgorithm.ExampleEssay(u"Another test ëṡṡäÿ", 1),
+            AIAlgorithm.ExampleEssay(u"Different score", 0),
+        ]
+        classifier = self.algorithm.train_classifier(examples)
+        scores = self._scores(classifier, INPUT_ESSAYS)
+
+        # Check that we got scores back.
+        # This is not a very rigorous assertion -- we're mainly
+        # checking that we got this far without an exception.
+        self.assertEqual(len(scores), len(INPUT_ESSAYS))
+
+    def test_no_examples(self):
+        with self.assertRaises(TrainingError):
+            self.algorithm.train_classifier([])
+
+    def test_json_serializable(self):
+        classifier = self.algorithm.train_classifier(EXAMPLES)
+        serialized = json.dumps(classifier)
+        deserialized = json.loads(serialized)
+
+        # This should not raise an exception
+        scores = self._scores(deserialized, INPUT_ESSAYS)
+        self.assertEqual(len(scores), len(INPUT_ESSAYS))
+
+    @mock.patch('openassessment.assessment.worker.algorithm.pickle')
+    def test_pickle_serialize_error(self, mock_pickle):
+        mock_pickle.dumps.side_effect = Exception("Test error!")
+        with self.assertRaises(TrainingError):
+            self.algorithm.train_classifier(EXAMPLES)
+
+    def test_pickle_deserialize_error(self):
+        classifier = self.algorithm.train_classifier(EXAMPLES)
+        with mock.patch('openassessment.assessment.worker.algorithm.pickle.loads') as mock_call:
+            mock_call.side_effect = Exception("Test error!")
+            with self.assertRaises(InvalidClassifier):
+                self.algorithm.score(u"Test ëṡṡäÿ", classifier, {})
+
+    def test_serialized_classifier_not_a_dict(self):
+        with self.assertRaises(InvalidClassifier):
+            self.algorithm.score(u"Test ëṡṡäÿ", "not a dict", {})

openassessment/assessment/test/test_ai_models.py

+# coding=utf-8
+"""
+Test AI Django models.
+"""
+import copy
+from django.test.utils import override_settings
+from openassessment.test_utils import CacheResetTest
+from openassessment.assessment.models import (
+    AIClassifierSet, AIClassifier, AIGradingWorkflow, AI_CLASSIFIER_STORAGE
+)
+from openassessment.assessment.serializers import rubric_from_dict
+from .constants import RUBRIC
+
+
+CLASSIFIERS_DICT = {
+    u"vøȼȺƀᵾłȺɍɏ": "test data",
+    u"ﻭɼค๓๓คɼ": "more test data"
+}
+COURSE_ID = u"†3߆ çøU®ß3"
+ITEM_ID = u"fake_item_id"
+
+
+class AIClassifierTest(CacheResetTest):
+    """
+    Tests for the AIClassifier model.
+    """
+
+    def test_upload_to_path_default(self):
+        # No path prefix provided in the settings
+        classifier = self._create_classifier()
+        components = classifier.classifier_data.name.split(u'/')
+        self.assertEqual(len(components), 2)
+        self.assertEqual(components[0], AI_CLASSIFIER_STORAGE)
+        self.assertGreater(len(components[1]), 0)
+
+    @override_settings(ORA2_FILE_PREFIX=u"ƒιℓє_ρяєƒιχ")
+    def test_upload_to_path_with_prefix(self):
+        classifier = self._create_classifier()
+        components = classifier.classifier_data.name.split(u'/')
+        self.assertEqual(len(components), 3)
+        self.assertEqual(components[0], u"ƒιℓє_ρяєƒιχ")
+        self.assertEqual(components[1], AI_CLASSIFIER_STORAGE)
+        self.assertGreater(len(components[2]), 0)
+
+    def _create_classifier(self):
+        """
+        Create and return an AIClassifier.
+        """
+        rubric = rubric_from_dict(RUBRIC)
+        classifier_set = AIClassifierSet.create_classifier_set(
+            CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID
+        )
+        return AIClassifier.objects.filter(classifier_set=classifier_set)[0]
+
+
+class AIClassifierSetTest(CacheResetTest):
+    """
+    Tests for the AIClassifierSet model.
+    """
+    def setUp(self):
+        rubric = rubric_from_dict(RUBRIC)
+        self.classifier_set = AIClassifierSet.create_classifier_set(
+            CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID
+        )
+
+    def test_cache_downloads(self):
+        # Retrieve the classifier dict twice, which should hit the caching code.
+        # We can check that we're using the cache by asserting that
+        # the number of database queries decreases.
+        with self.assertNumQueries(3):
+            first = self.classifier_set.classifiers_dict
+
+        with self.assertNumQueries(0):
+            second = self.classifier_set.classifiers_dict
+
+        # Verify that we got the same value both times
+        self.assertEqual(first, second)
+
+
+class AIGradingWorkflowTest(CacheResetTest):
+    """
+    Tests for the AIGradingWorkflow model.
+    """
+    CLASSIFIERS_DICT = {
+        u"vøȼȺƀᵾłȺɍɏ": "test data",
+        u"ﻭɼค๓๓คɼ": "more test data"
+    }
+    COURSE_ID = u"test"
+    ITEM_ID = u"test"
+    ALGORITHM_ID = "test"
+
+    def setUp(self):
+        """
+        Create a new grading workflow.
+        """
+        self.rubric = rubric_from_dict(RUBRIC)
+        self.workflow = AIGradingWorkflow.objects.create(
+            submission_uuid='test', essay_text='test',
+            rubric=self.rubric, algorithm_id=self.ALGORITHM_ID,
+            item_id=self.ITEM_ID, course_id=self.COURSE_ID
+        )
+
+        # Create a rubric with a similar structure, but different prompt
+        similar_rubric_dict = copy.deepcopy(RUBRIC)
+        similar_rubric_dict['prompt'] = 'Different prompt!'
+        self.similar_rubric = rubric_from_dict(similar_rubric_dict)
+
+    def test_assign_most_recent_classifier_set(self):
+        # No classifier sets are available
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertFalse(found)
+        self.assertIs(self.workflow.classifier_set, None)
+
+        # Same rubric (exact), but different course id
+        classifier_set = AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
+            "different course!", self.ITEM_ID
+        )
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertTrue(found)
+        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)
+
+        # Same rubric (exact) but different item id
+        classifier_set = AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
+            self.COURSE_ID, "different item!"
+        )
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertTrue(found)
+        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)
+
+        # Same rubric (exact), but different algorithm id
+        # Shouldn't change, since the algorithm ID doesn't match
+        AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, self.rubric, "different algorithm!",
+            self.COURSE_ID, self.ITEM_ID
+        )
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertTrue(found)
+        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)
+
+        # Same rubric *structure*, but in a different item
+        # Shouldn't change, since the rubric isn't an exact match.
+        AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, self.similar_rubric, self.ALGORITHM_ID,
+            self.COURSE_ID, "different item!"
+        )
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertTrue(found)
+        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)
+
+        # Same rubric *structure* AND in the same course/item
+        # This should replace our current classifier set
+        classifier_set = AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, self.similar_rubric, self.ALGORITHM_ID,
+            self.COURSE_ID, self.ITEM_ID
+        )
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertTrue(found)
+        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)
+
+        # Same rubric and same course/item
+        # This is the ideal, so we should always prefer it
+        classifier_set = AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
+            self.COURSE_ID, self.ITEM_ID
+        )
+        found = self.workflow.assign_most_recent_classifier_set()
+        self.assertTrue(found)
+        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

openassessment/assessment/test/test_peer.py

@@ -11,7 +11,7 @@ from nose.tools import raises
 from openassessment.test_utils import CacheResetTest
 from openassessment.assessment.api import peer as peer_api
 from openassessment.assessment.models import (
-    Assessment, AssessmentPart, AssessmentFeedback,
+    Assessment, AssessmentPart, AssessmentFeedback, AssessmentFeedbackOption,
     PeerWorkflow, PeerWorkflowItem
 )
 from openassessment.workflow import api as workflow_api
@@ -370,7 +370,7 @@ class TestPeerApi(CacheResetTest):
         with patch.object(PeerWorkflow.objects, "get_or_create") as mock_peer:
             mock_peer.side_effect = IntegrityError("Oh no!")
             # This should not raise an exception
-            peer_api.create_peer_workflow(tim_sub["uuid"])
+            peer_api.on_start(tim_sub["uuid"])
 
     @raises(peer_api.PeerAssessmentWorkflowError)
     def test_no_submission_found_closing_assessment(self):
@@ -1095,12 +1095,12 @@ class TestPeerApi(CacheResetTest):
             1
         )
 
-    @patch.object(Assessment.objects, 'filter')
     @raises(peer_api.PeerAssessmentInternalError)
-    def test_max_score_db_error(self, mock_filter):
-        mock_filter.side_effect = DatabaseError("Bad things happened")
+    def test_max_score_db_error(self):
         tim, _ = self._create_student_and_submission("Tim", "Tim's answer")
-        peer_api.get_rubric_max_scores(tim["uuid"])
+        with patch.object(Assessment.objects, 'filter') as mock_filter:
+            mock_filter.side_effect = DatabaseError("Bad things happened")
+            peer_api.get_rubric_max_scores(tim["uuid"])
 
     @patch.object(PeerWorkflow.objects, 'get')
     @raises(peer_api.PeerAssessmentInternalError)
@@ -1121,7 +1121,7 @@ class TestPeerApi(CacheResetTest):
     def test_error_on_assessment_creation(self, mock_filter):
         mock_filter.side_effect = DatabaseError("Bad things happened")
         submission = sub_api.create_submission(STUDENT_ITEM, ANSWER_ONE)
-        peer_api.create_peer_workflow(submission["uuid"])
+        peer_api.on_start(submission["uuid"])
         peer_api.create_assessment(
             submission["uuid"], STUDENT_ITEM["student_id"],
             ASSESSMENT_DICT['options_selected'],
@@ -1188,6 +1188,145 @@ class TestPeerApi(CacheResetTest):
         new_student_item = STUDENT_ITEM.copy()
         new_student_item["student_id"] = student
         submission = sub_api.create_submission(new_student_item, answer, date)
-        peer_api.create_peer_workflow(submission["uuid"])
+        peer_api.on_start(submission["uuid"])
         workflow_api.create_workflow(submission["uuid"], STEPS)
         return submission, new_student_item
+
+
+class PeerWorkflowTest(CacheResetTest):
+    """
+    Tests for the peer workflow model.
+    """
+    STUDENT_ITEM = {
+        'student_id': 'test_student',
+        'course_id': 'test_course',
+        'item_type': 'openassessment',
+        'item_id': 'test_item'
+    }
+
+    OTHER_STUDENT = {
+        'student_id': 'test_student_2',
+        'course_id': 'test_course',
+        'item_type': 'openassessment',
+        'item_id': 'test_item'
+    }
+
+    def test_create_item_multiple_available(self):
+        # Bugfix TIM-572
+        submitter_sub = sub_api.create_submission(self.STUDENT_ITEM, 'test answer')
+        submitter_workflow = PeerWorkflow.objects.create(
+            student_id=self.STUDENT_ITEM['student_id'],
+            item_id=self.STUDENT_ITEM['item_id'],
+            course_id=self.STUDENT_ITEM['course_id'],
+            submission_uuid=submitter_sub['uuid']
+        )
+        scorer_sub = sub_api.create_submission(self.OTHER_STUDENT, 'test answer 2')
+        scorer_workflow = PeerWorkflow.objects.create(
+            student_id=self.OTHER_STUDENT['student_id'],
+            item_id=self.OTHER_STUDENT['item_id'],
+            course_id=self.OTHER_STUDENT['course_id'],
+            submission_uuid=scorer_sub['uuid']
+        )
+
+        for _ in range(2):
+            PeerWorkflowItem.objects.create(
+                scorer=scorer_workflow,
+                author=submitter_workflow,
+                submission_uuid=submitter_sub['uuid']
+            )
+
+        # This used to cause an error when `get_or_create` returned multiple workflow items
+        PeerWorkflow.create_item(scorer_workflow, submitter_sub['uuid'])
+
+
+class AssessmentFeedbackTest(CacheResetTest):
+    """
+    Tests for assessment feedback.
+    This is feedback that students give in response to the peer assessments they receive.
+    """
+
+    def setUp(self):
+        self.feedback = AssessmentFeedback.objects.create(
+            submission_uuid='test_submission',
+            feedback_text='test feedback',
+        )
+
+    def test_default_options(self):
+        self.assertEqual(self.feedback.options.count(), 0)
+
+    def test_add_options_all_new(self):
+        # We haven't created any feedback options yet, so these should be created.
+        self.feedback.add_options(['I liked my assessment', 'I thought my assessment was unfair'])
+
+        # Check the feedback options
+        options = self.feedback.options.all()
+        self.assertEqual(len(options), 2)
+        self.assertEqual(options[0].text, 'I liked my assessment')
+        self.assertEqual(options[1].text, 'I thought my assessment was unfair')
+
+    def test_add_options_some_new(self):
+        # Create one feedback option in the database
+        AssessmentFeedbackOption.objects.create(text='I liked my assessment')
+
+        # Add feedback options.  The one that's new should be created.
+        self.feedback.add_options(['I liked my assessment', 'I thought my assessment was unfair'])
+
+        # Check the feedback options
+        options = self.feedback.options.all()
+        self.assertEqual(len(options), 2)
+        self.assertEqual(options[0].text, 'I liked my assessment')
+        self.assertEqual(options[1].text, 'I thought my assessment was unfair')
+
+    def test_add_options_empty(self):
+        # No options
+        self.feedback.add_options([])
+        self.assertEqual(len(self.feedback.options.all()), 0)
+
+        # Add an option
+        self.feedback.add_options(['test'])
+        self.assertEqual(len(self.feedback.options.all()), 1)
+
+        # Add an empty list of options
+        self.feedback.add_options([])
+        self.assertEqual(len(self.feedback.options.all()), 1)
+
+    def test_add_options_duplicates(self):
+
+        # Add some options, which will be created
+        self.feedback.add_options(['I liked my assessment', 'I thought my assessment was unfair'])
+
+        # Add some more options, one of which is a duplicate
+        self.feedback.add_options(['I liked my assessment', 'I disliked my assessment'])
+
+        # There should be three options
+        options = self.feedback.options.all()
+        self.assertEqual(len(options), 3)
+        self.assertEqual(options[0].text, 'I liked my assessment')
+        self.assertEqual(options[1].text, 'I thought my assessment was unfair')
+        self.assertEqual(options[2].text, 'I disliked my assessment')
+
+        # There should be only three options in the database
+        self.assertEqual(AssessmentFeedbackOption.objects.count(), 3)
+
+    def test_add_options_all_old(self):
+        # Add some options, which will be created
+        self.feedback.add_options(['I liked my assessment', 'I thought my assessment was unfair'])
+
+        # Add some more options, all of which are duplicates
+        self.feedback.add_options(['I liked my assessment', 'I thought my assessment was unfair'])
+
+        # There should be two options
+        options = self.feedback.options.all()
+        self.assertEqual(len(options), 2)
+        self.assertEqual(options[0].text, 'I liked my assessment')
+        self.assertEqual(options[1].text, 'I thought my assessment was unfair')
+
+        # There should be two options in the database
+        self.assertEqual(AssessmentFeedbackOption.objects.count(), 2)
+
+    def test_unicode(self):
+        # Create options with unicode
+        self.feedback.add_options([u'𝓘 𝓵𝓲𝓴𝓮𝓭 𝓶𝔂 𝓪𝓼𝓼𝓮𝓼𝓼𝓶𝓮𝓷𝓽', u'ﾉ ｲんougんｲ ﾶﾘ ﾑ丂丂乇丂丂ﾶ乇刀ｲ wﾑ丂 u刀ｷﾑﾉ尺'])
+
+        # There should be two options in the database
+        self.assertEqual(AssessmentFeedbackOption.objects.count(), 2)

openassessment/assessment/test/test_rubric.py

+# -*- coding: utf-8 -*-
+"""
+Tests for assessment models.
+"""
+
+import copy
+from openassessment.test_utils import CacheResetTest
+from openassessment.assessment.models import (
+    Rubric, Criterion, CriterionOption, InvalidOptionSelection
+)
+from openassessment.assessment.test.constants import RUBRIC
+
+
+class TestRubricOptionIds(CacheResetTest):
+    """
+    Test selection of options from a rubric.
+    """
+
+    NUM_CRITERIA = 4
+    NUM_OPTIONS = 3
+
+    def setUp(self):
+        """
+        Create a rubric in the database.
+        """
+        self.rubric = Rubric.objects.create()
+        self.criteria = [
+            Criterion.objects.create(
+                rubric=self.rubric,
+                name="test criterion {num}".format(num=num),
+                order_num=num,
+            ) for num in range(self.NUM_CRITERIA)
+        ]
+
+        self.options = dict()
+        for criterion in self.criteria:
+            self.options[criterion.name] = [
+                CriterionOption.objects.create(
+                    criterion=criterion,
+                    name="test option {num}".format(num=num),
+                    order_num=num,
+                    points=num
+                ) for num in range(self.NUM_OPTIONS)
+            ]
+
+    def test_option_ids(self):
+        options_ids = self.rubric.options_ids({
+            "test criterion 0": "test option 0",
+            "test criterion 1": "test option 1",
+            "test criterion 2": "test option 2",
+            "test criterion 3": "test option 0",
+        })
+        self.assertEqual(options_ids, set([
+            self.options['test criterion 0'][0].id,
+            self.options['test criterion 1'][1].id,
+            self.options['test criterion 2'][2].id,
+            self.options['test criterion 3'][0].id
+        ]))
+
+    def test_option_ids_different_order(self):
+        options_ids = self.rubric.options_ids({
+            "test criterion 0": "test option 0",
+            "test criterion 1": "test option 1",
+            "test criterion 2": "test option 2",
+            "test criterion 3": "test option 0",
+        })
+        self.assertEqual(options_ids, set([
+            self.options['test criterion 0'][0].id,
+            self.options['test criterion 1'][1].id,
+            self.options['test criterion 2'][2].id,
+            self.options['test criterion 3'][0].id
+        ]))
+
+    def test_option_ids_missing_criteria(self):
+        with self.assertRaises(InvalidOptionSelection):
+            self.rubric.options_ids({
+                "test criterion 0": "test option 0",
+                "test criterion 1": "test option 1",
+                "test criterion 3": "test option 2",
+            })
+
+    def test_option_ids_extra_criteria(self):
+        with self.assertRaises(InvalidOptionSelection):
+            self.rubric.options_ids({
+                "test criterion 0": "test option 0",
+                "test criterion 1": "test option 1",
+                "test criterion 2": "test option 2",
+                "test criterion 3": "test option 1",
+                "extra criterion": "test",
+            })
+
+    def test_option_ids_mutated_criterion_name(self):
+        with self.assertRaises(InvalidOptionSelection):
+            self.rubric.options_ids({
+                "test mutated criterion": "test option 1",
+                "test criterion 1": "test option 1",
+                "test criterion 2": "test option 2",
+                "test criterion 3": "test option 1",
+            })
+
+    def test_option_ids_mutated_option_name(self):
+        with self.assertRaises(InvalidOptionSelection):
+            self.rubric.options_ids({
+                "test criterion 0": "test option 1",
+                "test criterion 1": "test mutated option",
+                "test criterion 2": "test option 2",
+                "test criterion 3": "test option 1",
+            })
+
+    def test_options_ids_points(self):
+        options_ids = self.rubric.options_ids_for_points({
+            'test criterion 0': 0,
+            'test criterion 1': 1,
+            'test criterion 2': 2,
+            'test criterion 3': 1
+        })
+        self.assertEqual(options_ids, set([
+            self.options['test criterion 0'][0].id,
+            self.options['test criterion 1'][1].id,
+            self.options['test criterion 2'][2].id,
+            self.options['test criterion 3'][1].id
+        ]))
+
+    def test_options_ids_points_caching(self):
+        # First call: the dict is not cached
+        with self.assertNumQueries(1):
+            self.rubric.options_ids_for_points({
+                'test criterion 0': 0,
+                'test criterion 1': 1,
+                'test criterion 2': 2,
+                'test criterion 3': 1
+            })
+
+        # Second call: the dict is not cached
+        with self.assertNumQueries(0):
+            self.rubric.options_ids_for_points({
+                'test criterion 0': 1,
+                'test criterion 1': 2,
+                'test criterion 2': 1,
+                'test criterion 3': 0
+            })
+
+    def test_options_ids_first_of_duplicate_points(self):
+        # Change the first criterion options so that the second and third
+        # option have the same point value
+        self.options['test criterion 0'][1].points = 5
+        self.options['test criterion 0'][1].save()
+        self.options['test criterion 0'][2].points = 5
+        self.options['test criterion 0'][2].save()
+
+        # Should get the first option back
+        options_ids = self.rubric.options_ids_for_points({
+            'test criterion 0': 5,
+            'test criterion 1': 1,
+            'test criterion 2': 2,
+            'test criterion 3': 1
+        })
+        self.assertIn(self.options['test criterion 0'][1].id, options_ids)
+
+    def test_options_ids_points_invalid_selection(self):
+        with self.assertRaises(InvalidOptionSelection):
+            self.rubric.options_ids_for_points({
+                'test criterion 0': self.NUM_OPTIONS + 1,
+                'test criterion 1': 2,
+                'test criterion 2': 1,
+                'test criterion 3': 0
+            })
+
+    def test_structure_hash_identical(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+
+        # Same structure, but different text should have the same structure hash
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['prompt'] = 'altered!'
+        for criterion in altered_rubric['criteria']:
+            criterion['prompt'] = 'altered!'
+            for option in criterion['options']:
+                option['explanation'] = 'altered!'
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+
+        # Expect that the two hashes are the same
+        self.assertEqual(first_hash, second_hash)
+
+    def test_structure_hash_extra_keys(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+
+        # Same structure, add some extra keys
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['extra'] = 'extra!'
+        altered_rubric['criteria'][0]['extra'] = 'extra!'
+        altered_rubric['criteria'][0]['options'][0]['extra'] = 'extra!'
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+
+        # Expect that the two hashes are the same
+        self.assertEqual(first_hash, second_hash)
+
+    def test_structure_hash_criterion_order_changed(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['criteria'][0]['order_num'] = 5
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+        self.assertNotEqual(first_hash, second_hash)
+
+    def test_structure_hash_criterion_name_changed(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['criteria'][0]['name'] = 'altered!'
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+        self.assertNotEqual(first_hash, second_hash)
+
+    def test_structure_hash_option_order_changed(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['criteria'][0]['options'][0]['order_num'] = 5
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+        self.assertNotEqual(first_hash, second_hash)
+
+    def test_structure_hash_option_name_changed(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['criteria'][0]['options'][0]['name'] = 'altered!'
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+        self.assertNotEqual(first_hash, second_hash)
+
+    def test_structure_hash_option_points_changed(self):
+        first_hash = Rubric.structure_hash_from_dict(RUBRIC)
+        altered_rubric = copy.deepcopy(RUBRIC)
+        altered_rubric['criteria'][0]['options'][0]['points'] = 'altered!'
+        second_hash = Rubric.structure_hash_from_dict(altered_rubric)
+        self.assertNotEqual(first_hash, second_hash)

openassessment/assessment/test/test_student_training_api.py

@@ -26,7 +26,7 @@ class StudentTrainingAssessmentTest(CacheResetTest):
         Create a submission.
         """
         submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
-        training_api.create_student_training_workflow(submission['uuid'])
+        training_api.on_start(submission['uuid'])
         self.submission_uuid = submission['uuid']
 
     def test_training_workflow(self):
@@ -122,7 +122,7 @@ class StudentTrainingAssessmentTest(CacheResetTest):
 
     def test_submitter_is_finished_num_queries(self):
         # Complete the first training example
-        training_api.create_student_training_workflow(self.submission_uuid)
+        training_api.on_start(self.submission_uuid)
         training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
         training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
 
@@ -160,9 +160,6 @@ class StudentTrainingAssessmentTest(CacheResetTest):
         requirements = {'num_required': 1}
         self.assertFalse(training_api.submitter_is_finished(self.submission_uuid, requirements))
 
-        # But since we're not being assessed by others, the "assessment" should be finished.
-        self.assertTrue(training_api.assessment_is_finished(self.submission_uuid, requirements))
-
     def test_get_training_example_none_available(self):
         for example in EXAMPLES:
             training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
@@ -263,13 +260,6 @@ class StudentTrainingAssessmentTest(CacheResetTest):
         is_finished = training_api.submitter_is_finished(submission_uuid, requirements)
         self.assertEqual(is_finished, bool(num_completed >= num_required))
 
-        # Assessment is finished should always be true,
-        # since we're not being assessed by others.
-        self.assertTrue(training_api.assessment_is_finished(submission_uuid, requirements))
-
-        # At no point should we receive a score!
-        self.assertIs(training_api.get_score(submission_uuid, requirements), None)
-
     def _expected_example(self, input_example, rubric):
         """
         Return the training example we would expect to retrieve for an example.
@@ -323,7 +313,7 @@ class StudentTrainingAssessmentTest(CacheResetTest):
 
         """
         pre_submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
-        training_api.create_student_training_workflow(pre_submission['uuid'])
+        training_api.on_start(pre_submission['uuid'])
         for example in examples:
             training_api.get_training_example(pre_submission['uuid'], rubric, examples)
             training_api.assess_training_example(pre_submission['uuid'], example['options_selected'])

openassessment/locale/en/LC_MESSAGES/djangojs.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: 0.1a\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2014-06-09 09:30-0400\n"
+"POT-Creation-Date: 2014-06-09 11:02-0400\n"
 "PO-Revision-Date: 2014-06-04 15:41-0400\n"
 "Last-Translator: \n"
 "Language-Team: openedx-translation <openedx-translation@googlegroups.com>\n"
@@ -87,20 +87,26 @@ msgstr ""
 #: openassessment/xblock/static/js/src/oa_server.js:253
 #: openassessment/xblock/static/js/src/oa_server.js:293
 #: openassessment/xblock/static/js/src/oa_server.js:334
+#: openassessment/xblock/static/js/src/oa_server.js:368
 msgid "This assessment could not be submitted."
 msgstr ""
 
 #: openassessment/xblock/static/js/openassessment.min.js:1
-#: openassessment/xblock/static/js/src/oa_server.js:362
+#: openassessment/xblock/static/js/src/oa_server.js:392
+msgid "One or more rescheduling tasks failed."
+msgstr ""
+
+#: openassessment/xblock/static/js/openassessment.min.js:1
+#: openassessment/xblock/static/js/src/oa_server.js:420
 msgid "This problem could not be loaded."
 msgstr ""
 
 #: openassessment/xblock/static/js/openassessment.min.js:1
-#: openassessment/xblock/static/js/src/oa_server.js:391
+#: openassessment/xblock/static/js/src/oa_server.js:449
 msgid "This problem could not be saved."
 msgstr ""
 
 #: openassessment/xblock/static/js/openassessment.min.js:1
-#: openassessment/xblock/static/js/src/oa_server.js:421
+#: openassessment/xblock/static/js/src/oa_server.js:479
 msgid "The server could not be contacted."
 msgstr ""

openassessment/locale/eo/LC_MESSAGES/djangojs.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: 0.1a\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2014-06-09 09:30-0400\n"
+"POT-Creation-Date: 2014-06-09 11:02-0400\n"
 "PO-Revision-Date: 2014-06-04 15:41-0400\n"
 "Last-Translator: \n"
 "Language-Team: openedx-translation <openedx-translation@googlegroups.com>\n"
@@ -92,20 +92,26 @@ msgstr "Thïs féédßäçk çöüld nöt ßé süßmïttéd. Ⱡ'σяєм ιρ
 #: openassessment/xblock/static/js/src/oa_server.js:253
 #: openassessment/xblock/static/js/src/oa_server.js:293
 #: openassessment/xblock/static/js/src/oa_server.js:334
+#: openassessment/xblock/static/js/src/oa_server.js:368
 msgid "This assessment could not be submitted."
 msgstr "Thïs ässéssmént çöüld nöt ßé süßmïttéd. Ⱡ'σяєм ιρѕ#"
 
 #: openassessment/xblock/static/js/openassessment.min.js:1
-#: openassessment/xblock/static/js/src/oa_server.js:362
+#: openassessment/xblock/static/js/src/oa_server.js:392
+msgid "One or more rescheduling tasks failed."
+msgstr "Öné ör möré résçhédülïng täsks fäïléd. Ⱡ'σяєм ιρѕ#"
+
+#: openassessment/xblock/static/js/openassessment.min.js:1
+#: openassessment/xblock/static/js/src/oa_server.js:420
 msgid "This problem could not be loaded."
 msgstr "Thïs prößlém çöüld nöt ßé löädéd. Ⱡ'σяєм ι#"
 
 #: openassessment/xblock/static/js/openassessment.min.js:1
-#: openassessment/xblock/static/js/src/oa_server.js:391
+#: openassessment/xblock/static/js/src/oa_server.js:449
 msgid "This problem could not be saved."
 msgstr "Thïs prößlém çöüld nöt ßé sävéd. Ⱡ'σяєм ι#"
 
 #: openassessment/xblock/static/js/openassessment.min.js:1
-#: openassessment/xblock/static/js/src/oa_server.js:421
+#: openassessment/xblock/static/js/src/oa_server.js:479
 msgid "The server could not be contacted."
 msgstr "Thé sérvér çöüld nöt ßé çöntäçtéd. Ⱡ'σяєм ιρ#"

openassessment/management/commands/simulate_ai_grading_error.py

+# -*- coding: utf-8 -*-
+"""
+Simulate failure of the worker AI grading tasks.
+When the workers fail to successfully complete AI grading,
+the AI grading workflow in the database will never be marked complete.
+
+To simulate the error condition, therefore, we create incomplete
+AI grading workflows without scheduling a grading task.
+
+To recover, a staff member can reschedule incomplete grading tasks.
+
+"""
+from django.core.management.base import BaseCommand, CommandError
+from submissions import api as sub_api
+from openassessment.assessment.models import AIGradingWorkflow, AIClassifierSet
+from openassessment.assessment.serializers import rubric_from_dict
+from openassessment.assessment.worker.algorithm import AIAlgorithm
+
+
+class Command(BaseCommand):
+    """
+    Create submissions and AI incomplete grading workflows.
+    """
+
+    help = (
+        u"Simulate failure of the worker AI grading tasks "
+        u"by creating incomplete AI grading workflows in the database."
+    )
+
+    args = '<COURSE_ID> <PROBLEM_ID> <NUM_SUBMISSIONS> <ALGORITHM_ID>'
+
+    RUBRIC_OPTIONS = [
+        {
+            "order_num": 0,
+            "name": u"poor",
+            "explanation": u"Poor job!",
+            "points": 0,
+        },
+        {
+            "order_num": 1,
+            "name": u"good",
+            "explanation": u"Good job!",
+            "points": 1,
+        }
+    ]
+
+    RUBRIC = {
+        'prompt': u"Test prompt",
+        'criteria': [
+            {
+                "order_num": 0,
+                "name": u"vocabulary",
+                "prompt": u"Vocabulary",
+                "options": RUBRIC_OPTIONS
+            },
+            {
+                "order_num": 1,
+                "name": u"grammar",
+                "prompt": u"Grammar",
+                "options": RUBRIC_OPTIONS
+            }
+        ]
+    }
+
+    EXAMPLES = {
+        "vocabulary": [
+            AIAlgorithm.ExampleEssay(
+                text=u"World Food Day is celebrated every year around the world on 16 October in honor "
+                u"of the date of the founding of the Food and Agriculture "
+                u"Organization of the United Nations in 1945.",
+                score=0
+            ),
+            AIAlgorithm.ExampleEssay(
+                text=u"Since 1981, World Food Day has adopted a different theme each year "
+                u"in order to highlight areas needed for action and provide a common focus.",
+                score=1
+            ),
+        ],
+        "grammar": [
+            AIAlgorithm.ExampleEssay(
+                text=u"Most of the themes revolve around agriculture because only investment in agriculture ",
+                score=0
+            ),
+            AIAlgorithm.ExampleEssay(
+                text=u"In spite of the importance of agriculture as the driving force "
+                u"in the economies of many developing countries, this "
+                u"vital sector is frequently starved of investment.",
+                score=1
+            )
+        ]
+    }
+
+    STUDENT_ID = u'test_student'
+    ANSWER = {'answer': 'test answer'}
+
+    def handle(self, *args, **options):
+        """
+        Execute the command.
+
+        Args:
+            course_id (unicode): The ID of the course to create submissions/workflows in.
+            item_id (unicode): The ID of the problem in the course.
+            num_submissions (int): The number of submissions/workflows to create.
+            algorithm_id (unicode): The ID of the ML algorithm to use ("fake" or "ease")
+
+        Raises:
+            CommandError
+
+        """
+        if len(args) < 4:
+            raise CommandError(u"Usage: simulate_ai_grading_error {}".format(self.args))
+
+        # Parse arguments
+        course_id = args[0].decode('utf-8')
+        item_id = args[1].decode('utf-8')
+        num_submissions = int(args[2])
+        algorithm_id = args[3].decode('utf-8')
+
+        # Create the rubric model
+        rubric = rubric_from_dict(self.RUBRIC)
+
+        # Train classifiers
+        print u"Training classifiers using {algorithm_id}...".format(algorithm_id=algorithm_id)
+        algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
+        classifier_data = {
+            criterion_name: algorithm.train_classifier(example)
+            for criterion_name, example in self.EXAMPLES.iteritems()
+        }
+        print u"Successfully trained classifiers."
+
+        # Create the classifier set
+        classifier_set = AIClassifierSet.create_classifier_set(
+            classifier_data, rubric, algorithm_id, course_id, item_id
+        )
+        print u"Successfully created classifier set with id {}".format(classifier_set.pk)
+
+        # Create submissions and grading workflows
+        for num in range(num_submissions):
+            student_item = {
+                'course_id': course_id,
+                'item_id': item_id,
+                'item_type': 'openassessment',
+                'student_id': "{base}_{num}".format(base=self.STUDENT_ID, num=num)
+            }
+            submission = sub_api.create_submission(student_item, self.ANSWER)
+            workflow = AIGradingWorkflow.start_workflow(
+                submission['uuid'], self.RUBRIC, algorithm_id
+            )
+            workflow.classifier_set = classifier_set
+            workflow.save()
+            print u"{num}: Created incomplete grading workflow with UUID {uuid}".format(
+                num=num, uuid=workflow.uuid
+            )

openassessment/management/commands/upload_oa_data.py

@@ -52,7 +52,7 @@ class Command(BaseCommand):
         Execute the command.
 
         Args:
-            course_id (unicode): The ID of the course to create submissions for.
+            course_id (unicode): The ID of the course to use.
             s3_bucket_name (unicode): The name of the S3 bucket to upload to.
 
         Raises:

openassessment/management/tests/test_simulate_ai_grading_error.py

+# -*- coding: utf-8 -*-
+"""
+Tests for the simulate AI grading error management command.
+"""
+
+from django.test.utils import override_settings
+from openassessment.test_utils import CacheResetTest
+from openassessment.management.commands import simulate_ai_grading_error
+from openassessment.assessment.models import AIGradingWorkflow
+from openassessment.assessment.worker.grading import grade_essay
+
+
+class SimulateAIGradingErrorTest(CacheResetTest):
+    """
+    Tests for the simulate AI grading error management command.
+    """
+
+    COURSE_ID = u"TɘꙅT ↄoUᴙꙅɘ"
+    ITEM_ID = u"𝖙𝖊𝖘𝖙 𝖎𝖙𝖊𝖒"
+    NUM_SUBMISSIONS = 20
+
+    AI_ALGORITHMS = {
+        "fake": "openassessment.assessment.worker.algorithm.FakeAIAlgorithm"
+    }
+
+    @override_settings(ORA2_AI_ALGORITHMS=AI_ALGORITHMS)
+    def test_simulate_ai_grading_error(self):
+        # Run the command
+        cmd = simulate_ai_grading_error.Command()
+        cmd.handle(
+            self.COURSE_ID.encode('utf-8'),
+            self.ITEM_ID.encode('utf-8'),
+            self.NUM_SUBMISSIONS,
+            "fake"
+        )
+
+        # Check that the correct number of incomplete workflows
+        # were created.  These workflows should still have
+        # a classifier set, though, because otherwise they
+        # wouldn't have been scheduled for grading
+        # (that is, the submissions were made before classifier
+        # training completed).
+        incomplete_workflows = AIGradingWorkflow.objects.filter(
+            classifier_set__isnull=False,
+            completed_at__isnull=True
+        )
+        num_errors = incomplete_workflows.count()
+        self.assertEqual(self.NUM_SUBMISSIONS, num_errors)
+
+        # Verify that we can complete the workflows successfully
+        # (that is, make sure the classifier data is valid)
+        # We're calling a Celery task method here,
+        # but we're NOT using `apply_async`, so this will
+        # execute synchronously.
+        for workflow in incomplete_workflows:
+            grade_essay(workflow.uuid)
+
+        # Now there should be no incomplete workflows
+        remaining_incomplete = AIGradingWorkflow.objects.filter(
+            classifier_set__isnull=False,
+            completed_at__isnull=True
+        ).count()
+        self.assertEqual(remaining_incomplete, 0)

openassessment/management/tests/test_upload_oa_data.py

@@ -4,15 +4,15 @@ Tests for management command that uploads submission/assessment data.
 """
 from StringIO import StringIO
 import tarfile
-from django.test import TestCase
 import boto
 import moto
+from openassessment.test_utils import CacheResetTest
 from openassessment.management.commands import upload_oa_data
 from openassessment.workflow import api as workflow_api
 from submissions import api as sub_api
 
 
-class UploadDataTest(TestCase):
+class UploadDataTest(CacheResetTest):
     """
     Test the upload management command.  Archiving and upload are in-scope,
     but the contents of the generated CSV files are tested elsewhere.

openassessment/templates/openassessmentblock/grade/oa_grade_complete.html

@@ -38,10 +38,10 @@
 
                                 <span class="question__score">
                                   <span class="label sr">{% trans "Overall Grade" %}</span>
-                                  <span class="question__score__value">{{ score.points_earned }}</span>
+                                  <span class="question__score__value">{{ criterion.median_score }}</span>
                                   <span class="label label--divider sr">out of</span>
                                   <span class="question__score__potential">
-                                    {{ score.points_possible }}
+                                    {{ criterion.total_value }}
                                     <span class="unit">{% trans "Points" %}</span>
                                   </span>
                                 </span>
@@ -108,6 +108,30 @@
                                     {% endif %}
                                 {% endfor %}
 
+                                {% for part in example_based_assessment.parts %}
+                                    {% if part.option.criterion.name == criterion.name %}
+                                        <li class="answer example-based-assessment"
+                                            id="question--{{ criterion_num }}__answer--example-based">
+                                            <h5 class="answer__title">
+                                        <span class="answer__source">
+                                          <span class="answer__source__value">{% trans "Example-Based Assessment" %}</span>
+                                        </span>
+                                        <span class="answer__value">
+                                            <span class="answer__value__label sr">{% trans "Example-Based Assessment" %}: </span>
+                                            <span class="answer__value__value">
+                                                {{ part.option.name }}
+
+                                                <span class="ui-hint hint--top" data-hint="{{ part.option.explanation }}">
+                                                  <i class="ico icon-info-sign"
+                                                     title="{% blocktrans with name=part.option.name %}More information about {{ name }}{% endblocktrans %}"></i>
+                                                </span>
+                                            </span>
+                                        </span>
+                                            </h5>
+                                        </li>
+                                    {% endif %}
+                                {% endfor %}
+
                                 {% if criterion.feedback %}
                                 <li class="answer--feedback ui-toggle-visibility is--collapsed">
                                     <h5 class="answer--feedback__title ui-toggle-visibility__control">

openassessment/templates/openassessmentblock/grade/oa_grade_waiting.html

@@ -6,7 +6,7 @@
             <span class="wrapper--copy">
                 <span class="step__label">{% trans "Your Grade" %}:</span>
                 <span class="grade__value">
-                    <span class="grade__value__title">{% trans "Waiting for Peer Assessment" %}</span>
+                    <span class="grade__value__title">{% trans "Waiting for Assessments" %}</span>
                 </span>
             </span>
         </h2>
@@ -16,7 +16,13 @@
         <div class="wrapper--step__content">
             <div class="step__content">
                 <div class="grade__value__description">
-                    <p>{% trans "Your response is still undergoing peer assessment. After your peers have assessed your response, you'll see their comments and receive your final grade." %}</p>
+                    {% if waiting == 'peer' %}
+                        <p>{% trans "Your response is still undergoing peer assessment. After your peers have assessed your response, you'll see their comments and receive your final grade." %}</p>
+                    {% elif waiting == 'example-based' %}
+                        <p>{% trans "Your response is still undergoing example-based assessment. After your response has been assessed, you'll see the comments and receive your final grade." %}</p>
+                    {% elif waiting == 'all' %}
+                        <p>{% trans "Your response is still undergoing peer assessment and example-based assessment. After your peers have assessed your response and example-based assessment is complete, you'll see your peers' comments and receive your final grade." %}</p>
+                    {% endif %}
                 </div>
             </div>
         </div>

openassessment/templates/openassessmentblock/message/oa_message_complete.html

@@ -4,8 +4,12 @@
     <h3 class="message__title">{% trans "You Have Completed This Assignment" %} </h3>
     <div class="message__content">
         <p>
-            {% if waiting %}
-                {% trans "Your grade will be available when your peers have completed their assessments of your response." %}
+            {% if waiting == 'peer' %}
+                <p>{% trans "Your grade will be available when your peers have completed their assessments of your response." %}</p>
+            {% elif waiting == 'example-based' %}
+                <p>{% trans "Your grade will be available when the example-based assessment of your response is complete." %}</p>
+            {% elif waiting == 'all' %}
+                <p>{% trans "Your grade will be available when your peers have completed their assessments of your response and the example-based assessment of your response is complete." %}</p>
             {% else %}
                 <a data-behavior="ui-scroll" href="#openassessment__grade">{% trans "Review your grade and your assessment details." %}</a>
             {% endif %}

openassessment/templates/openassessmentblock/response/oa_response_submitted.html

@@ -21,7 +21,7 @@
 
         <div class="step__message message message--complete">
             <h3 class="message__title">{% trans "Your Response Has Been Submitted" %}</h3>
-            <div class="message__content">{% blocktrans %}You'll receive your grade after some of your peers have assessed your response and you complete the <a data-behavior=\"ui-scroll\" href=\"#openassessment__peer-assessment\">peer assessment</a> and <a data-behavior=\"ui-scroll\" href=\"#openassessment__self-assessment\">self assessment</a> steps{% endblocktrans %}.</div>
+            <div class="message__content">{% blocktrans %}You'll receive your grade after some of your peers have assessed your response and you complete the <a data-behavior="ui-scroll" href="#openassessment__peer-assessment">peer assessment</a> and <a data-behavior="ui-scroll" href="#openassessment__self-assessment">self assessment</a> steps{% endblocktrans %}.</div>
         </div>
 
         <div class="step__content">

openassessment/templates/openassessmentblock/staff_debug/staff_debug.html

@@ -78,6 +78,52 @@
             </table>
         </div>
 
+        {% if display_schedule_training %}
+        <div class="staff-info__classifierset ui-staff__content__section">
+            {% if classifierset %}
+            <table class="staff-info__classifierset__table" summary="{% trans "Classifier set" %}">
+                <caption class="title">{% trans "Classifier set" %}</caption>
+                <thead>
+                    <th abbr="Field" scope="col">{% trans "Field" %}</th>
+                    <th abbr="Value" scope="col">{% trans "Value" %}</th>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td class="value">{% trans "Created at" %}</td>
+                        <td class="value">{{ classifierset.created_at }}</td>
+                    </tr>
+                    <tr>
+                        <td class="value">{% trans "Algorithm ID" %}</td>
+                        <td class="value">{{ classifierset.algorithm_id }}</td>
+                    </tr>
+                    <tr>
+                        <td class="value">{% trans "Course ID" %}</td>
+                        <td class="value">{{ classifierset.course_id }}</td>
+                    </tr>
+                    <tr>
+                        <td class="value">{% trans "Item ID" %}</td>
+                        <td class="value">{{ classifierset.item_id }}</td>
+                    </tr>
+                </tbody>
+            </table>
+            {% else %}
+            {% trans "No classifiers are available for this problem" %}
+            {% endif %}
+        </div>
+
+        <div class="staff-info__status ui-staff__content__section">
+            <a aria-role="button" href="" id="schedule_training" class="action--submit"><span class="copy">{% trans "Schedule Example-Based Assessment Training" %}</span></a>
+            <div id="schedule_training_message"></div>
+        </div>
+        {% endif %}
+
+        {% if display_reschedule_unfinished_tasks %}
+        <div class="staff-info__status ui-staff__content__section">
+            <a aria-role="button" href="" id="reschedule_unfinished_tasks" class="action--submit"><span class="copy">{% trans "Reschedule All Unfinished Example-Based Assessment Grading Tasks" %}</span></a>
+            <div id="reschedule_unfinished_tasks_message"></div>
+        </div>
+        {% endif %}
+
         <div class="staff-info__student ui-staff__content__section">
             <div class="wrapper--input" class="staff-info__student__form">
                 <form id="openassessment_student_info_form">

openassessment/templates/openassessmentblock/staff_debug/student_info.html

@@ -16,6 +16,7 @@
             </div>
         </div>
 
+        {% if peer_assessments %}
         <div class="staff-info__status ui-staff__content__section">
             <h3 class="title">{% trans "Peer Assessments for This Student" %}</h3>
             {% for assessment in peer_assessments %}
@@ -55,7 +56,9 @@
                 {% endwith %}
             {%  endfor %}
         </div>
+            {% endif %}
 
+        {% if submitted_assessments %}
         <div class="staff-info__status ui-staff__content__section">
             <h3 class="title">{% trans "Peer Assessments Completed by This Student" %}</h3>
             {% for assessment in submitted_assessments %}
@@ -95,7 +98,9 @@
                 {% endwith %}
             {%  endfor %}
         </div>
+        {% endif %}
 
+        {% if self_assessment %}
         <div class="staff-info__status ui-staff__content__section">
             <h3 class="title">{% trans "Student's Self Assessment" %}</h3>
             <table class="staff-info__status__table" summary="{% trans "Self Assessment" %}">
@@ -124,6 +129,38 @@
                 </tbody>
             </table>
         </div>
+        {% endif %}
+
+        {% if example_based_assessment %}
+            <div class="staff-info__status ui-staff__content__section">
+                <h3 class="title">{% trans "Example-Based Assessment" %}</h3>
+                <table class="staff-info__status__table" summary="{% trans "Example Based Assessment" %}">
+                    <thead>
+                    <tr>
+                        <th abbr="Criterion" scope="col">{% trans "Criterion" %}</th>
+                        <th abbr="Selected Option" scope="col">{% trans "Selected Option" %}</th>
+                        <th abbr="Points" scope="col">{% trans "Points" %}</th>
+                        <th abbr="Points Possible" scope="col">{% trans "Points Possible" %}</th>
+                    </tr>
+                    </thead>
+
+                    <tbody>
+                    {% for criterion in rubric_criteria %}
+                        {% for part in example_based_assessment.parts %}
+                            {% if part.option.criterion.name == criterion.name %}
+                                <tr>
+                                    <td class="label">{{ criterion.name }}</td>
+                                    <td class="value">{{ part.option.name }}</td>
+                                    <td class="value">{{ part.option.points }}</td>
+                                    <td class="value">{{ criterion.total_value }}</td>
+                                </tr>
+                            {% endif %}
+                        {% endfor %}
+                    {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        {% endif %}
     </div>
     {% else %}
         {% trans "Couldn't find a response for this student." %}

openassessment/test_utils.py

 """
 Test utilities
 """
-from django.core.cache import cache
+from django.core.cache import cache, get_cache
 from django.test import TestCase
 
 
@@ -11,8 +11,18 @@ class CacheResetTest(TestCase):
     """
     def setUp(self):
         super(CacheResetTest, self).setUp()
-        cache.clear()
+        self._clear_all_caches()
 
     def tearDown(self):
         super(CacheResetTest, self).tearDown()
+        self._clear_all_caches()
+
+    def _clear_all_caches(self):
+        """
+        Clear the default cache and any custom caches.
+        """
         cache.clear()
+        get_cache(
+            'django.core.cache.backends.locmem.LocMemCache',
+            LOCATION='openassessment.ai.classifiers_dict'
+        ).clear()

openassessment/workflow/api.py

@@ -2,67 +2,28 @@
 Public interface for the Assessment Workflow.
 
 """
-import copy
 import logging
 
 from django.db import DatabaseError
 
 from openassessment.assessment.api import peer as peer_api
+from openassessment.assessment.api import ai as ai_api
 from openassessment.assessment.api import student_training as training_api
 from openassessment.assessment.errors import (
-    PeerAssessmentError, StudentTrainingInternalError
+    PeerAssessmentError, StudentTrainingInternalError, AIError
 )
 from submissions import api as sub_api
 from .models import AssessmentWorkflow, AssessmentWorkflowStep
 from .serializers import AssessmentWorkflowSerializer
+from .errors import (
+    AssessmentWorkflowInternalError, AssessmentWorkflowRequestError,
+    AssessmentWorkflowNotFoundError
+)
 
 logger = logging.getLogger(__name__)
 
 
-class AssessmentWorkflowError(Exception):
-    """An error that occurs during workflow actions.
-
-    This error is raised when the Workflow API cannot perform a requested
-    action.
-
-    """
-    pass
-
-
-class AssessmentWorkflowInternalError(AssessmentWorkflowError):
-    """An error internal to the Workflow API has occurred.
-
-    This error is raised when an error occurs that is not caused by incorrect
-    use of the API, but rather internal implementation of the underlying
-    services.
-
-    """
-    pass
-
-
-class AssessmentWorkflowRequestError(AssessmentWorkflowError):
-    """This error is raised when there was a request-specific error
-
-    This error is reserved for problems specific to the use of the API.
-
-    """
-
-    def __init__(self, field_errors):
-        Exception.__init__(self, repr(field_errors))
-        self.field_errors = copy.deepcopy(field_errors)
-
-
-class AssessmentWorkflowNotFoundError(AssessmentWorkflowError):
-    """This error is raised when no submission is found for the request.
-
-    If a state is specified in a call to the API that results in no matching
-    Submissions, this error may be raised.
-
-    """
-    pass
-
-
-def create_workflow(submission_uuid, steps):
+def create_workflow(submission_uuid, steps, on_init_params=None):
     """Begins a new assessment workflow.
 
     Create a new workflow that other assessments will record themselves against.
@@ -73,6 +34,10 @@ def create_workflow(submission_uuid, steps):
         steps (list): List of steps that are part of the workflow, in the order
             that the user must complete them. Example: `["peer", "self"]`
 
+    Kwargs:
+        on_init_params (dict): The parameters to pass to each assessment module
+            on init.  Keys are the assessment step names.
+
     Returns:
         dict: Assessment workflow information with the following
             `uuid` = UUID of this `AssessmentWorkflow`
@@ -102,8 +67,16 @@ def create_workflow(submission_uuid, steps):
             .format(submission_uuid, specific_err_msg)
         )
 
+    if on_init_params is None:
+        on_init_params = dict()
+
     try:
-        submission_dict = sub_api.get_submission_and_student(submission_uuid)
+        workflow = AssessmentWorkflow.start_workflow(submission_uuid, steps, on_init_params)
+        logger.info((
+            u"Started assessment workflow for "
+            u"submission UUID {uuid} with steps {steps}"
+        ).format(uuid=submission_uuid, steps=steps))
+        return AssessmentWorkflowSerializer(workflow).data
     except sub_api.SubmissionNotFoundError:
         err_msg = sub_err_msg("submission not found")
         logger.error(err_msg)
@@ -118,61 +91,17 @@ def create_workflow(submission_uuid, steps):
             u"retrieving submission {} failed with unknown error: {}"
             .format(submission_uuid, err)
         )
-
-    # Raise an error if they specify a step we don't recognize...
-    invalid_steps = set(steps) - set(AssessmentWorkflow.STEPS)
-    if invalid_steps:
-        raise AssessmentWorkflowRequestError(
-            u"The following steps were not recognized: {}; Must be one of {}".format(
-                invalid_steps, AssessmentWorkflow.STEPS
-            )
-        )
-
-    # We're not using a serializer to deserialize this because the only variable
-    # we're getting from the outside is the submission_uuid, which is already
-    # validated by this point.
-    status = AssessmentWorkflow.STATUS.peer
-    if steps[0] == "peer":
-        try:
-            peer_api.create_peer_workflow(submission_uuid)
-        except PeerAssessmentError as err:
-            err_msg = u"Could not create assessment workflow: {}".format(err)
-            logger.exception(err_msg)
-            raise AssessmentWorkflowInternalError(err_msg)
-    elif steps[0] == "self":
-        status = AssessmentWorkflow.STATUS.self
-    elif steps[0] == "training":
-        status = AssessmentWorkflow.STATUS.training
-        try:
-            training_api.create_student_training_workflow(submission_uuid)
-        except StudentTrainingInternalError as err:
-            err_msg = u"Could not create assessment workflow: {}".format(err)
-            logger.exception(err_msg)
-            raise AssessmentWorkflowInternalError(err_msg)
-
-    try:
-        workflow = AssessmentWorkflow.objects.create(
-            submission_uuid=submission_uuid,
-            status=status,
-            course_id=submission_dict['student_item']['course_id'],
-            item_id=submission_dict['student_item']['item_id'],
-        )
-        workflow_steps = [
-            AssessmentWorkflowStep(
-                workflow=workflow, name=step, order_num=i
-            )
-            for i, step in enumerate(steps)
-        ]
-        workflow.steps.add(*workflow_steps)
-    except (
-        DatabaseError,
-        sub_api.SubmissionError
-    ) as err:
-        err_msg = u"Could not create assessment workflow: {}".format(err)
+    except DatabaseError:
+        err_msg = u"Could not create assessment workflow for submission UUID: {}".format(submission_uuid)
+        logger.exception(err_msg)
+        raise AssessmentWorkflowInternalError(err_msg)
+    except:
+        err_msg = (
+            u"An unexpected error occurred while creating "
+            u"the workflow for submission UUID {}"
+        ).format(submission_uuid)
         logger.exception(err_msg)
         raise AssessmentWorkflowInternalError(err_msg)
-
-    return AssessmentWorkflowSerializer(workflow).data
 
 
 def get_workflow_for_submission(submission_uuid, assessment_requirements):
@@ -335,6 +264,10 @@ def update_from_assessments(submission_uuid, assessment_requirements):
 
     try:
         workflow.update_from_assessments(assessment_requirements)
+        logger.info((
+            u"Updated workflow for submission UUID {uuid} "
+            u"with requirements {reqs}"
+        ).format(uuid=submission_uuid, reqs=assessment_requirements))
         return _serialized_with_details(workflow, assessment_requirements)
     except PeerAssessmentError as err:
         err_msg = u"Could not update assessment workflow: {}".format(err)
@@ -364,6 +297,10 @@ def get_status_counts(course_id, item_id, steps):
         ]
 
     """
+    # The AI status exists for workflow logic, but no student will ever be in
+    # the AI status, so we should never return it.
+    statuses = steps + AssessmentWorkflow.STATUSES
+    if 'ai' in statuses: statuses.remove('ai')
     return [
         {
             "status": status,
@@ -373,7 +310,7 @@ def get_status_counts(course_id, item_id, steps):
                 item_id=item_id,
             ).count()
         }
-        for status in steps + AssessmentWorkflow.STATUSES
+        for status in statuses
     ]
 
 

openassessment/workflow/errors.py

+"""
+Errors defined by the workflow API.
+"""
+import copy
+
+
+class AssessmentWorkflowError(Exception):
+    """An error that occurs during workflow actions.
+
+    This error is raised when the Workflow API cannot perform a requested
+    action.
+
+    """
+    pass
+
+
+class AssessmentWorkflowInternalError(AssessmentWorkflowError):
+    """An error internal to the Workflow API has occurred.
+
+    This error is raised when an error occurs that is not caused by incorrect
+    use of the API, but rather internal implementation of the underlying
+    services.
+
+    """
+    pass
+
+
+class AssessmentWorkflowRequestError(AssessmentWorkflowError):
+    """This error is raised when there was a request-specific error
+
+    This error is reserved for problems specific to the use of the API.
+
+    """
+
+    def __init__(self, field_errors):
+        Exception.__init__(self, repr(field_errors))
+        self.field_errors = copy.deepcopy(field_errors)
+
+
+class AssessmentWorkflowNotFoundError(AssessmentWorkflowError):
+    """This error is raised when no submission is found for the request.
+
+    If a state is specified in a call to the API that results in no matching
+    Submissions, this error may be raised.
+
+    """
+    pass
+
+
+class AssessmentApiLoadError(AssessmentWorkflowInternalError):
+    """
+    The assessment API could not be loaded.
+    """
+    def __init__(self, assessment_name, api_path):
+        msg = u"Could not load assessment API for {} from {}".format(
+            assessment_name, api_path
+        )
+        super(AssessmentApiLoadError, self).__init__(msg)

openassessment/workflow/test/data/assessments.json

@@ -57,6 +57,66 @@
             },
             "self": {}
         }
+    },
+    "ai": {
+        "steps": ["ai"],
+        "requirements": {
+            "ai": {}
+        }
+    },
+    "ai_peer": {
+        "steps": ["ai", "peer"],
+        "requirements": {
+            "ai": {},
+            "peer": {
+                "must_grade": 5,
+                "must_be_graded_by": 3
+            }
+        }
+    },
+    "ai_training_peer": {
+        "steps": ["ai", "training", "peer"],
+        "requirements": {
+            "ai": {},
+            "training": {
+                "num_required": 2
+            },
+            "peer": {
+                "must_grade": 5,
+                "must_be_graded_by": 3
+            }
+        }
+    },
+    "ai_self": {
+        "steps": ["ai", "self"],
+        "requirements": {
+            "ai": {},
+            "self": {}
+        }
+    },
+    "ai_peer_self": {
+        "steps": ["ai", "peer", "self"],
+        "requirements": {
+            "ai": {},
+            "peer": {
+                "must_grade": 5,
+                "must_be_graded_by": 3
+            },
+            "self": {}
+        }
+    },
+    "ai_training_peer_self": {
+        "steps": ["ai", "training", "peer", "self"],
+        "requirements": {
+            "ai": {},
+            "training": {
+                "num_required": 2
+            },
+            "peer": {
+                "must_grade": 5,
+                "must_be_graded_by": 3
+            },
+            "self": {}
+        }
     }
-
 }
\ No newline at end of file

openassessment/workflow/test/events.py

-"""
-This is just a dummy event logger to test our ability to dyanmically change this
-value based on configuration. All this should go away when we start using the
-edx-analytics approved library (once that's ready to be used on prod).
-"""
-
-def fake_event_logger(event):
-    print event

openassessment/workflow/test/test_models.py

-from django.test import TestCase
-from mock import patch
-from nose.tools import raises
-
-from openassessment.workflow.models import emit_event
-from openassessment.workflow.test.events import fake_event_logger
-
-class TestEmitEvent(TestCase):
-
-    def test_emit_wired_correctly(self):
-        self.assertEqual(emit_event, fake_event_logger)

openassessment/workflow/test/test_signals.py

+"""
+Tests for Django signals and receivers defined by the workflow API.
+"""
+import mock
+from django.db import DatabaseError
+import ddt
+from submissions import api as sub_api
+from openassessment.test_utils import CacheResetTest
+from openassessment.workflow import api as workflow_api
+from openassessment.workflow.models import AssessmentWorkflow
+from openassessment.assessment.signals import assessment_complete_signal
+
+
+@ddt.ddt
+class UpdateWorkflowSignalTest(CacheResetTest):
+    """
+    Test for the update workflow signal.
+    """
+    STUDENT_ITEM = {
+        "student_id": "test student",
+        "item_id": "test item",
+        "course_id": "test course",
+        "item_type": "openassessment",
+    }
+
+    def setUp(self):
+        """
+        Create a submission.
+        """
+        submission = sub_api.create_submission(self.STUDENT_ITEM, "test answer")
+        self.submission_uuid = submission['uuid']
+
+    def test_update_signal_no_workflow(self):
+        # Without defining a workflow, send the signal
+        # The receiver should catch and log the exception
+        assessment_complete_signal.send(sender=None, submission_uuid=self.submission_uuid)
+
+    def test_update_signal_no_submission_uuid(self):
+        # Try to send the signal without specifying a submission UUID
+        # The receiver should catch and log the exception
+        assessment_complete_signal.send(sender=None)
+
+    def test_update_signal_updates_workflow(self):
+        # Start a workflow for the submission
+        workflow_api.create_workflow(self.submission_uuid, ['self'])
+
+        # Spy on the workflow update call
+        with mock.patch.object(AssessmentWorkflow, 'update_from_assessments') as mock_update:
+
+            # Send a signal to update the workflow
+            assessment_complete_signal.send(sender=None, submission_uuid=self.submission_uuid)
+
+            # Verify that the workflow model update was called
+            mock_update.assert_called_once_with(None)
+
+    @ddt.data(DatabaseError, IOError)
+    @mock.patch.object(AssessmentWorkflow.objects, 'get')
+    def test_errors(self, error, mock_call):
+        # Start a workflow for the submission
+        workflow_api.create_workflow(self.submission_uuid, ['self'])
+
+        # The receiver should catch and log the error
+        mock_call.side_effect = error("OH NO!")
+        assessment_complete_signal.send(sender=None, submission_uuid=self.submission_uuid)

openassessment/xblock/data_conversion.py

@@ -53,4 +53,23 @@ def convert_training_examples_list_to_dict(examples_list):
             }
         }
         for ex in examples_list
-    ]
+    ]
\ No newline at end of file
+
+
+def create_rubric_dict(prompt, criteria):
+    """
+    Construct a serialized rubric model in the format expected
+    by the assessments app.
+
+    Args:
+        prompt (unicode): The rubric prompt.
+        criteria (list of dict): The serialized rubric criteria.
+
+    Returns:
+        dict
+
+    """
+    return {
+        "prompt": prompt,
+        "criteria": criteria
+    }

openassessment/xblock/grade_mixin.py

@@ -9,6 +9,7 @@ from xblock.core import XBlock
 
 from openassessment.assessment.api import peer as peer_api
 from openassessment.assessment.api import self as self_api
+from openassessment.assessment.api import ai as ai_api
 from openassessment.assessment.errors import SelfAssessmentError, PeerAssessmentError
 from submissions import api as sub_api
 
@@ -51,7 +52,7 @@ class GradeMixin(object):
             if status == "done":
                 path, context = self.render_grade_complete(workflow)
             elif status == "waiting":
-                path = 'openassessmentblock/grade/oa_grade_waiting.html'
+                path, context = self.render_grade_waiting(workflow)
             elif status is None:
                 path = 'openassessmentblock/grade/oa_grade_not_started.html'
             else:  # status is 'self' or 'peer', which implies that the workflow is incomplete
@@ -61,6 +62,22 @@ class GradeMixin(object):
         else:
             return self.render_assessment(path, context)
 
+    def render_grade_waiting(self, workflow):
+        """
+        Render the grade waiting state.
+
+        Args:
+            workflow (dict): The serialized Workflow model.
+
+        Returns:
+            tuple of context (dict) and template_path (string)
+
+        """
+        context = {
+            "waiting": self.get_waiting_details(workflow["status_details"])
+        }
+        return 'openassessmentblock/grade/oa_grade_waiting.html', context
+
     def render_grade_complete(self, workflow):
         """
         Render the grade complete state.
@@ -75,19 +92,22 @@ class GradeMixin(object):
         assessment_steps = self.assessment_steps
         submission_uuid = workflow['submission_uuid']
 
+        example_based_assessment = None
+        self_assessment = None
+        feedback = None
+        peer_assessments = []
+        has_submitted_feedback = False
+
         if "peer-assessment" in assessment_steps:
             feedback = peer_api.get_assessment_feedback(submission_uuid)
             peer_assessments = peer_api.get_assessments(submission_uuid)
             has_submitted_feedback = feedback is not None
-        else:
-            feedback = None
-            peer_assessments = []
-            has_submitted_feedback = False
 
         if "self-assessment" in assessment_steps:
             self_assessment = self_api.get_assessment(submission_uuid)
-        else:
-            self_assessment = None
+
+        if "example-based-assessment" in assessment_steps:
+            example_based_assessment = ai_api.get_latest_assessment(submission_uuid)
 
         feedback_text = feedback.get('feedback', '') if feedback else ''
         student_submission = sub_api.get_submission(submission_uuid)
@@ -104,6 +124,7 @@ class GradeMixin(object):
             'student_submission': student_submission,
             'peer_assessments': peer_assessments,
             'self_assessment': self_assessment,
+            'example_based_assessment': example_based_assessment,
             'rubric_criteria': self._rubric_criteria_with_feedback(peer_assessments),
             'has_submitted_feedback': has_submitted_feedback,
         }
@@ -112,10 +133,13 @@ class GradeMixin(object):
         # Note that we are updating a *copy* of the rubric criteria stored in
         # the XBlock field
         max_scores = peer_api.get_rubric_max_scores(submission_uuid)
+        median_scores = None
         if "peer-assessment" in assessment_steps:
             median_scores = peer_api.get_assessment_median_scores(submission_uuid)
         elif "self-assessment" in assessment_steps:
             median_scores = self_api.get_assessment_scores_by_criteria(submission_uuid)
+        elif "example-based-assessment" in assessment_steps:
+            median_scores = ai_api.get_assessment_scores_by_criteria(submission_uuid)
 
         if median_scores is not None and max_scores is not None:
             for criterion in context["rubric_criteria"]:

openassessment/xblock/message_mixin.py

@@ -38,6 +38,8 @@ class MessageMixin(object):
 
         # Finds the cannonical status of the workflow and the is_closed status of the problem
         status = workflow.get('status')
+        status_details = workflow.get('status_details', {})
+
         is_closed = deadline_info.get('general').get('is_closed')
 
         # Finds the status_information which describes the closed status of the current step (defaults to submission)
@@ -53,7 +55,7 @@ class MessageMixin(object):
         # Render the instruction message based on the status of the workflow
         # and the closed status.
         if status == "done" or status == "waiting":
-            path, context = self.render_message_complete(status)
+            path, context = self.render_message_complete(status_details)
         elif is_closed or status_is_closed:
             path, context = self.render_message_closed(status_info)
         elif status == "self":
@@ -66,7 +68,7 @@ class MessageMixin(object):
             path, context = self.render_message_open(deadline_info)
         return self.render_assessment(path, context)
 
-    def render_message_complete(self, status):
+    def render_message_complete(self, status_details):
         """
         Renders the "Complete" message state (Either Waiting or Done)
 
@@ -76,10 +78,10 @@ class MessageMixin(object):
         Returns:
             The path (String) and context (dict) to render the "Complete" message template
         """
-
         context = {
-            "waiting": (status == "waiting")
+            "waiting": self.get_waiting_details(status_details),
         }
+
         return 'openassessmentblock/message/oa_message_complete.html', context
 
     def render_message_training(self, deadline_info):

openassessment/xblock/openassessmentblock.py

@@ -26,6 +26,7 @@ from openassessment.xblock.xml import parse_from_xml, serialize_content_to_xml
 from openassessment.xblock.staff_info_mixin import StaffInfoMixin
 from openassessment.xblock.workflow_mixin import WorkflowMixin
 from openassessment.workflow import api as workflow_api
+from openassessment.workflow.errors import AssessmentWorkflowError
 from openassessment.xblock.student_training_mixin import StudentTrainingMixin
 from openassessment.xblock.validation import validator
 from openassessment.xblock.resolve_dates import resolve_dates, DISTANT_PAST, DISTANT_FUTURE
@@ -69,6 +70,7 @@ UI_MODELS = {
 
 VALID_ASSESSMENT_TYPES = [
     "student-training",
+    "example-based-assessment",
     "peer-assessment",
     "self-assessment",
 ]
@@ -216,7 +218,7 @@ class OpenAssessmentBlock(
         # case we may have a score available.
         try:
             self.update_workflow_status()
-        except workflow_api.AssessmentWorkflowError:
+        except AssessmentWorkflowError:
             # Log the exception, but continue loading the page
             logger.exception('An error occurred while updating the workflow on page load.')
 
@@ -239,6 +241,19 @@ class OpenAssessmentBlock(
         return frag
 
     @property
+    def is_admin(self):
+        """
+        Check whether the user has global staff permissions.
+
+        Returns:
+            bool
+        """
+        if hasattr(self, 'xmodule_runtime'):
+            return getattr(self.xmodule_runtime, 'user_is_admin', False)
+        else:
+            return False
+
+    @property
     def is_course_staff(self):
         """
         Check whether the user has course staff permissions for this XBlock.
@@ -251,6 +266,8 @@ class OpenAssessmentBlock(
         else:
             return False
 
+
+
     @property
     def in_studio_preview(self):
         """
@@ -276,8 +293,9 @@ class OpenAssessmentBlock(
         """
         ui_models = [UI_MODELS["submission"]]
         for assessment in self.valid_assessments:
-            ui_model = UI_MODELS[assessment["name"]]
-            ui_models.append(dict(assessment, **ui_model))
+            ui_model = UI_MODELS.get(assessment["name"])
+            if ui_model:
+                ui_models.append(dict(assessment, **ui_model))
         ui_models.append(UI_MODELS["grade"])
         return ui_models
 
@@ -295,6 +313,10 @@ class OpenAssessmentBlock(
                 load('static/xml/unicode.xml')
             ),
             (
+                "OpenAssessmentBlock Example Based Rubric",
+                load('static/xml/example_based_example.xml')
+            ),
+            (
                 "OpenAssessmentBlock Poverty Rubric",
                 load('static/xml/poverty_rubric_example.xml')
             ),
@@ -515,6 +537,51 @@ class OpenAssessmentBlock(
         else:
             return False, None, open_range[0], open_range[1]
 
+    def get_waiting_details(self, status_details):
+        """
+        Returns the specific waiting status based on the given status_details.
+        This status can currently be peer, example-based, or both. This is
+        determined by checking that status details to see if all assessment
+        modules have been graded.
+
+        Args:
+            status_details (dict): A dictionary containing the details of each
+                assessment module status. This will contain keys such as
+                "peer" and "ai", referring to dictionaries, which in turn will
+                have the key "graded". If this key has a value set, these
+                assessment modules have been graded.
+
+        Returns:
+            A string of "peer", "exampled-based", or "all" to indicate which
+            assessment modules in the workflow are waiting on assessments.
+            Returns None if no module is waiting on an assessment.
+
+        Examples:
+            >>> now = dt.datetime.utcnow().replace(tzinfo=pytz.utc)
+            >>> status_details = {
+            >>>     'peer': {
+            >>>         'completed': None,
+            >>>         'graded': now
+            >>>     },
+            >>>     'ai': {
+            >>>         'completed': now,
+            >>>         'graded': None
+            >>>     }
+            >>> }
+            >>> self.get_waiting_details(status_details)
+            "peer"
+        """
+        waiting = None
+        peer_waiting = "peer" in status_details and not status_details["peer"]["graded"]
+        ai_waiting = "ai" in status_details and not status_details["ai"]["graded"]
+        if peer_waiting and ai_waiting:
+            waiting = "all"
+        elif peer_waiting:
+            waiting = "peer"
+        elif ai_waiting:
+            waiting = "example-based"
+        return waiting
+
     def is_released(self, step=None):
         """
         Check if a question has been released.

openassessment/xblock/peer_assessment_mixin.py

@@ -9,6 +9,8 @@ from openassessment.assessment.errors import (
     PeerAssessmentRequestError, PeerAssessmentInternalError, PeerAssessmentWorkflowError
 )
 import openassessment.workflow.api as workflow_api
+from openassessment.workflow.errors import AssessmentWorkflowError
+
 from .resolve_dates import DISTANT_FUTURE
 
 logger = logging.getLogger(__name__)
@@ -98,7 +100,7 @@ class PeerAssessmentMixin(object):
                 if assessment:
                     self.update_workflow_status(submission_uuid=assessment['submission_uuid'])
                 self.update_workflow_status()
-            except workflow_api.AssessmentWorkflowError:
+            except AssessmentWorkflowError:
                 logger.exception(
                     u"Workflow error occurred when submitting peer assessment "
                     u"for submission {}".format(self.submission_uuid)

openassessment/xblock/self_assessment_mixin.py

@@ -64,8 +64,9 @@ class SelfAssessmentMixin(object):
         # and `workflow_status` will be None.
         workflow = self.get_workflow_info()
         workflow_status = workflow.get('status')
+        self_complete = workflow.get('status_details', {}).get('self', {}).get('complete', False)
 
-        if workflow_status == 'waiting' or workflow_status == 'done':
+        if self_complete:
             path = 'openassessmentblock/self/oa_self_complete.html'
         elif workflow_status == 'self' or problem_closed:
             assessment = self_api.get_assessment(workflow.get("submission_uuid"))

openassessment/xblock/static/js/spec/oa_staff_info.js

 /**
-Tests for staff info.
-**/
+ Tests for OpenAssessment Student Training view.
+ **/
+
 describe("OpenAssessment.StaffInfoView", function() {
 
     // Stub server that returns dummy data for the staff info view
@@ -18,6 +19,23 @@ describe("OpenAssessment.StaffInfoView", function() {
                 defer.resolveWith(this, [fragment]);
             });
         };
+
+        this.scheduleTraining = function() {
+            var server = this;
+            return $.Deferred(function(defer) {
+                defer.resolveWith(server, [server.data]);
+            }).promise();
+        };
+
+        this.rescheduleUnfinishedTasks = function() {
+            var server = this;
+            return $.Deferred(function(defer) {
+                defer.resolveWith(server, [server.data]);
+            }).promise();
+        };
+
+        this.data = {};
+
     };
 
     // Stub base view
@@ -61,6 +79,30 @@ describe("OpenAssessment.StaffInfoView", function() {
         baseView = new StubBaseView();
     });
 
+    it("schedules training of AI classifiers", function() {
+        server.data = {
+            "success": true,
+            "workflow_uuid": "abc123",
+            "msg": "Great success."
+        };
+        spyOn(server, 'scheduleTraining').andCallThrough();
+
+        // Load the fixture
+        loadFixtures('oa_base.html');
+
+        // Load the view
+        var el = $("#openassessment-base").get(0);
+        var view = new OpenAssessment.StaffInfoView(el, server, baseView);
+        view.load();
+
+        // Submit the assessment
+        view.scheduleTraining();
+
+        // Expect that the assessment was sent to the server
+        expect(server.scheduleTraining).toHaveBeenCalled();
+    });
+
+
     it("Loads staff info if the page contains a course staff section", function() {
         // Load the fixture for the container page that DOES include a course staff section
         loadFixtures('oa_base_course_staff.html');
@@ -73,4 +115,45 @@ describe("OpenAssessment.StaffInfoView", function() {
         loadFixtures('oa_base.html');
         assertStaffInfoAjaxCall(false);
     });
+
+    it("reschedules training of AI tasks", function() {
+        server.data = {
+            "success": true,
+            "workflow_uuid": "abc123",
+            "msg": "Great success."
+        };
+
+        var el = $("#openassessment-base").get(0);
+        var view = new OpenAssessment.StaffInfoView(el, server, baseView);
+        view.load();
+
+        spyOn(server, 'rescheduleUnfinishedTasks').andCallThrough();
+
+        // Test the Rescheduling
+        view.rescheduleUnfinishedTasks();
+
+        // Expect that the server was instructed to reschedule Unifinished Taks
+        expect(server.rescheduleUnfinishedTasks).toHaveBeenCalled();
+    });
+
+    it("reschedules training of AI tasks", function() {
+        server.data = {
+            "success": false,
+            "workflow_uuid": "abc123",
+            "errMsg": "Stupendous Failure."
+        };
+
+        var el = $("#openassessment-base").get(0);
+        var view = new OpenAssessment.StaffInfoView(el, server, baseView);
+        view.load();
+
+        spyOn(server, 'rescheduleUnfinishedTasks').andCallThrough();
+
+        // Test the Rescheduling
+        view.rescheduleUnfinishedTasks();
+
+        // Expect that the server was instructed to reschedule Unifinished Taks
+        expect(server.rescheduleUnfinishedTasks).toHaveBeenCalled();
+    });
+
 });

openassessment/xblock/static/js/src/oa_edit.js

@@ -300,11 +300,9 @@ OpenAssessment.StudioView.prototype = {
 function OpenAssessmentEditor(runtime, element) {
 
     /**
-     Initialize the editing interface on page load.
-     **/
-
+    Initialize the editing interface on page load.
+    **/
     var server = new OpenAssessment.Server(runtime, element);
     var view = new OpenAssessment.StudioView(runtime, element, server);
     view.load();
-
-};
+}
\ No newline at end of file

openassessment/xblock/static/js/src/oa_server.js

@@ -337,6 +337,64 @@ OpenAssessment.Server.prototype = {
     },
 
     /**
+    Schedules classifier training for Example Based Assessment for this
+    Location.
+
+    Returns:
+        A JQuery promise, which resolves with a message indicating the results
+        of the scheduling request.
+
+    Example:
+        server.scheduleTraining().done(
+            function(msg) { console.log("Success!"); }
+            alert(msg);
+        ).fail(
+            function(errorMsg) { console.log(errorMsg); }
+        );
+    **/
+    scheduleTraining: function() {
+        var url = this.url('schedule_training');
+        return $.Deferred(function(defer) {
+            $.ajax({ type: "POST", url: url, data: "\"\""}).done(
+                function(data) {
+                    if (data.success) {
+                        defer.resolveWith(this, [data.msg]);
+                    }
+                    else {
+                        defer.rejectWith(this, [data.msg]);
+                    }
+                }
+            ).fail(function(data) {
+                    defer.rejectWith(this, [gettext('This assessment could not be submitted.')]);
+                });
+        });
+    },
+
+    /**
+    Reschedules grading tasks for example based assessments
+
+    Returns:
+        JQuery Promise which will resolve with a message indicating success or failure of the scheduling
+    **/
+    rescheduleUnfinishedTasks: function() {
+        var url = this.url('reschedule_unfinished_tasks');
+        return $.Deferred(function(defer) {
+            $.ajax({ type: "POST", url: url, data: "\"\""}).done(
+                function(data) {
+                    if (data.success) {
+                        defer.resolveWith(this, [data.msg]);
+                    }
+                    else {
+                        defer.rejectWith(this, [data.msg]);
+                    }
+                }
+            ).fail(function(data) {
+                    defer.rejectWith(this, [gettext('One or more rescheduling tasks failed.')]);
+            });
+        });
+    },
+
+    /**
     Load the XBlock's XML definition from the server.
 
     Returns:

openassessment/xblock/static/js/src/oa_staff_info.js

@@ -87,5 +87,54 @@ OpenAssessment.StaffInfoView.prototype = {
                 view.loadStudentInfo();
             }
         );
+
+        // Install a click handler for scheduling AI classifier training
+        sel.find('#schedule_training').click(
+            function(eventObject) {
+                eventObject.preventDefault();
+                view.scheduleTraining();
+            }
+        );
+
+        // Install a click handler for rescheduling unfinished AI tasks for this problem
+        sel.find('#reschedule_unfinished_tasks').click(
+            function(eventObject) {
+                eventObject.preventDefault();
+                view.rescheduleUnfinishedTasks();
+            }
+        );
+    },
+
+    /**
+     Sends a request to the server to schedule the training of classifiers for
+     this problem's Example Based Assessments.
+
+     **/
+    scheduleTraining: function() {
+        var view = this;
+        this.server.scheduleTraining().done(
+                function(msg) {
+                    $('#schedule_training_message', this.element).text(msg)
+                }
+            ).fail(function(errMsg) {
+                $('#schedule_training_message', this.element).text(errMsg)
+            });
+    },
+
+    /**
+     Begins the process of rescheduling all unfinished grading tasks. This incdludes
+     checking if the classifiers have been created, and grading any unfinished
+     student submissions.
+
+     **/
+    rescheduleUnfinishedTasks: function() {
+        var view = this;
+        this.server.rescheduleUnfinishedTasks().done(
+                function(msg) {
+                    $('#reschedule_unfinished_tasks_message', this.element).text(msg)
+                }
+            ).fail(function(errMsg) {
+                $('#reschedule_unfinished_tasks_message', this.element).text(errMsg)
+            });
     }
 };

openassessment/xblock/static/sass/oa/elements/_staff.scss

@@ -84,11 +84,11 @@
   }
 
   // UI - status (table)
-  .staff-info__status {
+  .staff-info__status, .staff-info__classifierset {
 
   }
 
-  .staff-info__status__table {
+  .staff-info__status__table, .staff-info__classifierset__table {
     @extend %copy-3;
     border-radius: ($baseline-v/10);
     color: $copy-staff-color;
@@ -132,4 +132,5 @@
 
     }
   }
+
 }

openassessment/xblock/static/sass/oa/utilities/_developer.scss

@@ -26,6 +26,13 @@
 // --------------------
 // Developer styles for Staff Section
 // --------------------
+.staff-info__status {
+  .action--submit {
+    @extend %btn--secondary;
+    @extend %action-2;
+    margin: ($baseline-v/2) ($baseline-v/2) ($baseline-v/2) ($baseline-v/2);
+  }
+}
 .staff-info__student {
   .label {
     color: $heading-staff-color;

openassessment/xblock/static/xml/censorship_rubric_example.xml

@@ -77,11 +77,11 @@
                 <select criterion="form" option="Old-timey letters"/>
             </example>
         </assessment>
+        <assessment name="self-assessment" />
         <assessment name="peer-assessment"
                     start="2013-12-20T19:00-7:00"
                     due="2014-12-21T22:22-7:00"
                     must_grade="1"
                     must_be_graded_by="1" />
-        <assessment name="self-assessment" />
     </assessments>
 </openassessment>

openassessment/xblock/static/xml/example_based_example.xml

+<openassessment>
+    <title>Example Based Example</title>
+    <assessments>
+        <assessment name="example-based-assessment" algorithm_id="fake">
+            <example>
+                <answer>Born in northern New South Wales, Dowling entered the Royal Australian Naval College in 1915. After graduating in 1919 he went to sea aboard various Royal Navy and RAN vessels, and later specialised in gunnery. In 1937, he was given command of the sloop HMAS Swan. Following the outbreak of World War II, he saw action in the Mediterranean theatre as executive officer of the Royal Navy cruiser HMS Naiad, and survived her sinking by a German U-boat in March 1942. Returning to Australia, he served as Director of Plans and later Deputy Chief of Naval Staff before taking command of the light cruiser HMAS Hobart in November 1944. His achievements in the South West Pacific earned him the Distinguished Service Order.
+
+                    Dowling took command of the RAN's first aircraft carrier, HMAS Sydney, in 1948. He became Chief of Naval Personnel in 1950, and Flag Officer Commanding HM Australian Fleet in 1953. Soon after taking up the position of CNS in February 1955, he was promoted to vice admiral and appointed a Companion of the Order of the Bath. As CNS he had to deal with shortages of money, manpower and equipment, and with the increasing role of the United States in Australia's defence planning, at the expense of traditional ties with Britain. Knighted in 1957, Dowling was Chairman of COSC from March 1959 until May 1961, when he retired from the military. In 1963 he was appointed a Knight Commander of the Royal Victorian Order and became Australian Secretary to HM Queen Elizabeth II, serving until his death in 1969.
+                </answer>
+                <select criterion="Ideas" option="Bad" />
+                <select criterion="Content" option="Bad" />
+            </example>
+            <example>
+                <answer>Roy Russell Dowling was born on 28 May 1901 in Condong, a township on the Tweed River in northern New South Wales. His parents were sugar cane inspector Russell Dowling and his wife Lily. The youth entered the Royal Australian Naval College (RANC) at Jervis Bay, Federal Capital Territory, in 1915. An underachiever academically, he excelled at sports, and became chief cadet captain before graduating in 1918 with the King's Medal, awarded for "gentlemanly bearing, character, good influence among his fellows and officer-like qualities".[1][2] The following year he was posted to Britain as a midshipman, undergoing training with the Royal Navy and seeing service on HMS Ramillies and HMS Venturous.[3] By January 1923 he was back in Australia, serving aboard the cruiser HMAS Adelaide. He was promoted to lieutenant in March.[4] In April 1924, Adelaide joined the Royal Navy's Special Service Squadron on its worldwide cruise, taking in New Zealand, Canada, the United States, Panama, and the West Indies, before docking in September at Portsmouth, England. There Dowling left the ship for his next appointment, training as a gunnery officer and serving in that capacity at HMS Excellent.
+                </answer>
+                <select criterion="Ideas" option="Good" />
+                <select criterion="Content" option="Bad" />
+            </example>
+            <example>
+                <answer>After his return to Australia in December 1926, Dowling spent eighteen months on HMAS Platypus and HMAS Anzac, where he continued to specialise in gunnery. In July 1928, he took on an instructional role at the gunnery school in Flinders Naval Depot on Western Port Bay, Victoria. He married Jessie Blanch in Melbourne on 8 May 1930; the couple had two sons and three daughters.[1][6] Jessie accompanied him on his next posting to Britain commencing in January 1931.</answer>
+                <select criterion="Ideas" option="Bad" />
+                <select criterion="Content" option="Good" />
+            </example>
+            <example>
+                <answer>He was promoted to lieutenant commander on 15 March, and was appointed gunnery officer on the light cruiser HMS Colombo in May. Dowling returned to Australia in January 1933, and was appointed squadron gunnery officer aboard the heavy cruiser HMAS Canberra that April.[1][4] The ship operated mainly within Australian waters over the next two years.[7] In July 1935, Dowling took charge of the gunnery school at Flinders Naval Depot. He was promoted to commander on 31 December 1936.[1][4] The following month, he assumed command of the newly commissioned Grimsby-class sloop HMAS Swan, carrying out duties in the South West Pacific.[8] Completing his tenure on Swan in January 1939, he was briefly assigned to the Navy Office, Melbourne, before returning to Britain in March for duty at HMS Pembroke, where he awaited posting aboard the yet-to-be-commissioned anti-aircraft cruiser, HMS Naiad.</answer>
+                <select criterion="Ideas" option="Good" />
+                <select criterion="Content" option="Good" />
+            </example>
+        </assessment>
+    </assessments>
+    <rubric>
+        <prompt>
+            Censorship in the Libraries
+
+            'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author
+
+            Write a persuasive essay to a newspaper reflecting your views on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.
+
+            Read for conciseness, clarity of thought, and form.
+        </prompt>
+        <criterion feedback="optional">
+            <name>Ideas</name>
+            <prompt>Determine if there is a unifying theme or main idea.</prompt>
+            <option points="0">
+                <name>Bad</name>
+                <explanation>Difficult for the reader to discern the main idea.  Too brief or too repetitive to establish or maintain a focus.</explanation>
+            </option>
+            <option points="3">
+                <name>Good</name>
+                <explanation>Presents a unifying theme or main idea, but may include minor tangents.  Stays somewhat focused on topic and task.</explanation>
+            </option>
+        </criterion>
+        <criterion>
+            <name>Content</name>
+            <prompt>Assess the content of the submission</prompt>
+            <option points="0">
+                <name>Bad</name>
+                <explanation>Includes little information with few or no details or unrelated details.  Unsuccessful in attempts to explore any facets of the topic.</explanation>
+            </option>
+            <option points="1">
+                <name>Good</name>
+                <explanation>Includes little information and few or no details.  Explores only one or two facets of the topic.</explanation>
+            </option>
+        </criterion>
+        <feedbackprompt>
+            (Optional) What aspects of this response stood out to you? What did it do well? How could it improve?
+        </feedbackprompt>
+    </rubric>
+</openassessment>
\ No newline at end of file

openassessment/xblock/student_training_mixin.py

@@ -7,6 +7,7 @@ from webob import Response
 from xblock.core import XBlock
 from openassessment.assessment.api import student_training
 import openassessment.workflow.api as workflow_api
+from openassessment.workflow.errors import AssessmentWorkflowError
 from openassessment.xblock.data_conversion import convert_training_examples_list_to_dict
 from .resolve_dates import DISTANT_FUTURE
 
@@ -182,12 +183,10 @@ class StudentTrainingMixin(object):
         else:
             try:
                 self.update_workflow_status()
-            except workflow_api.AssessmentWorkflowError:
-                logger.exception(
-                    u"Workflow error occurred when submitting peer assessment "
-                    u"for submission {uuid}".format(uuid=self.submission_uuid)
-                )
-                return {'success': False, 'msg': _('Could not update workflow status.')}
+            except AssessmentWorkflowError:
+                msg = _('Could not update workflow status.')
+                logger.exception(msg)
+                return {'success': False, 'msg': msg}
             return {
                 'success': True,
                 'msg': u'',

openassessment/xblock/submission_mixin.py

@@ -6,6 +6,7 @@ from xblock.core import XBlock
 
 from submissions import api
 from openassessment.workflow import api as workflow_api
+from openassessment.workflow.errors import AssessmentWorkflowError
 from .resolve_dates import DISTANT_FUTURE
 
 
@@ -73,7 +74,7 @@ class SubmissionMixin(object):
             except api.SubmissionRequestError as err:
                 status_tag = 'EBADFORM'
                 status_text = unicode(err.field_errors)
-            except (api.SubmissionError, workflow_api.AssessmentWorkflowError):
+            except (api.SubmissionError, AssessmentWorkflowError):
                 logger.exception("This response was not submitted.")
                 status_tag = 'EUNKNOWN'
             else:

openassessment/xblock/test/data/assessment_combo.json

@@ -37,9 +37,27 @@
         "is_released": false
     },
     "self_before_peer": {
-        "valid": false,
+        "valid": true,
+        "assessments": [
+            {
+                "name": "self-assessment"
+            },
+            {
+                "name": "peer-assessment",
+                "must_grade": 5,
+                "must_be_graded_by": 3
+            }
+        ],
+        "current_assessments": null,
+        "is_released": false
+    },
+    "student_training_self_then_peer": {
+        "valid": true,
         "assessments": [
             {
+                "name": "student-training"
+            },
+            {
                 "name": "self-assessment"
             },
             {

openassessment/xblock/test/data/example_based_assessment.xml

+<openassessment>
+    <title>Open Assessment Test</title>
+    <prompt>Example-based assessment</prompt>
+    <rubric>
+        <prompt>Read for conciseness, clarity of thought, and form.</prompt>
+        <criterion>
+            <name>Ideas</name>
+            <prompt>How good are the ideas?</prompt>
+            <option points="0">
+                <name>Poor</name>
+                <explanation>Poor job!</explanation>
+            </option>
+            <option points="1">
+                <name>Fair</name>
+                <explanation>Fair job</explanation>
+            </option>
+            <option points="3">
+                <name>Good</name>
+                <explanation>Good job</explanation>
+            </option>
+        </criterion>
+        <criterion>
+            <name>Content</name>
+            <prompt>How good is the content?</prompt>
+            <option points="0">
+                <name>Poor</name>
+                <explanation>Poor job!</explanation>
+            </option>
+            <option points="1">
+                <name>Fair</name>
+                <explanation>Fair job</explanation>
+            </option>
+            <option points="3">
+                <name>Good</name>
+                <explanation>Good job</explanation>
+            </option>
+        </criterion>
+    </rubric>
+    <assessments>
+        <assessment name="example-based-assessment" algorithm_id="fake">
+            <example>
+                <answer>Example Answer One</answer>
+                <select criterion="Ideas" option="Poor" />
+                <select criterion="Content" option="Poor" />
+            </example>
+            <example>
+                <answer>Example Answer Two</answer>
+                <select criterion="Ideas" option="Fair" />
+                <select criterion="Content" option="Fair" />
+            </example>
+            <example>
+                <answer>Example Answer Three</answer>
+                <select criterion="Ideas" option="Fair" />
+                <select criterion="Content" option="Good" />
+            </example>
+            <example>
+                <answer>Example Answer Four</answer>
+                <select criterion="Ideas" option="Poor" />
+                <select criterion="Content" option="Good" />
+            </example>
+        </assessment>
+        <assessment name="peer-assessment" must_grade="5" must_be_graded_by="3" />
+        <assessment name="self-assessment" />
+    </assessments>
+</openassessment>

openassessment/xblock/test/data/example_based_only.xml

+<openassessment>
+    <title>Open Assessment Test</title>
+    <prompt>Example-based assessment</prompt>
+    <rubric>
+        <prompt>Read for conciseness, clarity of thought, and form.</prompt>
+        <criterion>
+            <name>Ideas</name>
+            <prompt>How good are the ideas?</prompt>
+            <option points="0">
+                <name>Poor</name>
+                <explanation>Poor job!</explanation>
+            </option>
+            <option points="1">
+                <name>Fair</name>
+                <explanation>Fair job</explanation>
+            </option>
+            <option points="3">
+                <name>Good</name>
+                <explanation>Good job</explanation>
+            </option>
+        </criterion>
+        <criterion>
+            <name>Content</name>
+            <prompt>How good is the content?</prompt>
+            <option points="0">
+                <name>Poor</name>
+                <explanation>Poor job!</explanation>
+            </option>
+            <option points="1">
+                <name>Fair</name>
+                <explanation>Fair job</explanation>
+            </option>
+            <option points="3">
+                <name>Good</name>
+                <explanation>Good job</explanation>
+            </option>
+        </criterion>
+    </rubric>
+    <assessments>
+        <assessment name="example-based-assessment" algorithm_id="fake">
+            <example>
+                <answer>Example Answer One</answer>
+                <select criterion="Ideas" option="Poor" />
+                <select criterion="Content" option="Poor" />
+            </example>
+            <example>
+                <answer>Example Answer Two</answer>
+                <select criterion="Ideas" option="Fair" />
+                <select criterion="Content" option="Fair" />
+            </example>
+            <example>
+                <answer>Example Answer Three</answer>
+                <select criterion="Ideas" option="Fair" />
+                <select criterion="Content" option="Good" />
+            </example>
+            <example>
+                <answer>Example Answer Four</answer>
+                <select criterion="Ideas" option="Poor" />
+                <select criterion="Content" option="Good" />
+            </example>
+        </assessment>
+    </assessments>
+</openassessment>

openassessment/xblock/test/data/grade_scenario.xml

@@ -40,6 +40,28 @@
         </criterion>
     </rubric>
     <assessments>
+        <assessment name="example-based-assessment" algorithm_id="fake">
+            <example>
+                <answer>Example Answer One</answer>
+                <select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="Ġööḋ" />
+                <select criterion="Form" option="Poor" />
+            </example>
+            <example>
+                <answer>Example Answer Two</answer>
+                <select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="ﻉซƈﻉɭɭﻉกՇ" />
+                <select criterion="Form" option="Fair" />
+            </example>
+            <example>
+                <answer>Example Answer Three</answer>
+                <select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="Ġööḋ" />
+                <select criterion="Form" option="Good" />
+            </example>
+            <example>
+                <answer>Example Answer Four</answer>
+                <select criterion="𝓒𝓸𝓷𝓬𝓲𝓼𝓮" option="ﻉซƈﻉɭɭﻉกՇ" />
+                <select criterion="Form" option="Good" />
+            </example>
+        </assessment>
         <assessment name="peer-assessment" must_grade="2" must_be_graded_by="2" />
         <assessment name="self-assessment" />
     </assessments>