Merge pull request #31 from edx/sanchez/xblock_peer_eval

Sanchez/xblock peer eval

apps/openassessment/peer/api.py

@@ -8,11 +8,13 @@ import copy
 import logging
 
 from django.db import DatabaseError
+import math
 from openassessment.peer.models import PeerEvaluation
 
 from openassessment.peer.serializers import PeerEvaluationSerializer
-from submissions.models import Submission, StudentItem
-from submissions.serializers import SubmissionSerializer
+from submissions import api as submission_api
+from submissions.models import Submission, StudentItem, Score
+from submissions.serializers import SubmissionSerializer, StudentItemSerializer
 
 logger = logging.getLogger(__name__)
 
@@ -62,8 +64,13 @@ class PeerEvaluationInternalError(PeerEvaluationError):
     pass
 
 
-def create_evaluation(submission_uuid, scorer_id, assessment_dict,
-                      scored_at=None):
+def create_evaluation(
+        submission_uuid,
+        scorer_id,
+        required_evaluations_for_student,
+        required_evaluations_for_submission,
+        assessment_dict,
+        scored_at=None):
     """Creates an evaluation on the given submission.
 
     Evaluations are created based on feedback associated with a particular
@@ -75,6 +82,10 @@ def create_evaluation(submission_uuid, scorer_id, assessment_dict,
             Submission model.
         scorer_id (str): The user ID for the user giving this assessment. This
             is required to create an assessment on a submission.
+        required_evaluations_for_student (int): The number of evaluations
+            required for the student to receive a score for their submission.
+        required_evaluations_for_submission (int): The number of evaluations
+            required on the submission for it to be scored.
         assessment_dict (dict): All related information for the assessment. An
             assessment contains points_earned, points_possible, and feedback.
         scored_at (datetime): Optional argument to override the time in which
@@ -126,6 +137,35 @@ def create_evaluation(submission_uuid, scorer_id, assessment_dict,
         if not peer_serializer.is_valid():
             raise PeerEvaluationRequestError(peer_serializer.errors)
         peer_serializer.save()
+
+        # Check if the submission is finished and its Author has graded enough.
+        student_item = submission.student_item
+        _check_if_finished_and_create_score(
+            student_item,
+            submission,
+            required_evaluations_for_student,
+            required_evaluations_for_submission
+        )
+
+        # Check if the grader is finished and has enough evaluations
+        scorer_item = StudentItem.objects.get(
+            student_id=scorer_id,
+            item_id=student_item.item_id,
+            course_id=student_item.course_id,
+            item_type=student_item.item_type
+        )
+
+        scorer_submissions = Submission.objects.filter(
+            student_item=scorer_item
+        ).order_by("-attempt_number")
+
+        _check_if_finished_and_create_score(
+            scorer_item,
+            scorer_submissions[0],
+            required_evaluations_for_student,
+            required_evaluations_for_submission
+        )
+
         return peer_serializer.data
     except DatabaseError:
         error_message = u"An error occurred while creating evaluation {} for submission: {} by: {}".format(
@@ -137,6 +177,57 @@ def create_evaluation(submission_uuid, scorer_id, assessment_dict,
         raise PeerEvaluationInternalError(error_message)
 
 
+def _check_if_finished_and_create_score(student_item,
+                                        submission,
+                                        required_evaluations_for_student,
+                                        required_evaluations_for_submission):
+    """Basic function for checking if a student is finished with peer workflow.
+
+    Checks if the student is finished with the peer evaluation workflow. If the
+    student already has a final grade calculated, there is no need to proceed.
+    If they do not have a grade, the student has a final grade calculated.
+
+    """
+    if Score.objects.filter(student_item=student_item):
+        return
+
+    finished_evaluating = has_finished_required_evaluating(
+        student_item.student_id,
+        required_evaluations_for_student
+    )
+    evaluations = PeerEvaluation.objects.filter(submission=submission)
+    submission_finished = evaluations.count() >= required_evaluations_for_submission
+    scores = []
+    for evaluation in evaluations:
+        scores.append(evaluation.points_earned)
+    if finished_evaluating and submission_finished:
+        submission_api.set_score(
+            StudentItemSerializer(student_item).data,
+            SubmissionSerializer(submission).data,
+            _calculate_final_score(scores),
+            evaluations[0].points_possible
+        )
+
+
+def _calculate_final_score(scores):
+    """Final grade is calculated using integer values, rounding up.
+
+    If there is a true median score, it is returned. If there are two median
+    values, the average of those two values is returned, rounded up to the
+    greatest integer value.
+
+    """
+    total_scores = len(scores)
+    scores = sorted(scores)
+    median = int(math.ceil(total_scores / float(2)))
+    if total_scores == 0:
+        return 0
+    elif total_scores % 2:
+        return scores[median-1]
+    else:
+        return int(math.ceil(sum(scores[median-1:median+1])/float(2)))
+
+
 def has_finished_required_evaluating(student_id, required_evaluations):
     """Check if a student still needs to evaluate more submissions
 
@@ -162,7 +253,7 @@ def has_finished_required_evaluating(student_id, required_evaluations):
             while evaluating this workflow rule.
 
     Examples:
-        >>> has_finished_required_evaluating("Tim")
+        >>> has_finished_required_evaluating("Tim", 3)
         True
 
     """
@@ -228,7 +319,7 @@ def get_evaluations(submission_id):
         raise PeerEvaluationInternalError(error_message)
 
 
-def get_submission_to_evaluate(student_item_dict):
+def get_submission_to_evaluate(student_item_dict, required_num_evaluations):
     """Get a submission to peer evaluate.
 
     Retrieves a submission for evaluation for the given student_item. This will
@@ -243,6 +334,8 @@ def get_submission_to_evaluate(student_item_dict):
             item_id, course_id, and item_type, used to identify the unique
             question for the review, while the student_id is used to explicitly
             avoid giving the student their own submission.
+        required_num_evaluations (int): The number of evaluations a submission
+            requires before it has completed the peer evaluation process.
 
     Returns:
         dict: A peer submission for evaluation. This contains a 'student_item',
@@ -262,7 +355,7 @@ def get_submission_to_evaluate(student_item_dict):
         >>>    item_type="type_one",
         >>>    student_id="Bob",
         >>> )
-        >>> get_submission_to_evaluate(student_item_dict)
+        >>> get_submission_to_evaluate(student_item_dict, 3)
         {
             'student_item': 2,
             'attempt_number': 1,
@@ -279,12 +372,29 @@ def get_submission_to_evaluate(student_item_dict):
         item_id=student_item_dict["item_id"],
     ).exclude(student_id=student_item_dict["student_id"])
 
-    # TODO: We need a priority queue.
-    submission = Submission.objects.filter(student_item__in=student_items).order_by(
-        "submitted_at",
-        "-attempt_number")[:1]
+    submission = _get_first_submission_not_evaluated(
+        student_items,
+        student_item_dict["student_id"],
+        required_num_evaluations
+    )
     if not submission:
         raise PeerEvaluationWorkflowError(
             "There are no submissions available for evaluation."
         )
-    return SubmissionSerializer(submission[0]).data
+    return SubmissionSerializer(submission).data
+
+
+def _get_first_submission_not_evaluated(student_items, student_id, required_num_evaluations):
+    # TODO: We need a priority queue.
+    submissions = Submission.objects.filter(student_item__in=student_items).order_by(
+        "submitted_at",
+        "-attempt_number"
+    )
+    for submission in submissions:
+        evaluations = PeerEvaluation.objects.filter(submission=submission)
+        if evaluations.count() < required_num_evaluations:
+            already_evaluated = False
+            for evaluation in evaluations:
+                already_evaluated = already_evaluated or evaluation.scorer_id == student_id
+            if not already_evaluated:
+                return submission
\ No newline at end of file

apps/openassessment/peer/templates/evaluations.html

@@ -6,6 +6,7 @@
     <th>Submission UUID</th>
     <th>Points Earned</th>
     <th>Points Possible</th>
+    <th>Scored By</th>
     <th>Scored At</th>
     <th>Score Type</th>
     <th>Feedback</th>
@@ -15,6 +16,7 @@
             <td>{{ evaluation.points_earned }}</td>
             <td>{{ evaluation.points_possible }}</td>
             <td>{{ evaluation.scorer_id }}</td>
+            <td>{{ evaluation.scored_at }}</td>
             <td>{{ evaluation.score_type }}</td>
             <td>{{ evaluation.feedback }}</td>
         </tr>

apps/openassessment/peer/test/test_api.py

@@ -9,7 +9,7 @@ from mock import patch
 
 from openassessment.peer import api
 from openassessment.peer.models import PeerEvaluation
-from submissions.api import create_submission
+from submissions import api as sub_api
 from submissions.models import Submission
 from submissions.tests.test_api import STUDENT_ITEM, ANSWER_ONE
 
@@ -19,6 +19,9 @@ ASSESSMENT_DICT = dict(
     feedback="Your submission was thrilling.",
 )
 
+REQUIRED_GRADED = 5
+REQUIRED_GRADED_BY = 3
+
 MONDAY = datetime.datetime(2007, 9, 12, 0, 0, 0, 0, pytz.UTC)
 TUESDAY = datetime.datetime(2007, 9, 13, 0, 0, 0, 0, pytz.UTC)
 WEDNESDAY = datetime.datetime(2007, 9, 15, 0, 0, 0, 0, pytz.UTC)
@@ -28,20 +31,24 @@ THURSDAY = datetime.datetime(2007, 9, 16, 0, 0, 0, 0, pytz.UTC)
 @ddt
 class TestApi(TestCase):
     def test_create_evaluation(self):
-        submission = create_submission(STUDENT_ITEM, ANSWER_ONE)
+        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER_ONE)
         evaluation = api.create_evaluation(
             submission["uuid"],
             STUDENT_ITEM["student_id"],
+            REQUIRED_GRADED,
+            REQUIRED_GRADED_BY,
             ASSESSMENT_DICT
         )
         self._assert_evaluation(evaluation, **ASSESSMENT_DICT)
 
     @file_data('test_valid_evaluations.json')
     def test_get_evaluations(self, assessment_dict):
-        submission = create_submission(STUDENT_ITEM, ANSWER_ONE)
+        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER_ONE)
         api.create_evaluation(
             submission["uuid"],
             STUDENT_ITEM["student_id"],
+            REQUIRED_GRADED,
+            REQUIRED_GRADED_BY,
             assessment_dict
         )
         evaluations = api.get_evaluations(submission["uuid"])
@@ -50,10 +57,12 @@ class TestApi(TestCase):
 
     @file_data('test_valid_evaluations.json')
     def test_get_evaluations_with_date(self, assessment_dict):
-        submission = create_submission(STUDENT_ITEM, ANSWER_ONE)
+        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER_ONE)
         api.create_evaluation(
             submission["uuid"],
             STUDENT_ITEM["student_id"],
+            REQUIRED_GRADED,
+            REQUIRED_GRADED_BY,
             assessment_dict,
             MONDAY
         )
@@ -62,17 +71,61 @@ class TestApi(TestCase):
         self._assert_evaluation(evaluations[0], **assessment_dict)
         self.assertEqual(evaluations[0]["scored_at"], MONDAY)
 
-    def test_student_finished_evaluating(self):
+    def test_peer_evaluation_workflow(self):
+        tim = self._create_student_and_submission("Tim", "Tim's answer")
         bob = self._create_student_and_submission("Bob", "Bob's answer")
         sally = self._create_student_and_submission("Sally", "Sally's answer")
         jim = self._create_student_and_submission("Jim", "Jim's answer")
+        buffy = self._create_student_and_submission("Buffy", "Buffy's answer")
+        xander = self._create_student_and_submission("Xander", "Xander's answer")
+
+        # Tim should not have a score, because he has not evaluated enough
+        # peer submissions.
+        scores = sub_api.get_score(STUDENT_ITEM)
+        self.assertFalse(scores)
+
+        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
+        api.create_evaluation(
+            bob["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        api.create_evaluation(
+            sally["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
+        api.create_evaluation(
+            jim["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
+        api.create_evaluation(
+            buffy["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        self.assertFalse(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
+        api.create_evaluation(
+            xander["uuid"], "Tim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        self.assertTrue(api.has_finished_required_evaluating("Tim", REQUIRED_GRADED))
+
+        # Tim should not have a score, because his submission does not have
+        # enough evaluations.
+        scores = sub_api.get_score(STUDENT_ITEM)
+        self.assertFalse(scores)
+
+        api.create_evaluation(
+            tim["uuid"], "Bob", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        api.create_evaluation(
+            tim["uuid"], "Sally", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+        api.create_evaluation(
+            tim["uuid"], "Jim", REQUIRED_GRADED, REQUIRED_GRADED_BY, ASSESSMENT_DICT
+        )
+
+        # Tim has met the critera, and should now have a score.
+        scores = sub_api.get_score(STUDENT_ITEM)
+        self.assertTrue(scores)
+        self.assertEqual(6, scores[0]["points_earned"])
+        self.assertEqual(12, scores[0]["points_possible"])
 
-        self.assertFalse(api.has_finished_required_evaluating("Tim", 3))
-        api.create_evaluation(bob["uuid"], "Tim", ASSESSMENT_DICT)
-        api.create_evaluation(sally["uuid"], "Tim", ASSESSMENT_DICT)
-        self.assertFalse(api.has_finished_required_evaluating("Tim", 3))
-        api.create_evaluation(jim["uuid"], "Tim", ASSESSMENT_DICT)
-        self.assertTrue(api.has_finished_required_evaluating("Tim", 3))
 
     @raises(api.PeerEvaluationRequestError)
     def test_bad_configuration(self):
@@ -86,7 +139,7 @@ class TestApi(TestCase):
         )
         self._create_student_and_submission("Jim", "Jim's answer", THURSDAY)
 
-        submission = api.get_submission_to_evaluate(STUDENT_ITEM)
+        submission = api.get_submission_to_evaluate(STUDENT_ITEM, 3)
         self.assertIsNotNone(submission)
         self.assertEqual(submission["answer"], u"Bob's answer")
         self.assertEqual(submission["student_item"], 2)
@@ -95,7 +148,7 @@ class TestApi(TestCase):
     @raises(api.PeerEvaluationWorkflowError)
     def test_no_submissions_to_evaluate_for_tim(self):
         self._create_student_and_submission("Tim", "Tim's answer", MONDAY)
-        api.get_submission_to_evaluate(STUDENT_ITEM)
+        api.get_submission_to_evaluate(STUDENT_ITEM, 3)
 
     """
     Some Error Checking Tests against DB failures.
@@ -105,32 +158,47 @@ class TestApi(TestCase):
     @raises(api.PeerEvaluationInternalError)
     def test_error_on_evaluation_creation(self, mock_filter):
         mock_filter.side_effect = DatabaseError("Bad things happened")
-        submission = create_submission(STUDENT_ITEM, ANSWER_ONE)
+        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER_ONE)
         api.create_evaluation(
             submission["uuid"],
             STUDENT_ITEM["student_id"],
+            REQUIRED_GRADED,
+            REQUIRED_GRADED_BY,
             ASSESSMENT_DICT,
             MONDAY
         )
 
     @patch.object(PeerEvaluation.objects, 'filter')
-    @raises(api.PeerEvaluationInternalError)
+    @raises(sub_api.SubmissionInternalError)
     def test_error_on_get_evaluation(self, mock_filter):
-        submission = create_submission(STUDENT_ITEM, ANSWER_ONE)
+        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER_ONE)
         api.create_evaluation(
             submission["uuid"],
             STUDENT_ITEM["student_id"],
+            REQUIRED_GRADED,
+            REQUIRED_GRADED_BY,
             ASSESSMENT_DICT,
             MONDAY
         )
         mock_filter.side_effect = DatabaseError("Bad things happened")
         api.get_evaluations(submission["uuid"])
 
+    def test_choose_score(self):
+        self.assertEqual(0, api._calculate_final_score([]))
+        self.assertEqual(5, api._calculate_final_score([5]))
+        # average of 5, 6, rounded down.
+        self.assertEqual(6, api._calculate_final_score([5, 6]))
+        self.assertEqual(14, api._calculate_final_score([5, 6, 12, 16, 22, 53]))
+        self.assertEqual(14, api._calculate_final_score([6, 5, 12, 53, 16, 22]))
+        self.assertEqual(16, api._calculate_final_score([5, 6, 12, 16, 22, 53, 102]))
+        self.assertEqual(16, api._calculate_final_score([16, 6, 12, 102, 22, 53, 5]))
+
+
     @staticmethod
     def _create_student_and_submission(student, answer, date=None):
         new_student_item = STUDENT_ITEM.copy()
         new_student_item["student_id"] = student
-        return create_submission(new_student_item, answer, date)
+        return sub_api.create_submission(new_student_item, answer, date)
 
     def _assert_evaluation(self, evaluation, points_earned, points_possible,
                            feedback):

apps/openassessment/xblock/openassessmentblock.py

@@ -3,8 +3,10 @@
 import pkg_resources
 
 from mako.template import Template
+from openassessment.peer.api import PeerEvaluationWorkflowError
 
 from submissions import api
+from openassessment.peer import api as peer_api
 
 from xblock.core import XBlock
 from xblock.fields import List, Scope, String
@@ -13,6 +15,99 @@ from xblock.fragment import Fragment
 
 mako_default_filters = ['unicode', 'h', 'trim']
 
+EXAMPLE_POVERTY_RUBRIC = (
+    "OpenAssessmentBlock Poverty Rubric",
+    """
+        <vertical_demo>
+
+            <openassessment start="2014-12-19T23:00-7:00" due="2014-12-21T23:00-7:00">
+                <prompt>
+                    Given the state of the world today, what do you think should be done to
+                    combat poverty? Please answer in a short essay of 200-300 words.
+                </prompt>
+                <rubric>
+                    Read for conciseness, clarity of thought, and form.
+                    <criterion name="concise">
+                        How concise is it?
+                        <option val="0">Neal Stephenson (late)</option>
+                        <option val="1">HP Lovecraft</option>
+                        <option val="3">Robert Heinlein</option>
+                        <option val="4">Neal Stephenson (early)</option>
+                        <option val="5">Earnest Hemingway</option>
+                    </criterion>
+                    <criterion name="clearheaded">
+                        How clear is the thinking?
+                        <option val="0">Yogi Berra</option>
+                        <option val="1">Hunter S. Thompson</option>
+                        <option val="2">Robert Heinlein</option>
+                        <option val="3">Isaac Asimov</option>
+                        <option val="10">Spock</option>
+                    </criterion>
+                    <criterion name="form">
+                        Lastly, how is it's form? Punctuation, grammar, and spelling all count.
+                        <option val="0">lolcats</option>
+                        <option val="1">Facebook</option>
+                        <option val="2">Reddit</option>
+                        <option val="3">metafilter</option>
+                        <option val="4">Usenet, 1996</option>
+                        <option val="5">The Elements of Style</option>
+                    </criterion>
+                </rubric>
+                <evals>
+                    <peereval start="2014-12-20T19:00-7:00"
+                      due="2014-12-21T22:22-7:00"
+                      must_grade="5"
+                      must_be_graded_by="3" />
+                    <selfeval/>
+                </evals>
+            </openassessment>
+
+        </vertical_demo>
+    """
+)
+
+EXAMPLE_CENSORSHIP_RUBRIC = (
+    "OpenAssessmentBlock Censorship Rubric",
+    """
+    <vertical_demo>
+
+        <openassessment start="2013-12-19T23:00-7:00" due="2014-12-21T23:00-7:00">
+            <prompt>
+                What do you think about censorship in libraries? I think it's pretty great.
+            </prompt>
+            <rubric>
+                Read for conciseness, clarity of thought, and form.
+                <criterion name="concise">
+                    How concise is it?
+                    <option val="0">The Bible</option>
+                    <option val="1">Earnest Hemingway</option>
+                    <option val="3">Matsuo Basho</option>
+                </criterion>
+                <criterion name="clearheaded">
+                    How clear is the thinking?
+                    <option val="0">Eric</option>
+                    <option val="1">John</option>
+                    <option val="2">Ian</option>
+                </criterion>
+                <criterion name="form">
+                    Lastly, how is it's form? Punctuation, grammar, and spelling all count.
+                    <option val="0">IRC</option>
+                    <option val="1">Real Email</option>
+                    <option val="2">Old-timey letters</option>
+                </criterion>
+            </rubric>
+            <evals>
+                <selfeval/>
+                <peereval start="2014-12-20T19:00-7:00"
+                  due="2014-12-21T22:22-7:00"
+                  must_grade="5"
+                  must_be_graded_by="3" />
+            </evals>
+        </openassessment>
+
+    </vertical_demo>
+    """
+)
 
 class OpenAssessmentBlock(XBlock):
     """Displays a question and gives an area where students can compose a response."""
@@ -47,7 +142,7 @@ class OpenAssessmentBlock(XBlock):
 
     def _get_student_item_dict(self):
         """Create a student_item_dict from our surrounding context.
-        
+
         See also: submissions.api for details.
         """
         item_id, student_id = self._get_xblock_trace()
@@ -69,18 +164,28 @@ class OpenAssessmentBlock(XBlock):
         trace = self._get_xblock_trace()
         student_item_dict = self._get_student_item_dict()
         previous_submissions = api.get_submissions(student_item_dict)
-        if previous_submissions:  # XXX: until workflow better, move on w/ prev submit
+        try:
+            # HACK: Replace with proper workflow.
+            peer_eval = self._hack_get_peer_eval()
+            peer_submission = peer_api.get_submission_to_evaluate(student_item_dict, peer_eval["must_be_graded_by"])
+        except PeerEvaluationWorkflowError:
+            peer_submission = False
+
+        if previous_submissions and peer_submission:  # XXX: until workflow better, move on w/ prev submit
             html = Template(load("static/html/oa_rubric.html"),
                             default_filters=mako_default_filters,
                             input_encoding='utf-8',
                            )
-            frag = Fragment(html.render_unicode(xblock_trace=trace, 
+            frag = Fragment(html.render_unicode(xblock_trace=trace,
+                                                peer_submission=peer_submission,
                                                 rubric_instructions=self.rubric_instructions,
                                                 rubric_criteria=self.rubric_criteria,
                                                ))
             frag.add_css(load("static/css/openassessment.css"))
             frag.add_javascript(load("static/js/src/oa_assessment.js"))
             frag.initialize_js('OpenAssessmentBlock')
+        elif previous_submissions:
+            return Fragment(u"<div>There are no submissions to review.</div>")
         else:                     # XXX: until workflow better, submit until submitted
             html = Template(load("static/html/oa_submission.html"),
                             default_filters=mako_default_filters,
@@ -92,10 +197,42 @@ class OpenAssessmentBlock(XBlock):
             frag.initialize_js('OpenAssessmentBlock')
         return frag
 
+    def _hack_get_peer_eval(self):
+        # HACK: Forcing Peer Eval, we'll get the Eval config.
+        for next_eval in self.rubric_evals:
+            if next_eval["type"] == "peereval":
+                return next_eval
+
     @XBlock.json_handler
     def assess(self, data, suffix=''):
+        # HACK: Replace with proper workflow.
+        peer_eval = self._hack_get_peer_eval()
         """Place an assessment into Openassessment system"""
-        return (False, "Assessment handler is not implemented yet.")
+        # TODO: We're not doing points possible in a good way, need to refactor
+        # the rubric criteria type, Joe has thoughts on this.
+        student_item_dict = self._get_student_item_dict()
+
+        points_possible = sum(
+             max(int(val) for val in criteria if val.isdigit())
+             for criteria in self.rubric_criteria
+        )
+        assessment_dict = {
+            "points_earned": map(int, data["points_earned"]),
+            "points_possible": points_possible,
+            "feedback": "Not yet implemented.",
+        }
+        evaluation = peer_api.create_evaluation(
+            data["submission_uuid"],
+            student_item_dict["student_id"],
+            int(peer_eval["must_grade"]),
+            int(peer_eval["must_be_graded_by"]),
+            assessment_dict
+        )
+
+        # Temp kludge until we fix JSON serialization for datetime
+        evaluation["scored_at"] = str(evaluation["scored_at"])
+
+        return evaluation, "Success"
 
     @XBlock.json_handler
     def submit(self, data, suffix=''):
@@ -156,98 +293,9 @@ class OpenAssessmentBlock(XBlock):
                 block.runtime.add_node_as_child(block, child, id_generator)
         return block
 
-    # Arbitrary attributes can be defined on the 
+    # Arbitrary attributes can be defined on the
     @staticmethod
     def workbench_scenarios():
         """A canned scenario for display in the workbench."""
-        return [
-            ("OpenAssessmentBlock Poverty Rubric", 
-             """
-<vertical_demo>
-
-<openassessment start="2014-12-19T23:00-7:00" due="2014-12-21T23:00-7:00">
-  <prompt>
-    Given the state of the world today, what do you think should be done to
-    combat poverty? Please answer in a short essay of 200-300 words.
-  </prompt>
-  <rubric>
-    Read for conciseness, clarity of thought, and form.
-    <criterion name="concise">
-      How concise is it?
-      <option val="0">Neal Stephenson (late)</option>
-      <option val="1">HP Lovecraft</option>
-      <option val="3">Robert Heinlein</option>
-      <option val="4">Neal Stephenson (early)</option>
-      <option val="5">Earnest Hemingway</option>
-    </criterion>
-    <criterion name="clearheaded">
-      How clear is the thinking?
-      <option val="0">The Unabomber</option>
-      <option val="1">Hunter S. Thompson</option>
-      <option val="2">Robert Heinlein</option>
-      <option val="3">Isaac Asimov</option>
-      <option val="55">Spock</option>
-    </criterion>
-    <criterion name="form">
-      Lastly, how is it's form? Punctuation, grammar, and spelling all count.
-      <option val="0">lolcats</option>
-      <option val="1">Facebook</option>
-      <option val="2">Reddit</option>
-      <option val="3">metafilter</option>
-      <option val="4">Usenet, 1996</option>
-      <option val="99">The Elements of Style</option>
-    </criterion>
-  </rubric>
-  <evals>
-    <peereval start="2014-12-20T19:00-7:00"
-              due="2014-12-21T22:22-7:00"
-              must_grade="5"
-              must_be_graded_by="3" />
-    <selfeval/>
-  </evals>
-</openassessment>
-
-</vertical_demo>
-             """),
-            ("OpenAssessmentBlock Censorship Rubric", 
-             """
-<vertical_demo>
-
-<openassessment start="2013-12-19T23:00-7:00" due="2014-12-21T23:00-7:00">
-  <prompt>
-    What do you think about censorship in libraries? I think it's pretty great.
-  </prompt>
-  <rubric>
-    Read for conciseness, clarity of thought, and form.
-    <criterion name="concise">
-      How concise is it?
-      <option val="0">The Bible</option>
-      <option val="1">Earnest Hemingway</option>
-      <option val="3">Matsuo Basho</option>
-    </criterion>
-    <criterion name="clearheaded">
-      How clear is the thinking?
-      <option val="0">Eric</option>
-      <option val="1">John</option>
-      <option val="2">Ian</option>
-    </criterion>
-    <criterion name="form">
-      Lastly, how is it's form? Punctuation, grammar, and spelling all count.
-      <option val="0">IRC</option>
-      <option val="1">Real Email</option>
-      <option val="2">Old-timey letters</option>
-    </criterion>
-  </rubric>
-  <evals>
-    <selfeval/>
-    <peereval start="2014-12-20T19:00-7:00"
-              due="2014-12-21T22:22-7:00"
-              must_grade="5"
-              must_be_graded_by="3" />
-  </evals>
-</openassessment>
-
-</vertical_demo>
-             """),
-        ]
+        return [EXAMPLE_POVERTY_RUBRIC, EXAMPLE_CENSORSHIP_RUBRIC,]
 

apps/openassessment/xblock/static/html/oa_rubric.html

 <!-- START OpenAssessmentBlock HTML -->
 <div class="openassessment_block" id="openassessment_block_${xblock_trace[0]}">
+    <div id="peer_submission_uuid" hidden="true">${peer_submission["uuid"]}</div>
+    <p>${peer_submission["answer"]}</p>
     <p class="openassessment_prompt" 
        id="openassessment_rubric_instructions_${xblock_trace[0]}">${rubric_instructions}</p>
     % for criterion in rubric_criteria:
      <div>
       <p class="openassessment_prompt">${criterion["instructions"]}</p>
       % for value in sorted([k for k in criterion.keys() if k != 'name' and k != 'instructions']):
-       <input type="radio" value="${value}">${criterion[value]}</input>
+       <input name="${criterion['name']}" type="radio" value="${value}">${criterion[value]}</input>
       % endfor
      </div>
-    % endfor 
-    <input type="button" 
-           class="openassessment_submit" id="openassessment_submit_${xblock_trace[0]}" value="Submit" />
+    % endfor
+        <input type="button"
+               class="openassessment_submit" id="openassessment_submit_${xblock_trace[0]}" value="Submit" />
 </div>
-<div class="openassessment_response_status_block" id=openassessment_response_status_block_${xblock_trace[0]}">
+<div class="openassessment_response_status_block" id="openassessment_response_status_block_${xblock_trace[0]}">
     This message should be invisible; please upgrade your browser.
 </div> 
 <!-- END OpenAssessmentBlock HTML -->

apps/openassessment/xblock/static/js/src/oa_assessment.js

@@ -7,6 +7,15 @@ function OpenAssessmentBlock(runtime, element) {
     var click_msg = '<p class="clickhere">(click here to dismiss this message)</p>';
     /* Sample Debug Console: http://localhost:8000/submissions/Joe_Bloggs/TestCourse/u_3 */
 
+    function prepare_assessment_post(element) {
+        selector = $("input[type=radio]:checked", element);
+        values = [];
+        for (i=0; i<selector.length; i++) {
+            values[i] = selector[i].value;
+        }
+        return {"submission_uuid":$("div#peer_submission_uuid")[0].innerText, "points_earned":values};
+    }
+
     function displayStatus(result) {
         status = result[0]
         error_msg = result[1]
@@ -26,7 +35,7 @@ function OpenAssessmentBlock(runtime, element) {
             type: "POST",
             url: handlerUrl,
             /* data: JSON.stringify({"submission": $('.openassessment_submission', element).val()}), */
-            data: JSON.stringify({"assessment": "I'm not sure how to stringify a form"}),
+            data: JSON.stringify(prepare_assessment_post(element)),
             success: displayStatus
         });
     });

apps/openassessment/xblock/test/test_openassessment.py

@@ -12,6 +12,50 @@ from workbench.runtime import WorkbenchRuntime
 from submissions import api
 from submissions.api import SubmissionRequestError, SubmissionInternalError
 
+RUBRIC_CONFIG = """
+    <openassessment start="2014-12-19T23:00-7:00" due="2014-12-21T23:00-7:00">
+        <prompt>
+            Given the state of the world today, what do you think should be done to
+            combat poverty? Please answer in a short essay of 200-300 words.
+        </prompt>
+        <rubric>
+            Read for conciseness, clarity of thought, and form.
+            <criterion name="concise">
+                How concise is it?
+                <option val="0">Neal Stephenson (late)</option>
+                <option val="1">HP Lovecraft</option>
+                <option val="3">Robert Heinlein</option>
+                <option val="4">Neal Stephenson (early)</option>
+                <option val="5">Earnest Hemingway</option>
+            </criterion>
+            <criterion name="clearheaded">
+                How clear is the thinking?
+                <option val="0">Yogi Berra</option>
+                <option val="1">Hunter S. Thompson</option>
+                <option val="2">Robert Heinlein</option>
+                <option val="3">Isaac Asimov</option>
+                <option val="10">Spock</option>
+            </criterion>
+            <criterion name="form">
+                Lastly, how is it's form? Punctuation, grammar, and spelling all count.
+                <option val="0">lolcats</option>
+                <option val="1">Facebook</option>
+                <option val="2">Reddit</option>
+                <option val="3">metafilter</option>
+                <option val="4">Usenet, 1996</option>
+                <option val="5">The Elements of Style</option>
+            </criterion>
+        </rubric>
+        <evals>
+            <peereval start="2014-12-20T19:00-7:00"
+              due="2014-12-21T22:22-7:00"
+              must_grade="5"
+              must_be_graded_by="3" />
+            <selfeval/>
+        </evals>
+    </openassessment>
+"""
+
 
 class TestOpenAssessment(TestCase):
 
@@ -22,11 +66,7 @@ class TestOpenAssessment(TestCase):
         self.runtime = WorkbenchRuntime()
         self.runtime.user_id = "Bob"
         assessment_id = self.runtime.parse_xml_string(
-            """<openassessment
-                  prompt="This is my prompt. There are many like it, but this one is mine."
-                  course_id="RopesCourse"
-                />
-            """, self.runtime.id_generator)
+            RUBRIC_CONFIG, self.runtime.id_generator)
         self.assessment = self.runtime.get_block(assessment_id)
         self.default_json_submission = json.dumps({"submission": "This is my answer to this test question!"})
 

apps/submissions/api.py

@@ -8,8 +8,8 @@ import logging
 from django.db import DatabaseError
 from django.utils.encoding import force_unicode
 
-from submissions.serializers import SubmissionSerializer, StudentItemSerializer
-from submissions.models import Submission, StudentItem
+from submissions.serializers import SubmissionSerializer, StudentItemSerializer, ScoreSerializer
+from submissions.models import Submission, StudentItem, Score
 
 logger = logging.getLogger(__name__)
 
@@ -212,20 +212,129 @@ def get_submissions(student_item_dict, limit=None):
     if limit:
         submission_models = submission_models[:limit]
 
-    return [SubmissionSerializer(submission).data for submission in
-            submission_models]
+    return SubmissionSerializer(submission_models, many=True).data
 
 
 def get_score(student_item):
-    pass
+    """Get the score for a particular student item
+
+    Each student item should have a unique score. This function will return the
+    score if it is available. A score is only calculated for a student item if
+    it has completed the workflow for a particular assessment module.
+
+    Args:
+        student_item (dict): The dictionary representation of a student item.
+            Function returns the score related to this student item.
+
+    Returns:
+        score (dict): The score associated with this student item. None if there
+            is no score found.
+
+    Raises:
+        SubmissionInternalError: Raised if a score cannot be retrieved because
+            of an internal server error.
+
+    Examples:
+        >>> student_item = {
+        >>>     "student_id":"Tim",
+        >>>     "course_id":"TestCourse",
+        >>>     "item_id":"u_67",
+        >>>     "item_type":"openassessment"
+        >>> }
+        >>>
+        >>> get_score(student_item)
+        [{
+            'student_item': 2,
+            'submission': 2,
+            'points_earned': 8,
+            'points_possible': 20,
+            'created_at': datetime.datetime(2014, 2, 7, 18, 30, 1, 807911, tzinfo=<UTC>)
+        }]
+
+    """
+    student_item_model = StudentItem.objects.get(**student_item)
+    scores = Score.objects.filter(student_item=student_item_model)
+    return ScoreSerializer(scores, many=True).data
 
 
 def get_scores(course_id, student_id, types=None):
     pass
 
 
-def set_score(student_item):
-    pass
+def set_score(student_item, submission, score, points_possible):
+    """Set a score for a particular student item, submission pair.
+
+    Sets the score for a particular student item and submission pair. This score
+    is calculated externally to the API.
+
+    Args:
+        student_item (dict): The student item associated with this score. This
+            dictionary must contain a course_id, student_id, and item_id.
+        submission (dict): The submission associated with this score. This
+            dictionary must contain all submission fields to properly get a
+            unique submission item.
+        score (int): The score to associate with the given submission and
+            student item.
+        points_possible (int): The total points possible for this particular
+            student item.
+
+    Returns:
+        (dict): The dictionary representation of the saved score.
+
+    Raises:
+        SubmissionInternalError: Thrown if there was an internal error while
+            attempting to save the score.
+        SubmissionRequestError: Thrown if the given student item or submission
+            are not found.
+
+    Examples:
+        >>> student_item_dict = dict(
+        >>>    student_id="Tim",
+        >>>    item_id="item_1",
+        >>>    course_id="course_1",
+        >>>    item_type="type_one"
+        >>> )
+        >>>
+        >>> submission_dict = dict(
+        >>>    student_item=2,
+        >>>    attempt_number=1,
+        >>>    submitted_at=datetime.datetime(2014, 1, 29, 23, 14, 52, 649284, tzinfo=<UTC>),
+        >>>    created_at=datetime.datetime(2014, 1, 29, 17, 14, 52, 668850, tzinfo=<UTC>),
+        >>>    answer=u'The answer is 42.'
+        >>> )
+        >>> set_score(student_item_dict, submission_dict, 11, 12)
+        {
+            'student_item': 2,
+            'submission': 1,
+            'points_earned': 11,
+            'points_possible': 12,
+            'created_at': datetime.datetime(2014, 2, 7, 20, 6, 42, 331156, tzinfo=<UTC>)
+        }
+
+    """
+    try:
+        student_item_model = StudentItem.objects.get(**student_item)
+        submission_model = Submission.objects.get(**submission)
+    except DatabaseError:
+        error_msg = u"Could not retrieve student item: {} or submission {}.".format(
+            student_item, submission
+        )
+        logger.exception(error_msg)
+        raise SubmissionRequestError(error_msg)
+
+    score = ScoreSerializer(
+        data={
+            "student_item": student_item_model.pk,
+            "submission": submission_model.pk,
+            "points_earned": score,
+            "points_possible": points_possible,
+        }
+    )
+    if not score.is_valid():
+        logger.exception(score.errors)
+        raise SubmissionInternalError(score.errors)
+    score.save()
+    return score.data
 
 
 def _get_or_create_student_item(student_item_dict):
@@ -262,7 +371,8 @@ def _get_or_create_student_item(student_item_dict):
         try:
             return StudentItem.objects.get(**student_item_dict)
         except StudentItem.DoesNotExist:
-            student_item_serializer = StudentItemSerializer(data=student_item_dict)
+            student_item_serializer = StudentItemSerializer(
+                data=student_item_dict)
             if not student_item_serializer.is_valid():
                 raise SubmissionRequestError(student_item_serializer.errors)
             return student_item_serializer.save()

apps/submissions/tests/test_api.py

@@ -7,7 +7,7 @@ from nose.tools import raises
 from mock import patch
 import pytz
 
-from submissions.api import create_submission, get_submissions, SubmissionRequestError, SubmissionInternalError
+from submissions import api as api
 from submissions.models import Submission
 from submissions.serializers import StudentItemSerializer
 
@@ -31,79 +31,84 @@ ANSWER_TWO = u"this is my other answer!"
 
 @ddt
 class TestApi(TestCase):
+
+    """
+    Testing Submissions
+    """
+
     def test_create_submission(self):
-        submission = create_submission(STUDENT_ITEM, ANSWER_ONE)
+        submission = api.create_submission(STUDENT_ITEM, ANSWER_ONE)
         self._assert_submission(submission, ANSWER_ONE, 1, 1)
 
     def test_get_submissions(self):
-        create_submission(STUDENT_ITEM, ANSWER_ONE)
-        create_submission(STUDENT_ITEM, ANSWER_TWO)
-        submissions = get_submissions(STUDENT_ITEM)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE)
+        api.create_submission(STUDENT_ITEM, ANSWER_TWO)
+        submissions = api.get_submissions(STUDENT_ITEM)
 
         self._assert_submission(submissions[1], ANSWER_ONE, 1, 1)
         self._assert_submission(submissions[0], ANSWER_TWO, 1, 2)
 
     def test_two_students(self):
-        create_submission(STUDENT_ITEM, ANSWER_ONE)
-        create_submission(SECOND_STUDENT_ITEM, ANSWER_TWO)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE)
+        api.create_submission(SECOND_STUDENT_ITEM, ANSWER_TWO)
 
-        submissions = get_submissions(STUDENT_ITEM)
+        submissions = api.get_submissions(STUDENT_ITEM)
         self.assertEqual(1, len(submissions))
         self._assert_submission(submissions[0], ANSWER_ONE, 1, 1)
 
-        submissions = get_submissions(SECOND_STUDENT_ITEM)
+        submissions = api.get_submissions(SECOND_STUDENT_ITEM)
         self.assertEqual(1, len(submissions))
         self._assert_submission(submissions[0], ANSWER_TWO, 2, 1)
 
 
     @file_data('test_valid_student_items.json')
     def test_various_student_items(self, valid_student_item):
-        create_submission(valid_student_item, ANSWER_ONE)
-        submission = get_submissions(valid_student_item)[0]
+        api.create_submission(valid_student_item, ANSWER_ONE)
+        submission = api.get_submissions(valid_student_item)[0]
         self._assert_submission(submission, ANSWER_ONE, 1, 1)
 
     def test_get_latest_submission(self):
         past_date = datetime.datetime(2007, 9, 12, 0, 0, 0, 0, pytz.UTC)
         more_recent_date = datetime.datetime(2007, 9, 13, 0, 0, 0, 0, pytz.UTC)
-        create_submission(STUDENT_ITEM, ANSWER_ONE, more_recent_date)
-        create_submission(STUDENT_ITEM, ANSWER_TWO, past_date)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE, more_recent_date)
+        api.create_submission(STUDENT_ITEM, ANSWER_TWO, past_date)
 
         # Test a limit on the submissions
-        submissions = get_submissions(STUDENT_ITEM, 1)
+        submissions = api.get_submissions(STUDENT_ITEM, 1)
         self.assertEqual(1, len(submissions))
         self.assertEqual(ANSWER_ONE, submissions[0]["answer"])
         self.assertEqual(more_recent_date.year,
                          submissions[0]["submitted_at"].year)
 
     def test_set_attempt_number(self):
-        create_submission(STUDENT_ITEM, ANSWER_ONE, None, 2)
-        submissions = get_submissions(STUDENT_ITEM)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE, None, 2)
+        submissions = api.get_submissions(STUDENT_ITEM)
         self._assert_submission(submissions[0], ANSWER_ONE, 1, 2)
 
-    @raises(SubmissionRequestError)
+    @raises(api.SubmissionRequestError)
     @file_data('test_bad_student_items.json')
     def test_error_checking(self, bad_student_item):
-        create_submission(bad_student_item, -100)
+        api.create_submission(bad_student_item, -100)
 
-    @raises(SubmissionRequestError)
+    @raises(api.SubmissionRequestError)
     def test_error_checking_submissions(self):
-        create_submission(STUDENT_ITEM, ANSWER_ONE, None, -1)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE, None, -1)
 
     @patch.object(Submission.objects, 'filter')
-    @raises(SubmissionInternalError)
+    @raises(api.SubmissionInternalError)
     def test_error_on_submission_creation(self, mock_filter):
         mock_filter.side_effect = DatabaseError("Bad things happened")
-        create_submission(STUDENT_ITEM, ANSWER_ONE)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE)
 
     @patch.object(StudentItemSerializer, 'save')
-    @raises(SubmissionInternalError)
+    @raises(api.SubmissionInternalError)
     def test_create_student_item_validation(self, mock_save):
         mock_save.side_effect = DatabaseError("Bad things happened")
-        create_submission(STUDENT_ITEM, ANSWER_ONE)
+        api.create_submission(STUDENT_ITEM, ANSWER_ONE)
 
     def test_unicode_enforcement(self):
-        create_submission(STUDENT_ITEM, "Testing unicode answers.")
-        submissions = get_submissions(STUDENT_ITEM, 1)
+        api.create_submission(STUDENT_ITEM, "Testing unicode answers.")
+        submissions = api.get_submissions(STUDENT_ITEM, 1)
         self.assertEqual(u"Testing unicode answers.", submissions[0]["answer"])
 
     def _assert_submission(self, submission, expected_answer, expected_item,
@@ -111,4 +116,29 @@ class TestApi(TestCase):
         self.assertIsNotNone(submission)
         self.assertEqual(submission["answer"], expected_answer)
         self.assertEqual(submission["student_item"], expected_item)
-        self.assertEqual(submission["attempt_number"], expected_attempt)
+        self.assertEqual(submission["attempt_number"], expected_attempt)
\ No newline at end of file
+
+    """
+    Testing Scores
+    """
+
+    def test_create_score(self):
+        submission = api.create_submission(STUDENT_ITEM, ANSWER_ONE)
+        self._assert_submission(submission, ANSWER_ONE, 1, 1)
+
+        score = api.set_score(STUDENT_ITEM, submission, 11, 12)
+        self._assert_score(score, 11, 12)
+
+    def test_get_score(self):
+        self.test_create_score()
+        scores = api.get_score(STUDENT_ITEM)
+        self._assert_score(scores[0], 11, 12)
+
+    def _assert_score(
+            self,
+            score,
+            expected_points_earned,
+            expected_points_possible):
+        self.assertIsNotNone(score)
+        self.assertEqual(score["points_earned"], expected_points_earned)
+        self.assertEqual(score["points_possible"], expected_points_possible)
\ No newline at end of file

manage.py

@@ -10,7 +10,7 @@ if __name__ == "__main__":
     if 'test' in sys.argv or 'harvest' in sys.argv:
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.test")
     else:
-        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.base")
+        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.dev")
 
     from django.core.management import execute_from_command_line
 

requirements/base.txt

@@ -8,4 +8,3 @@ django-extensions==1.3.3
 djangorestframework==2.3.5
 Mako==0.9.1
 pytz==2013.9
-django-pdb==0.3.2

requirements/dev.txt

@@ -3,6 +3,7 @@
 # Debug tools
 bpython==0.12
 django-debug-toolbar==0.11.0
+django-pdb==0.3.2
 sqlparse==0.1.10
 
 # Doc generation

settings/base.py

@@ -107,7 +107,6 @@ MIDDLEWARE_CLASSES = (
     'django.contrib.messages.middleware.MessageMiddleware',
     # Uncomment the next line for simple clickjacking protection:
     # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'django_pdb.middleware.PdbMiddleware',  # Needed to enable shell-on-crash behavior
 )
 
 ROOT_URLCONF = 'urls'
@@ -131,7 +130,6 @@ INSTALLED_APPS = (
 
     # Third party
     'django_extensions',
-    'django_pdb',            # Allows post-mortem debugging on exceptions
 
     # XBlock
     'workbench',

settings/dev.py

+"""
+Dev-specific Django settings.
+"""
+
+# Inherit from base settings
+from .base import *
+
+MIDDLEWARE_CLASSES += (
+    'django_pdb.middleware.PdbMiddleware',  # Needed to enable shell-on-crash behavior
+)
+
+INSTALLED_APPS += (
+    'django_pdb',            # Allows post-mortem debugging on exceptions
+)
\ No newline at end of file