Generate assessment and submission CSV data.

Added management command to upload archive of CSV data to S3
Add a scenario with unicode in all rubric strings
Add tests for unicode CSV reports
Added progress indicator to upload management command
Optimize database queries for CSV reports.
Fix issue with duplicate assessment feedback options.

apps/openassessment/assessment/test/test_serializers.py

@@ -19,7 +19,7 @@ class TestRubricDeserialization(CacheResetTest):
     def test_rubric_only_created_once(self):
         # Make sure sending the same Rubric data twice only creates one Rubric,
         # and returns a reference to it the next time.
-        rubric_data = json_data('rubric_data/project_plan_rubric.json')
+        rubric_data = json_data('data/rubric/project_plan_rubric.json')
 
         r1 = rubric_from_dict(rubric_data)
 
@@ -32,14 +32,14 @@ class TestRubricDeserialization(CacheResetTest):
 
     def test_rubric_requires_positive_score(self):
         with self.assertRaises(InvalidRubric):
-            rubric_from_dict(json_data('rubric_data/no_points.json'))
+            rubric_from_dict(json_data('data/rubric/no_points.json'))
 
 
 class TestCriterionDeserialization(CacheResetTest):
 
     def test_empty_criteria(self):
         with self.assertRaises(InvalidRubric) as cm:
-            rubric_from_dict(json_data('rubric_data/empty_criteria.json'))
+            rubric_from_dict(json_data('data/rubric/empty_criteria.json'))
         self.assertEqual(
             cm.exception.errors,
             {'criteria': [u'Must have at least one criterion']}
@@ -47,7 +47,7 @@ class TestCriterionDeserialization(CacheResetTest):
 
     def test_missing_criteria(self):
         with self.assertRaises(InvalidRubric) as cm:
-            rubric_from_dict(json_data('rubric_data/missing_criteria.json'))
+            rubric_from_dict(json_data('data/rubric/missing_criteria.json'))
         self.assertEqual(
             cm.exception.errors,
             {'criteria': [u'This field is required.']}
@@ -58,7 +58,7 @@ class TestCriterionOptionDeserialization(CacheResetTest):
 
     def test_empty_options(self):
         with self.assertRaises(InvalidRubric) as cm:
-            rubric_from_dict(json_data('rubric_data/empty_options.json'))
+            rubric_from_dict(json_data('data/rubric/empty_options.json'))
         self.assertEqual(
             cm.exception.errors,
             {
@@ -71,7 +71,7 @@ class TestCriterionOptionDeserialization(CacheResetTest):
 
     def test_missing_options(self):
         with self.assertRaises(InvalidRubric) as cm:
-            rubric_from_dict(json_data('rubric_data/missing_options.json'))
+            rubric_from_dict(json_data('data/rubric/missing_options.json'))
         self.assertEqual(
             cm.exception.errors,
             {

apps/openassessment/management/commands/upload_oa_data.py

+"""
+Generate CSV files for submission and assessment data, then upload to S3.
+"""
+import sys
+import os
+import os.path
+import datetime
+import shutil
+import tempfile
+import tarfile
+import boto
+from boto.s3.key import Key
+from django.core.management.base import BaseCommand, CommandError
+from openassessment.data import CsvWriter
+
+
+class Command(BaseCommand):
+    """
+    Create and upload CSV files for submission and assessment data.
+    """
+
+    help = 'Create and upload CSV files for submission and assessment data.'
+    args = '<COURSE_ID> <S3_BUCKET_NAME>'
+
+    OUTPUT_CSV_PATHS = {
+        output_name: "{}.csv".format(output_name)
+        for output_name in CsvWriter.MODELS
+    }
+
+    URL_EXPIRATION_HOURS = 24
+    PROGRESS_INTERVAL = 10
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+        self._history = list()
+        self._submission_counter = 0
+
+    @property
+    def history(self):
+        """
+        Return the upload history, which is useful for testing.
+
+        Returns:
+            list of dictionaries with keys 'url' and 'key'
+
+        """
+        return self._history
+
+    def handle(self, *args, **options):
+        """
+        Execute the command.
+
+        Args:
+            course_id (unicode): The ID of the course to create submissions for.
+            s3_bucket_name (unicode): The name of the S3 bucket to upload to.
+
+        Raises:
+            CommandError
+
+        """
+        if len(args) < 2:
+            raise CommandError(u'Usage: upload_oa_data {}'.format(self.args))
+
+        course_id, s3_bucket = args[0].decode('utf-8'), args[1].decode('utf-8')
+        csv_dir = tempfile.mkdtemp()
+
+        try:
+            print u"Generating CSV files for course '{}'".format(course_id)
+            self._dump_to_csv(course_id, csv_dir)
+            print u"Creating archive of CSV files in {}".format(csv_dir)
+            archive_path = self._create_archive(csv_dir)
+            print u"Uploading {} to {}/{}".format(archive_path, s3_bucket, course_id)
+            url = self._upload(course_id, archive_path, s3_bucket)
+            print "== Upload successful =="
+            print u"Download URL (expires in {} hours):\n{}".format(self.URL_EXPIRATION_HOURS, url)
+        finally:
+            # Assume that the archive was created in the directory,
+            # so to clean up we just need to delete the directory.
+            shutil.rmtree(csv_dir)
+
+    def _dump_to_csv(self, course_id, csv_dir):
+        """
+        Create CSV files for submission/assessment data in a directory.
+
+        Args:
+            course_id (unicode): The ID of the course to dump data from.
+            csv_dir (unicode): The absolute path to the directory in which to create CSV files.
+
+        Returns:
+            None
+        """
+        output_streams = {
+            name: open(os.path.join(csv_dir, rel_path), 'w')
+            for name, rel_path in self.OUTPUT_CSV_PATHS.iteritems()
+        }
+        csv_writer = CsvWriter(output_streams, self._progress_callback)
+        csv_writer.write_to_csv(course_id)
+
+    def _create_archive(self, dir_path):
+        """
+        Create an archive of a directory.
+
+        Args:
+            dir_path (unicode): The absolute path to the directory containing the CSV files.
+
+        Returns:
+            unicode: Absolute path to the archive.
+
+        """
+        tarball_name = u"{}.tar.gz".format(
+            datetime.datetime.utcnow().strftime("%Y-%m-%dT%H_%M")
+        )
+        tarball_path = os.path.join(dir_path, tarball_name)
+        with tarfile.open(tarball_path, "w:gz") as tar:
+            for rel_path in self.OUTPUT_CSV_PATHS.values():
+                tar.add(os.path.join(dir_path, rel_path), arcname=rel_path)
+        return tarball_path
+
+    def _upload(self, course_id, file_path, s3_bucket):
+        """
+        Upload a file.
+
+        Args:
+            course_id (unicode): The ID of the course.
+            file_path (unicode): Absolute path to the file to upload.
+            s3_bucket (unicode): Name of the S3 bucket where the file will be uploaded.
+
+        Returns:
+            str: URL to access the uploaded archive.
+
+        """
+        conn = boto.connect_s3()
+        bucket = conn.get_bucket(s3_bucket)
+        key_name = os.path.join(course_id, os.path.split(file_path)[1])
+        key = Key(bucket=bucket, name=key_name)
+        key.set_contents_from_filename(file_path)
+        url = key.generate_url(self.URL_EXPIRATION_HOURS * 3600)
+
+        # Store the key and url in the history
+        self._history.append({'key': key_name, 'url': url})
+
+        return url
+
+    def _progress_callback(self):
+        """
+        Indicate progress to the user as submissions are processed.
+        """
+        self._submission_counter += 1
+        if self._submission_counter > 0 and self._submission_counter % self.PROGRESS_INTERVAL == 0:
+            sys.stdout.write('.')
+            sys.stdout.flush()

apps/openassessment/management/tests/test_upload_oa_data.py

+# -*- coding: utf-8 -*-
+"""
+Tests for management command that uploads submission/assessment data.
+"""
+from StringIO import StringIO
+import tarfile
+from django.test import TestCase
+import boto
+import moto
+from openassessment.management.commands import upload_oa_data
+from openassessment.workflow import api as workflow_api
+from submissions import api as sub_api
+
+
+class UploadDataTest(TestCase):
+    """
+    Test the upload management command.  Archiving and upload are in-scope,
+    but the contents of the generated CSV files are tested elsewhere.
+    """
+
+    COURSE_ID = u"TɘꙅT ↄoUᴙꙅɘ"
+    BUCKET_NAME = u"com.example.data"
+    CSV_NAMES = [
+        "assessment.csv", "assessment_part.csv",
+        "assessment_feedback.csv", "assessment_feedback_option.csv",
+        "submission.csv", "score.csv",
+    ]
+
+    @moto.mock_s3
+    def test_upload(self):
+        # Create an S3 bucket using the fake S3 implementation
+        conn = boto.connect_s3()
+        conn.create_bucket(self.BUCKET_NAME)
+
+        # Create some submissions to ensure that we cover
+        # the progress indicator code.
+        for index in range(50):
+            student_item = {
+                'student_id': "test_user_{}".format(index),
+                'course_id': self.COURSE_ID,
+                'item_id': 'test_item',
+                'item_type': 'openassessment',
+            }
+            submission_text = "test submission {}".format(index)
+            submission = sub_api.create_submission(student_item, submission_text)
+            workflow_api.create_workflow(submission['uuid'])
+
+        # Create and upload the archive of CSV files
+        # This should generate the files even though
+        # we don't have any data available.
+        cmd = upload_oa_data.Command()
+        cmd.handle(self.COURSE_ID.encode('utf-8'), self.BUCKET_NAME)
+
+        # Retrieve the uploaded file from the fake S3 implementation
+        self.assertEqual(len(cmd.history), 1)
+        bucket = conn.get_all_buckets()[0]
+        key = bucket.get_key(cmd.history[0]['key'])
+        contents = StringIO(key.get_contents_as_string())
+
+        # Expect that the contents contain all the expected CSV files
+        with tarfile.open(mode="r:gz", fileobj=contents) as tar:
+            file_sizes = {
+                member.name: member.size
+                for member in tar.getmembers()
+            }
+            for csv_name in self.CSV_NAMES:
+                self.assertIn(csv_name, file_sizes)
+                self.assertGreater(file_sizes[csv_name], 0)
+
+        # Expect that we generated a URL for the bucket
+        url = cmd.history[0]['url']
+        self.assertIn("https://{}".format(self.BUCKET_NAME), url)

apps/openassessment/tests/data/db_fixtures/empty.json

+[
+  {
+    "pk": 1,
+    "model": "assessment.rubric",
+    "fields": {
+      "content_hash": "7405a513d9f99b62dd561816f20cdb90b09b8060"
+    }
+  },
+  {
+    "pk": 1,
+    "model": "assessment.criterion",
+    "fields": {
+      "order_num": 0,
+      "prompt": "How concise is it?",
+      "rubric": 1,
+      "name": "concise"
+    }
+  },
+  {
+    "pk": 2,
+    "model": "assessment.criterion",
+    "fields": {
+      "order_num": 1,
+      "prompt": "How clear is the thinking?",
+      "rubric": 1,
+      "name": "clear-headed"
+    }
+  },
+  {
+    "pk": 3,
+    "model": "assessment.criterion",
+    "fields": {
+      "order_num": 2,
+      "prompt": "Lastly, how is its form? Punctuation, grammar, and spelling all count.",
+      "rubric": 1,
+      "name": "form"
+    }
+  },
+  {
+    "pk": 1,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 0,
+      "explanation": "\n                    In \"Cryptonomicon\", Stephenson spent multiple pages talking about breakfast cereal.\n                    While hilarious, in recent years his work has been anything but 'concise'.\n                ",
+      "points": 0,
+      "criterion": 1,
+      "name": "Neal Stephenson (late)"
+    }
+  },
+  {
+    "pk": 2,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 1,
+      "explanation": "\n                    If the author wrote something cyclopean that staggers the mind, score it thus.\n                ",
+      "points": 1,
+      "criterion": 1,
+      "name": "HP Lovecraft"
+    }
+  },
+  {
+    "pk": 3,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 2,
+      "explanation": "\n                    Tight prose that conveys a wealth of information about the world in relatively\n                    few words. Example, \"The door irised open and he stepped inside.\"\n                ",
+      "points": 3,
+      "criterion": 1,
+      "name": "Robert Heinlein"
+    }
+  },
+  {
+    "pk": 4,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 3,
+      "explanation": "\n                    When Stephenson still had an editor, his prose was dense, with anecdotes about\n                    nitrox abuse implying main characters' whole life stories.\n                ",
+      "points": 4,
+      "criterion": 1,
+      "name": "Neal Stephenson (early)"
+    }
+  },
+  {
+    "pk": 5,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 4,
+      "explanation": "\n                    Score the work this way if it makes you weep, and the removal of a single\n                    word would make you sneer.\n                ",
+      "points": 5,
+      "criterion": 1,
+      "name": "Earnest Hemingway"
+    }
+  },
+  {
+    "pk": 6,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 0,
+      "explanation": "",
+      "points": 0,
+      "criterion": 2,
+      "name": "Yogi Berra"
+    }
+  },
+  {
+    "pk": 7,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 1,
+      "explanation": "",
+      "points": 1,
+      "criterion": 2,
+      "name": "Hunter S. Thompson"
+    }
+  },
+  {
+    "pk": 8,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 2,
+      "explanation": "",
+      "points": 2,
+      "criterion": 2,
+      "name": "Robert Heinlein"
+    }
+  },
+  {
+    "pk": 9,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 3,
+      "explanation": "",
+      "points": 3,
+      "criterion": 2,
+      "name": "Isaac Asimov"
+    }
+  },
+  {
+    "pk": 10,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 4,
+      "explanation": "\n                    Coolly rational, with a firm grasp of the main topics, a crystal-clear train of thought,\n                    and unemotional examination of the facts.  This is the only item explained in this category,\n                    to show that explained and unexplained items can be mixed.\n                ",
+      "points": 10,
+      "criterion": 2,
+      "name": "Spock"
+    }
+  },
+  {
+    "pk": 11,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 0,
+      "explanation": "",
+      "points": 0,
+      "criterion": 3,
+      "name": "lolcats"
+    }
+  },
+  {
+    "pk": 12,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 1,
+      "explanation": "",
+      "points": 1,
+      "criterion": 3,
+      "name": "Facebook"
+    }
+  },
+  {
+    "pk": 13,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 2,
+      "explanation": "",
+      "points": 2,
+      "criterion": 3,
+      "name": "Reddit"
+    }
+  },
+  {
+    "pk": 14,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 3,
+      "explanation": "",
+      "points": 3,
+      "criterion": 3,
+      "name": "metafilter"
+    }
+  },
+  {
+    "pk": 15,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 4,
+      "explanation": "",
+      "points": 4,
+      "criterion": 3,
+      "name": "Usenet, 1996"
+    }
+  },
+  {
+    "pk": 16,
+    "model": "assessment.criterionoption",
+    "fields": {
+      "order_num": 5,
+      "explanation": "",
+      "points": 5,
+      "criterion": 3,
+      "name": "The Elements of Style"
+    }
+  }
+]
\ No newline at end of file

apps/openassessment/tests/data/write_to_csv.json

+{
+    "empty": {
+        "fixture": "db_fixtures/empty.json",
+        "course_id": "edX/Enchantment_101/April_1",
+        "expected_csv": {
+            "assessment": [
+                [
+                    "id", "submission_uuid", "scored_at", "scorer_id", "score_type",
+                    "points_possible", "feedback"
+                ]
+            ],
+            "assessment_feedback": [
+                ["submission_uuid", "feedback_text", "options"]
+            ],
+            "assessment_part": [
+                ["assessment_id", "points_earned", "criterion_name", "option_name", "feedback"]
+            ],
+            "assessment_feedback_option": [
+                ["id", "text"]
+            ],
+            "submission": [
+                ["uuid", "student_id", "item_id", "submitted_at", "created_at", "raw_answer"]
+            ],
+            "score": [
+                ["submission_uuid", "points_earned", "points_possible", "created_at"]
+            ]
+        }
+    },
+
+    "submitted": {
+        "fixture": "db_fixtures/submitted.json",
+        "course_id": "edX/Enchantment_101/April_1",
+        "expected_csv": {
+            "submission": [
+                ["uuid", "student_id", "item_id", "submitted_at", "created_at", "raw_answer"],
+                [
+                    "cf5190b8-d0aa-11e3-a734-14109fd8dc43",
+                    "student_1", "openassessmentblock-poverty-rubric.openassessment.d0.u0",
+                    "2014-04-30 21:02:59.234000+00:00", "2014-04-30 21:02:59.241000+00:00",
+                    "{\"text\": \"Lorem ipsum dolor sit amet\"}"
+                ]
+            ]
+        }
+    },
+
+    "peer_assessed": {
+        "fixture": "db_fixtures/peer_assessed.json",
+        "course_id": "edX/Enchantment_101/April_1",
+        "expected_csv": {
+            "assessment": [
+                [
+                    "id", "submission_uuid", "scored_at", "scorer_id", "score_type",
+                    "points_possible", "feedback"
+                ],
+                [
+                    "1", "cf5190b8-d0aa-11e3-a734-14109fd8dc43",
+                    "2014-04-30 21:06:35.019000+00:00",
+                    "other",
+                    "PE",
+                    "20",
+                    "Donec consequat vitae ante in pellentesque."
+                ]
+            ],
+            "assessment_part": [
+                ["assessment_id", "points_earned", "criterion_name", "option_name", "feedback"],
+                ["1", "4", "concise", "Neal Stephenson (early)", "Praesent ac lorem ac nunc tincidunt ultricies sit amet ut magna."],
+                ["1", "5", "form", "The Elements of Style", "Fusce varius, elit ut blandit consequat, odio ante mollis lectus"],
+                ["1", "3", "clear-headed", "Isaac Asimov", ""]
+            ]
+        }
+    },
+
+    "self_assessed": {
+        "fixture": "db_fixtures/self_assessed.json",
+        "course_id": "edX/Enchantment_101/April_1",
+        "expected_csv": {
+            "assessment": [
+                [
+                    "id", "submission_uuid", "scored_at", "scorer_id", "score_type",
+                    "points_possible", "feedback"
+                ],
+                [
+                    "1", "cf5190b8-d0aa-11e3-a734-14109fd8dc43",
+                    "2014-04-30 21:06:35.019000+00:00",
+                    "other",
+                    "PE",
+                    "20",
+                    "Donec consequat vitae ante in pellentesque."
+                ],
+                [
+                    "2", "28cebeca-d0ab-11e3-a6ab-14109fd8dc43",
+                    "2014-04-30 21:06:59.953000+00:00",
+                    "other",
+                    "SE",
+                    "20",
+                    ""
+                ]
+            ],
+            "assessment_part": [
+                ["assessment_id", "points_earned", "criterion_name", "option_name", "feedback"],
+                ["1", "4", "concise", "Neal Stephenson (early)", "Praesent ac lorem ac nunc tincidunt ultricies sit amet ut magna."],
+                ["1", "5", "form", "The Elements of Style", "Fusce varius, elit ut blandit consequat, odio ante mollis lectus"],
+                ["1", "3", "clear-headed", "Isaac Asimov", ""],
+                ["2", "5", "concise", "Earnest Hemingway", ""],
+                ["2", "5", "form", "The Elements of Style", ""],
+                ["2", "10", "clear-headed", "Spock", ""]
+            ]
+        }
+    },
+
+    "scored": {
+        "fixture": "db_fixtures/scored.json",
+        "course_id": "edX/Enchantment_101/April_1",
+        "expected_csv": {
+            "score": [
+                ["submission_uuid", "points_earned", "points_possible", "created_at"],
+                [
+                    "cf5190b8-d0aa-11e3-a734-14109fd8dc43",
+                    "12", "20",
+                    "2014-04-30 21:07:53.534000+00:00"
+                ],
+                [
+                    "28cebeca-d0ab-11e3-a6ab-14109fd8dc43",
+                    "17", "20",
+                    "2014-04-30 21:07:46.524000+00:00"
+                ]
+            ]
+        }
+    },
+
+    "feedback_on_assessment": {
+        "fixture": "db_fixtures/feedback_on_assessment.json",
+        "course_id": "edX/Enchantment_101/April_1",
+        "expected_csv": {
+            "assessment_feedback": [
+                ["submission_uuid", "feedback_text", "options"],
+                [
+                    "1783758f-d0ae-11e3-b495-14109fd8dc43",
+                    "Feedback on assessment",
+                    "1,2"
+                ],
+                [
+                    "387d840a-d0ae-11e3-bb0e-14109fd8dc43",
+                    "Feedback on assessment",
+                    "1,2"
+                ]
+            ],
+            "assessment_feedback_option": [
+                ["id", "text"],
+                ["1", "These assessments were useful."],
+                ["2", "I disagree with one or more of the peer assessments of my response."]
+            ]
+        }
+    }
+}

apps/openassessment/tests/test_data.py

+# -*- coding: utf-8 -*-
+"""
+Tests for openassessment data aggregation.
+"""
+
+import os.path
+from StringIO import StringIO
+import csv
+from django.core.management import call_command
+import ddt
+from openassessment.test_utils import CacheResetTest
+from submissions import api as sub_api
+from openassessment.workflow import api as workflow_api
+from openassessment.data import CsvWriter
+
+
+@ddt.ddt
+class CsvWriterTest(CacheResetTest):
+    """
+    Test for writing openassessment data to CSV.
+    """
+    longMessage = True
+    maxDiff = None
+
+    @ddt.file_data('data/write_to_csv.json')
+    def test_write_to_csv(self, data):
+        # Create in-memory buffers for the CSV file data
+        output_streams = self._output_streams(data['expected_csv'].keys())
+
+        # Load the database fixture
+        # We use the database fixture to ensure that this test will
+        # catch backwards-compatibility issues even if the Django model
+        # implementation or API calls change.
+        self._load_fixture(data['fixture'])
+
+        # Write the data to CSV
+        writer = CsvWriter(output_streams)
+        writer.write_to_csv(data['course_id'])
+
+        # Check that the CSV matches what we expected
+        for output_name, expected_csv in data['expected_csv'].iteritems():
+            output_buffer = output_streams[output_name]
+            output_buffer.seek(0)
+            actual_csv = csv.reader(output_buffer)
+            for expected_row in expected_csv:
+                try:
+                    actual_row = actual_csv.next()
+                except StopIteration:
+                    actual_row = None
+                self.assertEqual(
+                    actual_row, expected_row,
+                    msg="Output name: {}".format(output_name)
+                )
+
+            # Check for extra rows
+            try:
+                extra_row = actual_csv.next()
+            except StopIteration:
+                extra_row = None
+
+            if extra_row is not None:
+                self.fail(u"CSV contains extra row: {}".format(extra_row))
+
+    def test_many_submissions(self):
+        # Create a lot of submissions
+        num_submissions = 234
+        for index in range(num_submissions):
+            student_item = {
+                'student_id': "test_user_{}".format(index),
+                'course_id': 'test_course',
+                'item_id': 'test_item',
+                'item_type': 'openassessment',
+            }
+            submission_text = "test submission {}".format(index)
+            submission = sub_api.create_submission(student_item, submission_text)
+            workflow_api.create_workflow(submission['uuid'])
+
+        # Generate a CSV file for the submissions
+        output_streams = self._output_streams(['submission'])
+        writer = CsvWriter(output_streams)
+        writer.write_to_csv('test_course')
+
+        # Parse the generated CSV
+        content = output_streams['submission'].getvalue()
+        rows = content.split('\n')
+
+        # Remove the first row (header) and last row (blank line)
+        rows = rows[1:-1]
+
+        # Check that we have the right number of rows
+        self.assertEqual(len(rows), num_submissions)
+
+    def test_other_course_id(self):
+        # Try a course ID with no submissions
+        self._load_fixture('db_fixtures/scored.json')
+        output_streams = self._output_streams(CsvWriter.MODELS)
+        writer = CsvWriter(output_streams)
+        writer.write_to_csv('other_course')
+
+        # Expect that each output has only two lines (the header and a blank line)
+        # since this course has no submissions
+        for output in output_streams.values():
+            content = output.getvalue()
+            rows = content.split('\n')
+            self.assertEqual(len(rows), 2)
+
+    def test_unicode(self):
+        # Flush out unicode errors
+        self._load_fixture('db_fixtures/unicode.json')
+        output_streams = self._output_streams(CsvWriter.MODELS)
+        CsvWriter(output_streams).write_to_csv(u"𝓽𝓮𝓼𝓽_𝓬𝓸𝓾𝓻𝓼𝓮")
+
+        # Check that data ended up in the reports
+        for output in output_streams.values():
+            content = output.getvalue()
+            rows = content.split('\n')
+            self.assertGreater(len(rows), 2)
+
+    def _output_streams(self, names):
+        """
+        Create in-memory buffers.
+
+        Args:
+            names (list of unicode): The output names.
+
+        Returns:
+            dict: map of output names to StringIO objects.
+
+        """
+        output_streams = dict()
+
+        for output_name in names:
+            output_buffer = StringIO()
+            self.addCleanup(output_buffer.close)
+            output_streams[output_name] = output_buffer
+
+        return output_streams
+
+    def _load_fixture(self, fixture_relpath):
+        """
+        Load a database fixture into the test database.
+
+        Args:
+            fixture_relpath (unicode): Path to the fixture,
+                relative to the test/data directory.
+
+        Returns:
+            None
+        """
+        fixture_path = os.path.join(
+            os.path.dirname(__file__), 'data', fixture_relpath
+        )
+        print "Loading database fixtures from {}".format(fixture_path)
+        call_command('loaddata', fixture_path)

apps/openassessment/xblock/openassessmentblock.py

@@ -305,6 +305,10 @@ class OpenAssessmentBlock(
         """
         return [
             (
+                "OpenAssessmentBlock Unicode",
+                load('static/xml/unicode.xml')
+            ),
+            (
                 "OpenAssessmentBlock Poverty Rubric",
                 load('static/xml/poverty_rubric_example.xml')
             ),

apps/openassessment/xblock/static/xml/censorship_rubric_example.xml

-<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
 <openassessment>
     <title>
         Censorship in Public Libraries

apps/openassessment/xblock/static/xml/poverty_rubric_example.xml

-<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
 <openassessment submission_due="2015-03-11T18:20">
     <title>
         Global Poverty

apps/openassessment/xblock/static/xml/promptless_rubric_example.xml

-<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
 <openassessment>
     <title>
         Promptless rubric

apps/openassessment/xblock/static/xml/unicode.xml

+<openassessment>
+    <title>υηι¢σ∂є тєѕт</title>
+    <rubric>
+        <prompt>Uᴎiↄobɘ qᴙomqT</prompt>
+        <criterion feedback='optional'>
+            <name>ȼønȼɨsɇ</name>
+            <prompt>Нош соисіѕэ іѕ іт?</prompt>
+            <option points="0">
+                <name>ｱoo尺</name>
+                <explanation>ʇɥǝ ʍɹıʇǝɹ pıp ɐ dooɹ ɾoq.</explanation>
+            </option>
+            <option points="1">
+                <name>ꟻAiᴙ</name>
+                <explanation>Ṫḧë ẅṛïẗëṛ ḋïḋ ä ḟäïṛ jöḅ.</explanation>
+            </option>
+            <option points="3">
+                <name>Ǥøøđ</name>
+                <explanation>ᴛʜᴇ ᴡʀɪᴛᴇʀ ᴅɪᴅ ᴀ ɢᴏᴏᴅ ᴊᴏʙ.</explanation>
+            </option>
+        </criterion>
+        <criterion>
+            <name>őń-tőṕíć</name>
+            <prompt>ωαѕ тнє ωяιтєя ση тσρι¢?</prompt>
+            <option points="0">
+                <name>ｱoo尺</name>
+                <explanation>ʇɥǝ ʍɹıʇǝɹ pıp ɐ dooɹ ɾoq.</explanation>
+            </option>
+            <option points="1">
+                <name>ꟻAiᴙ</name>
+                <explanation>Ṫḧë ẅṛïẗëṛ ḋïḋ ä ḟäïṛ jöḅ.</explanation>
+            </option>
+            <option points="3">
+                <name>Ǥøøđ</name>
+                <explanation>ᴛʜᴇ ᴡʀɪᴛᴇʀ ᴅɪᴅ ᴀ ɢᴏᴏᴅ ᴊᴏʙ.</explanation>
+            </option>
+        </criterion>
+    </rubric>
+    <assessments>
+        <assessment name="peer-assessment" must_grade="1" must_be_graded_by="1" />
+        <assessment name="self-assessment" />
+    </assessments>
+</openassessment>

requirements/base.txt

 # edX Internal Requirements
-git+https://github.com/edx/XBlock.git@3b6e4218bd326f84dbeb0baed7b2b7813ffea3dd#egg=XBlock
+git+https://github.com/edx/XBlock.git@fc5fea25c973ec66d8db63cf69a817ce624f5ef5#egg=XBlock
 git+https://github.com/edx/xblock-sdk.git@643900aadcb18aaeb7fe67271ca9dbf36e463ee6#egg=xblock-sdk
 
 # Third Party Requirements
+boto==2.13.3
 defusedxml==0.4.1
 dogapi==1.2.1
 django>=1.4,<1.5

requirements/test.txt

 # Grab everything in base requirements
 -r base.txt
 
+# There's a unicode bug in the httpretty==0.8 (used by moto)
+# Once a new version gets released on PyPi we can use that instead.
+git+https://github.com/gabrielfalcao/HTTPretty.git@4c2b10925c86c9b6299c1a04ae334d89fe007ae2#egg=httpretty
+
 ddt==0.7.0
 django-nose==1.2
 mock==1.0.1
+moto==0.2.22
 nose==1.3.0
 coverage==3.7.1
 pep8==1.4.6