Added CSV Support

Change-Id: I364eadff8cf5ce4598895923c3770defcef2fc5d

Makefile

@@ -53,4 +53,4 @@ loaddata: syncdb
 	python manage.py loaddata courses education_levels single_course_activity course_enrollment_birth_year course_enrollment_education course_enrollment_gender problem_response_answer_distribution course_enrollment_daily countries course_enrollment_country --database=analytics
 
 demo: clean requirements loaddata
-	python manage.py set_api_key analytics analytics
+	python manage.py set_api_key edx edx

analytics_data_api/v0/models.py

@@ -31,7 +31,7 @@ class CourseActivityByWeek(models.Model):
 
 class BaseCourseEnrollment(models.Model):
     course = models.ForeignKey(Course, null=False)
-    date = models.DateField(null=False)
+    date = models.DateField(null=False, db_index=True)
     count = models.IntegerField(null=False)
 
     class Meta(object):

analytics_data_api/v0/tests/test_views.py

 # NOTE: Full URLs are used throughout these tests to ensure that the API contract is fulfilled. The URLs should *not*
 # change for versions greater than 1.0.0. Tests target a specific version of the API, additional tests should be added
 # for subsequent versions if there are breaking changes introduced in those versions.
-
+import StringIO
+import csv
 import datetime
 import random
 
@@ -9,11 +10,9 @@ from django.conf import settings
 from django_dynamic_fixture import G
 import pytz
 
-from analytics_data_api.v0.models import CourseEnrollmentByBirthYear, CourseEnrollmentByEducation, EducationLevel, \
-    CourseEnrollmentByGender, CourseActivityByWeek, Course, ProblemResponseAnswerDistribution, CourseEnrollmentDaily, \
-    Country, \
-    CourseEnrollmentByCountry
+from analytics_data_api.v0 import models
 from analytics_data_api.v0.serializers import ProblemResponseAnswerDistributionSerializer
+from analytics_data_api.v0.tests.utils import flatten
 from analyticsdataserver.tests import TestCaseWithAuthentication
 
 
@@ -21,16 +20,16 @@ class CourseActivityLastWeekTest(TestCaseWithAuthentication):
     def setUp(self):
         super(CourseActivityLastWeekTest, self).setUp()
         self.course_id = 'edX/DemoX/Demo_Course'
-        self.course = G(Course, course_id=self.course_id)
+        self.course = G(models.Course, course_id=self.course_id)
         interval_start = '2014-05-24T00:00:00Z'
         interval_end = '2014-06-01T00:00:00Z'
-        G(CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
+        G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
           activity_type='posted_forum', count=100)
-        G(CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
+        G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
           activity_type='attempted_problem', count=200)
-        G(CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
+        G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
           activity_type='any', count=300)
-        G(CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
+        G(models.CourseActivityByWeek, course=self.course, interval_start=interval_start, interval_end=interval_end,
           activity_type='played_video', count=400)
 
     def test_activity(self):
@@ -89,7 +88,7 @@ class CourseEnrollmentViewTestCase(object):
     def _get_non_existent_course_id(self):
         course_id = random.randint(100, 9999)
 
-        if not Course.objects.filter(course_id=course_id).exists():
+        if not models.Course.objects.filter(course_id=course_id).exists():
             return course_id
 
         return self._get_non_existent_course_id()
@@ -105,12 +104,38 @@ class CourseEnrollmentViewTestCase(object):
         self.assertEquals(response.status_code, 404)
 
     def test_get(self):
+        # Validate the basic response status
         response = self.authenticated_get('/api/v0/courses/%s%s' % (self.course.course_id, self.path,))
         self.assertEquals(response.status_code, 200)
 
+        # Validate the actual data
         expected = self.get_expected_response(*self.model.objects.filter(date=self.date))
         self.assertEquals(response.data, expected)
 
+    def test_get_csv(self):
+        path = '/api/v0/courses/%s%s' % (self.course.course_id, self.path,)
+        csv_content_type = 'text/csv'
+        response = self.authenticated_get(path, HTTP_ACCEPT=csv_content_type)
+
+        # Validate the basic response status and content code
+        self.assertEquals(response.status_code, 200)
+        self.assertEquals(response['Content-Type'].split(';')[0], csv_content_type)
+
+        # Validate the actual data
+        data = self.get_expected_response(*self.model.objects.filter(date=self.date))
+        data = map(flatten, data)
+
+        # The CSV renderer sorts the headers alphabetically
+        fieldnames = sorted(data[0].keys())
+
+        # Generate the expected CSV output
+        expected = StringIO.StringIO()
+        writer = csv.DictWriter(expected, fieldnames)
+        writer.writeheader()
+        writer.writerows(data)
+
+        self.assertEqual(response.content, expected.getvalue())
+
     def test_get_with_intervals(self):
         expected = self.get_expected_response(*self.model.objects.filter(date=self.date))
         self.assertIntervalFilteringWorks(expected, self.date, self.date + datetime.timedelta(days=1))
@@ -141,11 +166,11 @@ class CourseEnrollmentViewTestCase(object):
 
 class CourseEnrollmentByBirthYearViewTests(TestCaseWithAuthentication, CourseEnrollmentViewTestCase):
     path = '/enrollment/birth_year'
-    model = CourseEnrollmentByBirthYear
+    model = models.CourseEnrollmentByBirthYear
 
     @classmethod
     def setUpClass(cls):
-        cls.course = G(Course)
+        cls.course = G(models.Course)
         cls.date = datetime.date(2014, 1, 1)
         G(cls.model, course=cls.course, date=cls.date, birth_year=1956)
         G(cls.model, course=cls.course, date=cls.date, birth_year=1986)
@@ -153,8 +178,9 @@ class CourseEnrollmentByBirthYearViewTests(TestCaseWithAuthentication, CourseEnr
         G(cls.model, course=cls.course, date=cls.date - datetime.timedelta(days=10), birth_year=1986)
 
     def get_expected_response(self, *args):
-        return [{'course_id': ce.course.course_id, 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
-                 'birth_year': ce.birth_year} for ce in args]
+        return [
+            {'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
+             'birth_year': ce.birth_year} for ce in args]
 
     def test_get(self):
         response = self.authenticated_get('/api/v0/courses/%s%s' % (self.course.course_id, self.path,))
@@ -170,13 +196,13 @@ class CourseEnrollmentByBirthYearViewTests(TestCaseWithAuthentication, CourseEnr
 
 class CourseEnrollmentByEducationViewTests(TestCaseWithAuthentication, CourseEnrollmentViewTestCase):
     path = '/enrollment/education/'
-    model = CourseEnrollmentByEducation
+    model = models.CourseEnrollmentByEducation
 
     @classmethod
     def setUpClass(cls):
-        cls.el1 = G(EducationLevel, name='Doctorate', short_name='doctorate')
-        cls.el2 = G(EducationLevel, name='Top Secret', short_name='top_secret')
-        cls.course = G(Course)
+        cls.el1 = G(models.EducationLevel, name='Doctorate', short_name='doctorate')
+        cls.el2 = G(models.EducationLevel, name='Top Secret', short_name='top_secret')
+        cls.course = G(models.Course)
         cls.date = datetime.date(2014, 1, 1)
         G(cls.model, course=cls.course, date=cls.date, education_level=cls.el1)
         G(cls.model, course=cls.course, date=cls.date, education_level=cls.el2)
@@ -184,26 +210,28 @@ class CourseEnrollmentByEducationViewTests(TestCaseWithAuthentication, CourseEnr
           education_level=cls.el2)
 
     def get_expected_response(self, *args):
-        return [{'course_id': ce.course.course_id, 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
-                 'education_level': {'name': ce.education_level.name, 'short_name': ce.education_level.short_name}} for
-                ce in args]
+        return [
+            {'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
+             'education_level': {'name': ce.education_level.name, 'short_name': ce.education_level.short_name}} for
+            ce in args]
 
 
 class CourseEnrollmentByGenderViewTests(TestCaseWithAuthentication, CourseEnrollmentViewTestCase):
     path = '/enrollment/gender/'
-    model = CourseEnrollmentByGender
+    model = models.CourseEnrollmentByGender
 
     @classmethod
     def setUpClass(cls):
-        cls.course = G(Course)
+        cls.course = G(models.Course)
         cls.date = datetime.date(2014, 1, 1)
         G(cls.model, course=cls.course, gender='m', date=cls.date, count=34)
         G(cls.model, course=cls.course, gender='f', date=cls.date, count=45)
         G(cls.model, course=cls.course, gender='f', date=cls.date - datetime.timedelta(days=2), count=45)
 
     def get_expected_response(self, *args):
-        return [{'course_id': ce.course.course_id, 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
-                 'gender': ce.gender} for ce in args]
+        return [
+            {'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
+             'gender': ce.gender} for ce in args]
 
 
 # pylint: disable=no-member,no-value-for-parameter
@@ -217,7 +245,7 @@ class AnswerDistributionTests(TestCaseWithAuthentication):
         cls.module_id = "i4x://org/num/run/problem/RANDOMNUMBER"
         cls.part_id1 = "i4x-org-num-run-problem-RANDOMNUMBER_2_1"
         cls.ad1 = G(
-            ProblemResponseAnswerDistribution,
+            models.ProblemResponseAnswerDistribution,
             course_id=cls.course_id,
             module_id=cls.module_id,
             part_id=cls.part_id1
@@ -238,33 +266,36 @@ class AnswerDistributionTests(TestCaseWithAuthentication):
 
 
 class CourseEnrollmentViewTests(TestCaseWithAuthentication, CourseEnrollmentViewTestCase):
-    model = CourseEnrollmentDaily
+    model = models.CourseEnrollmentDaily
     path = '/enrollment'
 
     @classmethod
     def setUpClass(cls):
-        cls.course = G(Course)
+        cls.course = G(models.Course)
         cls.date = datetime.date(2014, 1, 1)
         G(cls.model, course=cls.course, date=cls.date, count=203)
         G(cls.model, course=cls.course, date=cls.date - datetime.timedelta(days=5), count=203)
 
     def get_expected_response(self, *args):
-        return [{'course_id': ce.course.course_id, 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT)}
-                for ce in args]
+        return [
+            {'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT)}
+            for ce in args]
 
 
 class CourseEnrollmentByLocationViewTests(TestCaseWithAuthentication, CourseEnrollmentViewTestCase):
     path = '/enrollment/location/'
-    model = CourseEnrollmentByCountry
+    model = models.CourseEnrollmentByCountry
 
     def get_expected_response(self, *args):
-        return [{'course_id': ce.course.course_id, 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
-                 'country': {'code': ce.country.code, 'name': ce.country.name}} for ce in args]
+        return [
+            {'course_id': str(ce.course.course_id), 'count': ce.count, 'date': ce.date.strftime(settings.DATE_FORMAT),
+             'country': {'code': ce.country.code, 'name': ce.country.name}} for ce in args]
 
     @classmethod
     def setUpClass(cls):
-        cls.course = G(Course)
+        cls.course = G(models.Course)
         cls.date = datetime.date(2014, 1, 1)
-        G(cls.model, course=cls.course, country=G(Country), count=455, date=cls.date)
-        G(cls.model, course=cls.course, country=G(Country), count=356, date=cls.date)
-        G(cls.model, course=cls.course, country=G(Country), count=12, date=cls.date - datetime.timedelta(days=29))
+        G(cls.model, course=cls.course, country=G(models.Country), count=455, date=cls.date)
+        G(cls.model, course=cls.course, country=G(models.Country), count=356, date=cls.date)
+        G(cls.model, course=cls.course, country=G(models.Country), count=12,
+          date=cls.date - datetime.timedelta(days=29))

analytics_data_api/v0/tests/utils.py

+import collections
+
+
+def flatten(dictionary, parent_key='', sep='.'):
+    """
+    Flatten dictionary
+
+    http://stackoverflow.com/a/6027615
+    """
+    items = []
+    for key, value in dictionary.items():
+        new_key = parent_key + sep + key if parent_key else key
+        if isinstance(value, collections.MutableMapping):
+            items.extend(flatten(value, new_key).items())
+        else:
+            items.append((new_key, value))
+    return dict(items)

analyticsdataserver/settings/base.py

 """Common settings and globals."""
 
-
 from os.path import abspath, basename, dirname, join, normpath
 from sys import stderr
 
@@ -250,6 +249,11 @@ REST_FRAMEWORK = {
         # For the browseable API
         'rest_framework.authentication.SessionAuthentication',
     ),
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+        'rest_framework.renderers.BrowsableAPIRenderer',
+        'rest_framework_csv.renderers.CSVRenderer',
+    )
 }
 ########## END REST FRAMEWORK CONFIGURATION
 

analyticsdataserver/settings/local.py

@@ -85,5 +85,5 @@ ENABLE_ADMIN_SITE = True
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 
 SWAGGER_SETTINGS = {
-    'api_key': 'analytics'
+    'api_key': 'edx'
 }

analyticsdataserver/tests.py

 from contextlib import contextmanager
-from functools import partial
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db.utils import ConnectionHandler, DatabaseError
 from django.test import TestCase
 from django.test.utils import override_settings
-from mock import patch, Mock
+
 import mock
 from rest_framework.authtoken.models import Token
 
@@ -15,13 +14,16 @@ class TestCaseWithAuthentication(TestCase):
     def setUp(self):
         super(TestCaseWithAuthentication, self).setUp()
         test_user = User.objects.create_user('tester', 'test@example.com', 'testpassword')
-        token = Token.objects.create(user=test_user)
-        self.authenticated_get = partial(self.client.get, HTTP_AUTHORIZATION='Token ' + token.key, follow=True)
+        self.token = Token.objects.create(user=test_user)
+
+    def authenticated_get(self, path, data=None, follow=True, **extra):
+        data = data or {}
+        return self.client.get(path, data, follow, HTTP_AUTHORIZATION='Token ' + self.token.key, **extra)
 
 
 @contextmanager
 def no_database():
-    cursor_mock = Mock(side_effect=DatabaseError)
+    cursor_mock = mock.Mock(side_effect=DatabaseError)
     with mock.patch('django.db.backends.util.CursorWrapper', cursor_mock):
         yield
 
@@ -58,7 +60,7 @@ class OperationalEndpointsTest(TestCaseWithAuthentication):
     @staticmethod
     @contextmanager
     def override_database_connections(databases):
-        with patch('analyticsdataserver.views.connections', ConnectionHandler(databases)):
+        with mock.patch('analyticsdataserver.views.connections', ConnectionHandler(databases)):
             yield
 
     @override_settings(ANALYTICS_DATABASE='reporting')

requirements/base.txt

@@ -5,3 +5,4 @@ django-model-utils==1.4.0
 djangorestframework==2.3.5
 ipython==2.1.0
 django-rest-swagger==0.1.14
+djangorestframework-csv==1.3.3