Merge pull request #128 from open-craft/jill/learner-api-csv

Adds CSV support to the LearnerListView

analytics_data_api/constants/learner.py

-LEARNER_API_DEFAULT_LIST_PAGE_SIZE = 25
-
 SEGMENTS = ["highly_engaged", "disengaging", "struggling", "inactive", "unenrolled"]

analytics_data_api/renderers.py

+"""
+Custom REST framework renderers common to all versions of the API.
+"""
+from rest_framework_csv.renderers import CSVRenderer
+from ordered_set import OrderedSet
+
+
+class ResultsOnlyRendererMixin(object):
+    """
+    Render data using just the results array.
+
+    Use with PaginatedHeadersMixin to preserve the pagination links in the response header.
+    """
+    results_field = 'results'
+
+    def render(self, data, *args, **kwargs):
+        """
+        Replace the rendered data with just what is in the results_field.
+        """
+        if not isinstance(data, list):
+            data = data.get(self.results_field, [])
+        return super(ResultsOnlyRendererMixin, self).render(data, *args, **kwargs)
+
+
+class DynamicFieldsCsvRenderer(CSVRenderer):
+    """
+    Allows the `fields` query parameter to determine which fields should be
+    returned in the response, and in what order.
+
+    Note that if no header is provided, and the fields_param query string
+    parameter is not found in the request, the fields are rendered in
+    alphabetical order.
+    """
+    # Name of the query string parameter to check for the fields list
+    # Set to None to ensure that any request fields will not override
+    fields_param = 'fields'
+
+    # Seperator character(s) to split the fields parameter list
+    fields_sep = ','
+
+    # Set to None to flatten lists into one heading per value.
+    # Otherwise, concatenate lists delimiting with the given string.
+    concatenate_lists_sep = ', '
+
+    def flatten_list(self, l):
+        if self.concatenate_lists_sep is None:
+            return super(DynamicFieldsCsvRenderer, self).flatten_list(l)
+        return {'': self.concatenate_lists_sep.join(l)}
+
+    def get_header(self, data, renderer_context):
+        """Return the list of header fields, determined by class settings and context."""
+
+        # Start with the previously-set list of header fields
+        header = renderer_context.get('header', self.header)
+
+        # If no previous set, then determine the candidates from the data
+        if header is None:
+            header = set()
+            data = self.flatten_data(data)
+            for item in data:
+                header.update(list(item.keys()))
+
+            # Alphabetize header fields by default, since
+            # flatten_data() makes field order indeterminate.
+            header = sorted(header)
+
+        # If configured to, examine the query parameters for the requsted header fields
+        request = renderer_context.get('request')
+        if request is not None and self.fields_param is not None:
+
+            request_fields = request.query_params.get(self.fields_param)
+            if request_fields is not None:
+
+                requested = OrderedSet()
+                for request_field in request_fields.split(self.fields_sep):
+
+                    # Only fields in the original candidate header set are valid
+                    if request_field in header:
+                        requested.update((request_field,))
+
+                header = requested  # pylint: disable=redefined-variable-type
+
+        return header
+
+    def render(self, data, media_type=None, renderer_context=None, writer_opts=None):
+        """Override the default "get headers" behaviour, then render the data."""
+        renderer_context = renderer_context or {}
+        self.header = self.get_header(data, renderer_context)
+        return super(DynamicFieldsCsvRenderer, self).render(data, media_type, renderer_context, writer_opts)
+
+
+class PaginatedCsvRenderer(ResultsOnlyRendererMixin, DynamicFieldsCsvRenderer):
+    """
+    Render results-only CSV data with dynamically-determined fields.
+    """
+    media_type = 'text/csv'

analytics_data_api/tests/test_renderers.py

+"""
+Tests for the custom REST framework renderers.
+"""
+from mock import MagicMock, PropertyMock
+from django.test import TestCase
+
+from analytics_data_api.renderers import PaginatedCsvRenderer
+
+
+class PaginatedCsvRendererTests(TestCase):
+
+    def setUp(self):
+        super(PaginatedCsvRendererTests, self).setUp()
+        self.renderer = PaginatedCsvRenderer()
+        self.data = {'results': [
+            {
+                'string': 'ab,c',
+                'list': ['a', 'b', 'c'],
+                'dict': {'a': 1, 'b': 2, 'c': 3},
+            }, {
+                'string': 'def',
+                'string2': 'ghi',
+                'list': ['d', 'e', 'f', 'g'],
+                'dict': {'d': 4, 'b': 5, 'c': 6},
+            },
+        ]}
+        self.context = {}
+
+    def set_request(self, params=None):
+        request = MagicMock()
+        mock_params = PropertyMock(return_value=params)
+        type(request).query_params = mock_params
+        self.context['request'] = request
+
+    def test_csv_media_type(self):
+        self.assertEqual(self.renderer.media_type, 'text/csv')
+
+    def test_render(self):
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'dict.a,dict.b,dict.c,dict.d,list,string,string2\r\n'
+                          '1,2,3,,"a, b, c","ab,c",\r\n'
+                          ',5,6,4,"d, e, f, g",def,ghi\r\n')
+
+    def test_render_fields(self):
+        self.set_request(dict(fields='string2,invalid,dict.b,list,dict.a,string'))
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'string2,dict.b,list,dict.a,string\r\n'
+                          ',2,"a, b, c",1,"ab,c"\r\n'
+                          'ghi,5,"d, e, f, g",,def\r\n')
+
+    def test_render_flatten_lists(self):
+        self.renderer.concatenate_lists_sep = None
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'dict.a,dict.b,dict.c,dict.d,list.0,list.1,list.2,list.3,string,string2\r\n'
+                          '1,2,3,,a,b,c,,"ab,c",\r\n'
+                          ',5,6,4,d,e,f,g,def,ghi\r\n')
+
+    def test_render_fields_flatten_lists(self):
+        self.renderer.concatenate_lists_sep = None
+        self.set_request(dict(fields='string2,invalid,list.2,dict.a,list.1,string'))
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'string2,list.2,dict.a,list.1,string\r\n'
+                          ',c,1,b,"ab,c"\r\n'
+                          'ghi,f,,e,def\r\n')
+
+    def test_render_fields_limit_headers(self):
+        self.renderer.header = ('string2', 'invalid', 'dict.a')
+        self.set_request(dict(fields='string2,invalid,dict.b,list,dict.a,string'))
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'string2,invalid,dict.a\r\n'
+                          ',,1\r\n'
+                          'ghi,,\r\n')

analytics_data_api/v0/serializers.py

@@ -7,7 +7,6 @@ from rest_framework.response import Response
 from analytics_data_api.constants import (
     engagement_events,
     enrollment_modes,
-    learner,
 )
 from analytics_data_api.v0 import models
 
@@ -409,8 +408,8 @@ class EdxPaginationSerializer(pagination.PageNumberPagination):
     Adds values to the response according to edX REST API Conventions.
     """
     page_size_query_param = 'page_size'
-    page_size = learner.LEARNER_API_DEFAULT_LIST_PAGE_SIZE
-    max_page_size = 100  # TODO -- tweak during load testing
+    page_size = getattr(settings, 'DEFAULT_PAGE_SIZE', 25)
+    max_page_size = getattr(settings, 'MAX_PAGE_SIZE', 100)  # TODO -- tweak during load testing
 
     def get_paginated_response(self, data):
         # The output is more readable with num_pages included not at the end, but

analytics_data_api/v0/tests/views/__init__.py

 import json
+import StringIO
+import csv
 
 from opaque_keys.edx.keys import CourseKey
 from rest_framework import status
 
 from analytics_data_api.utils import get_filename_safe_course_id
+from analytics_data_api.v0.tests.utils import flatten
+
 
 DEMO_COURSE_ID = u'course-v1:edX+DemoX+Demo_2014'
 SANITIZED_DEMO_COURSE_ID = get_filename_safe_course_id(DEMO_COURSE_ID)
@@ -39,3 +43,48 @@ class VerifyCourseIdMixin(object):
             u"developer_message": u"Course id/key {} malformed.".format(course_id)
         }
         self.assertDictEqual(json.loads(response.content), expected)
+
+
+class VerifyCsvResponseMixin(object):
+
+    def assertCsvResponseIsValid(self, response, expected_filename, expected_data=None, expected_headers=None):
+
+        # Validate the basic response status, content type, and filename
+        self.assertEquals(response.status_code, 200)
+        if expected_data:
+            self.assertEquals(response['Content-Type'].split(';')[0], 'text/csv')
+        self.assertEquals(response['Content-Disposition'], u'attachment; filename={}'.format(expected_filename))
+
+        # Validate other response headers
+        if expected_headers:
+            for header_name, header_content in expected_headers.iteritems():
+                self.assertEquals(response.get(header_name), header_content)
+
+        # Validate the content data
+        if expected_data:
+            data = map(flatten, expected_data)
+
+            # The CSV renderer sorts the headers alphabetically
+            fieldnames = sorted(data[0].keys())
+
+            # Generate the expected CSV output
+            expected = StringIO.StringIO()
+            writer = csv.DictWriter(expected, fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+            self.assertEqual(response.content, expected.getvalue())
+        else:
+            self.assertEqual(response.content, '')
+
+    def assertResponseFields(self, response, fields):
+        content_type = response.get('Content-Type', '').split(';')[0]
+        self.assertEquals(content_type, 'text/csv')
+
+        data = StringIO.StringIO(response.content)
+        reader = csv.reader(data)
+        rows = []
+        for row in reader:
+            rows.append(row)
+        # Just check the header row
+        self.assertGreater(len(rows), 1)
+        self.assertEqual(rows[0], fields)

analytics_data_api/v0/tests/views/test_courses.py

@@ -3,8 +3,6 @@
 # change for versions greater than 1.0.0. Tests target a specific version of the API, additional tests should be added
 # for subsequent versions if there are breaking changes introduced in those versions.
 
-import StringIO
-import csv
 import datetime
 from itertools import groupby
 import urllib
@@ -18,8 +16,9 @@ from analytics_data_api.constants.country import get_country
 from analytics_data_api.v0 import models
 from analytics_data_api.constants import country, enrollment_modes, genders
 from analytics_data_api.v0.models import CourseActivityWeekly
-from analytics_data_api.v0.tests.utils import flatten
-from analytics_data_api.v0.tests.views import DemoCourseMixin, DEMO_COURSE_ID, SANITIZED_DEMO_COURSE_ID
+from analytics_data_api.v0.tests.views import (
+    DemoCourseMixin, VerifyCsvResponseMixin, DEMO_COURSE_ID, SANITIZED_DEMO_COURSE_ID,
+)
 from analyticsdataserver.tests import TestCaseWithAuthentication
 
 
@@ -38,7 +37,7 @@ class DefaultFillTestMixin(object):
 
 
 # pylint: disable=no-member
-class CourseViewTestCaseMixin(DemoCourseMixin):
+class CourseViewTestCaseMixin(DemoCourseMixin, VerifyCsvResponseMixin):
     model = None
     api_root_path = '/api/v0/'
     path = None
@@ -66,7 +65,8 @@ class CourseViewTestCaseMixin(DemoCourseMixin):
         """
         raise NotImplementedError
 
-    def get_csv_filename(self):
+    @property
+    def csv_filename(self):
         return u'edX-DemoX-Demo_2014--{0}.csv'.format(self.csv_filename_slug)
 
     def test_get_not_found(self):
@@ -93,28 +93,12 @@ class CourseViewTestCaseMixin(DemoCourseMixin):
         csv_content_type = 'text/csv'
         response = self.authenticated_get(path, HTTP_ACCEPT=csv_content_type)
 
-        # Validate the basic response status, content type, and filename
-        self.assertEquals(response.status_code, 200)
-        self.assertEquals(response['Content-Type'].split(';')[0], csv_content_type)
-        self.assertEquals(response['Content-Disposition'], u'attachment; filename={}'.format(filename))
-
-        # Validate the actual data
         data = self.format_as_response(*self.get_latest_data(course_id=course_id))
-        data = map(flatten, data)
-
-        # The CSV renderer sorts the headers alphabetically
-        fieldnames = sorted(data[0].keys())
-
-        # Generate the expected CSV output
-        expected = StringIO.StringIO()
-        writer = csv.DictWriter(expected, fieldnames)
-        writer.writeheader()
-        writer.writerows(data)
-        self.assertEqual(response.content, expected.getvalue())
+        self.assertCsvResponseIsValid(response, filename, data)
 
     def test_get_csv(self):
         """ Verify the endpoint returns data that has been properly converted to CSV. """
-        self.assertCSVIsValid(self.course_id, self.get_csv_filename())
+        self.assertCSVIsValid(self.course_id, self.csv_filename)
 
     def test_get_csv_with_deprecated_key(self):
         """

analytics_data_api/v0/tests/views/test_learners.py

@@ -19,10 +19,15 @@ from django.core import management
 from analyticsdataserver.tests import TestCaseWithAuthentication
 from analytics_data_api.constants import engagement_events
 from analytics_data_api.v0.models import ModuleEngagementMetricRanges
-from analytics_data_api.v0.tests.views import DemoCourseMixin, VerifyCourseIdMixin
+from analytics_data_api.v0.views import CsvViewMixin, PaginatedHeadersMixin
+from analytics_data_api.v0.tests.views import (
+    DemoCourseMixin, VerifyCourseIdMixin, VerifyCsvResponseMixin,
+)
 
 
-class LearnerAPITestMixin(object):
+class LearnerAPITestMixin(CsvViewMixin):
+    filename_slug = 'learners'
+
     """Manages an elasticsearch index for testing the learner API."""
     def setUp(self):
         """Creates the index and defines a mapping."""
@@ -123,6 +128,20 @@ class LearnerAPITestMixin(object):
         )
         self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX)
 
+    def expected_page_url(self, course_id, page, page_size):
+        """
+        Returns a paginated URL for the given parameters.
+        As with PageNumberPagination, if page=1, it's omitted from the query string.
+        """
+        if page is None:
+            return None
+        course_q = urlencode({'course_id': course_id})
+        page_q = '&page={}'.format(page) if page and page > 1 else ''
+        page_size_q = '&page_size={}'.format(page_size) if page_size > 0 else ''
+        return 'http://testserver/api/v0/learners/?{course_q}{page_q}{page_size_q}'.format(
+            course_q=course_q, page_q=page_q, page_size_q=page_size_q,
+        )
+
 
 @ddt.ddt
 class LearnerTests(VerifyCourseIdMixin, LearnerAPITestMixin, TestCaseWithAuthentication):
@@ -427,8 +446,6 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
 
     def test_pagination(self):
         usernames = ['a', 'b', 'c', 'd', 'e']
-        expected_page_url_template = 'http://testserver/api/v0/learners/?' \
-            '{course_query}&page={page}&page_size={page_size}'
         self.create_learners([{'username': username, 'course_id': self.course_id} for username in usernames])
 
         response = self._get(self.course_id, page_size=2)
@@ -437,9 +454,7 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
             {
                 'count': len(usernames),
                 'previous': None,
-                'next': expected_page_url_template.format(
-                    course_query=urlencode({'course_id': self.course_id}), page=2, page_size=2
-                ),
+                'next': self.expected_page_url(self.course_id, page=2, page_size=2),
                 'num_pages': 3
             },
             payload
@@ -451,9 +466,7 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
         self.assertDictContainsSubset(
             {
                 'count': len(usernames),
-                'previous': expected_page_url_template.format(
-                    course_query=urlencode({'course_id': self.course_id}), page=2, page_size=2
-                ),
+                'previous': self.expected_page_url(self.course_id, page=2, page_size=2),
                 'next': None,
                 'num_pages': 3
             },
@@ -469,20 +482,163 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
         ({'course_id': 'edX/DemoX/Demo_Course', 'segments': 'a', 'ignore_segments': 'b'}, 'illegal_parameter_values'),
         ({'course_id': 'edX/DemoX/Demo_Course', 'order_by': 'a_non_existent_field'}, 'illegal_parameter_values'),
         ({'course_id': 'edX/DemoX/Demo_Course', 'sort_order': 'bad_value'}, 'illegal_parameter_values'),
-        ({'course_id': 'edX/DemoX/Demo_Course', 'page': -1}, 'illegal_parameter_values'),
-        ({'course_id': 'edX/DemoX/Demo_Course', 'page': 0}, 'illegal_parameter_values'),
-        ({'course_id': 'edX/DemoX/Demo_Course', 'page': 'bad_value'}, 'illegal_parameter_values'),
-        ({'course_id': 'edX/DemoX/Demo_Course', 'page_size': 'bad_value'}, 'illegal_parameter_values'),
-        ({'course_id': 'edX/DemoX/Demo_Course', 'page_size': 101}, 'illegal_parameter_values'),
+        ({'course_id': 'edX/DemoX/Demo_Course', 'page': -1}, 'Invalid page.', 404),
+        ({'course_id': 'edX/DemoX/Demo_Course', 'page': 0}, 'Invalid page.', 404),
+        ({'course_id': 'edX/DemoX/Demo_Course', 'page': 'bad_value'}, 'Invalid page.', 404),
         ({'course_id': 'edX/DemoX/Demo_Course', 'segments': 'a_non_existent_segment'}, 'illegal_parameter_values'),
         ({'course_id': 'edX/DemoX/Demo_Course', 'ignore_segments': 'a_non_existent_segment'},
          'illegal_parameter_values'),
     )
     @ddt.unpack
-    def test_bad_request(self, parameters, expected_error_code):
+    def test_bad_request(self, parameters, expected_error_code, expected_status_code=400):
         response = self.authenticated_get('/api/v0/learners/', parameters)
-        self.assertEqual(response.status_code, 400)
-        self.assertEqual(json.loads(response.content)['error_code'], expected_error_code)
+        self.assertEqual(response.status_code, expected_status_code)
+        response_json = json.loads(response.content)
+        self.assertEqual(response_json.get('error_code', response_json.get('detail')), expected_error_code)
+
+
+@ddt.ddt
+class LearnerCsvListTests(LearnerAPITestMixin, VerifyCourseIdMixin,
+                          VerifyCsvResponseMixin, TestCaseWithAuthentication):
+    """Tests for the learner list CSV endpoint."""
+    def setUp(self):
+        super(LearnerCsvListTests, self).setUp()
+        self.course_id = 'edX/DemoX/Demo_Course'
+        self.create_update_index('2015-09-28')
+        self.path = '/api/v0/learners/'
+
+    def test_empty_csv(self):
+        """ Verify the endpoint returns data that has been properly converted to CSV. """
+        response = self.authenticated_get(
+            self.path,
+            dict(course_id=self.course_id),
+            True,
+            HTTP_ACCEPT='text/csv'
+        )
+        self.assertCsvResponseIsValid(response, self.get_csv_filename(), [], {'Link': None})
+
+    def test_csv_pagination(self):
+        """ Verify the endpoint returns properly paginated CSV data"""
+
+        # Create learners, using a cohort name with a comma, to test escaping.
+        usernames = ['victor', 'olga', 'gabe', ]
+        commaCohort = 'Lions, Tigers, & Bears'
+        self.create_learners([{'username': username, 'course_id': self.course_id, 'cohort': commaCohort}
+                              for username in usernames])
+
+        # Set last_updated index date
+        last_updated = '2015-09-28'
+        self.create_update_index(last_updated)
+
+        # Render CSV with one learner per page
+        page_size = 1
+        prev_page = None
+
+        for idx, username in enumerate(sorted(usernames)):
+            page = idx + 1
+            response = self.authenticated_get(
+                self.path,
+                dict(course_id=self.course_id, page=page, page_size=page_size),
+                True,
+                HTTP_ACCEPT='text/csv'
+            )
+
+            # Construct expected content data
+            expected_data = [{
+                "username": username,
+                "enrollment_mode": 'honor',
+                "name": username,
+                "email": "{}@example.com".format(username),
+                "account_url": "http://lms-host/{}".format(username),
+                "cohort": commaCohort,
+                "engagements.problems_attempted": 0,
+                "engagements.problems_completed": 0,
+                "engagements.videos_viewed": 0,
+                "engagements.discussion_contributions": 0,
+                "engagements.problem_attempts_per_completed": None,
+                "enrollment_date": '2015-01-28',
+                "last_updated": last_updated,
+                "segments": None,
+                "user_id": None,
+                "language": None,
+                "location": None,
+                "year_of_birth": None,
+                "level_of_education": None,
+                "gender": None,
+                "mailing_address": None,
+                "city": None,
+                "country": None,
+                "goals": None,
+            }]
+
+            # Construct expected links header from pagination data
+            prev_url = self.expected_page_url(self.course_id, prev_page, page_size)
+            next_page = page + 1
+            next_url = None
+            if next_page <= len(usernames):
+                next_url = self.expected_page_url(self.course_id, next_page, page_size)
+            expected_links = PaginatedHeadersMixin.get_paginated_links(dict(next=next_url, previous=prev_url))
+
+            self.assertCsvResponseIsValid(response, self.get_csv_filename(), expected_data, {'Link': expected_links})
+            prev_page = page
+
+    @ddt.data(
+        # fields deliberately out of alphabetical order
+        (['username', 'cohort', 'last_updated', 'email'],
+         ['username', 'cohort', 'last_updated', 'email']),
+        # valid fields interpersed with invalid fields
+        (['foo', 'username', 'bar', 'email', 'name', 'baz'],
+         ['username', 'email', 'name']),
+        # list fields are returned concatenated as one field,
+        # and dict fields are listed separately
+        (['segments', 'username', 'engagements.videos_viewed', 'engagements.problems_attempted'],
+         ['segments', 'username', 'engagements.videos_viewed', 'engagements.problems_attempted']),
+        # empty fields list returns all the fields, in alphabetical order.
+        ([],
+         ['account_url',
+          'city',
+          'cohort',
+          'country',
+          'email',
+          'engagements.discussion_contributions',
+          'engagements.problem_attempts_per_completed',
+          'engagements.problems_attempted',
+          'engagements.problems_completed',
+          'engagements.videos_viewed',
+          'enrollment_date',
+          'enrollment_mode',
+          'gender',
+          'goals',
+          'language',
+          'last_updated',
+          'level_of_education',
+          'location',
+          'mailing_address',
+          'name',
+          'segments',
+          'user_id',
+          'username',
+          'year_of_birth']),
+    )
+    @ddt.unpack
+    def test_csv_fields(self, fields, valid_fields):
+
+        # Create learners, using a cohort name with a comma, to test escaping.
+        usernames = ['victor', 'olga', 'gabe', ]
+        commaCohort = 'Lions, Tigers, & Bears'
+        self.create_learners([{'username': username, 'course_id': self.course_id, 'cohort': commaCohort}
+                              for username in usernames])
+
+        # Render CSV with given fields list
+        response = self.authenticated_get(
+            self.path,
+            dict(course_id=self.course_id, fields=','.join(fields)),
+            True,
+            HTTP_ACCEPT='text/csv'
+        )
+
+        # Check that response contains the valid fields, in the expected order
+        self.assertResponseFields(response, valid_fields)
 
 
 @ddt.ddt

analytics_data_api/v0/views/__init__.py

 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 
+from django.utils import timezone
+from rest_framework.response import Response
 from analytics_data_api.v0.exceptions import (CourseNotSpecifiedError, CourseKeyMalformedError)
 
 
@@ -21,3 +23,81 @@ class CourseViewMixin(object):
         except InvalidKeyError:
             raise CourseKeyMalformedError(course_id=self.course_id)
         return super(CourseViewMixin, self).get(request, *args, **kwargs)
+
+
+class PaginatedHeadersMixin(object):
+    """
+    If the response is paginated, then augment it with this response header:
+
+    * Link: list of next and previous pagination URLs, e.g.
+        <next_url>; rel="next", <previous_url>; rel="prev"
+
+    Format follows the github API convention:
+        https://developer.github.com/guides/traversing-with-pagination/
+
+    Useful with PaginatedCsvRenderer, so that previous/next links aren't lost when returning CSV data.
+
+    """
+    # TODO: When we upgrade to Django REST API v3.1, define a custom DEFAULT_PAGINATION_CLASS
+    # instead of using this mechanism:
+    #   http://www.django-rest-framework.org/api-guide/pagination/#header-based-pagination
+
+    def get(self, request, *args, **kwargs):
+        """
+        Stores pagination links in a response header.
+        """
+        response = super(PaginatedHeadersMixin, self).get(request, args, kwargs)
+        link = self.get_paginated_links(response.data)
+        if link:
+            response['Link'] = link
+        return response
+
+    @staticmethod
+    def get_paginated_links(data):
+        """
+        Returns the links string.
+        """
+        # Un-paginated data is returned as a list, not a dict.
+        next_url = None
+        prev_url = None
+        if isinstance(data, dict):
+            next_url = data.get('next')
+            prev_url = data.get('previous')
+
+        if next_url is not None and prev_url is not None:
+            link = '<{next_url}>; rel="next", <{prev_url}>; rel="prev"'
+        elif next_url is not None:
+            link = '<{next_url}>; rel="next"'
+        elif prev_url is not None:
+            link = '<{prev_url}>; rel="prev"'
+        else:
+            link = ''
+
+        return link.format(next_url=next_url, prev_url=prev_url)
+
+
+class CsvViewMixin(object):
+    """
+    Augments a text/csv response with this header:
+
+    * Content-Disposition: allows the client to download the response as a file attachment.
+    """
+    # Default filename slug for CSV download files
+    filename_slug = 'report'
+
+    def get_csv_filename(self):
+        """
+        Returns the filename for the CSV download.
+        """
+        course_key = CourseKey.from_string(self.course_id)
+        course_id = u'-'.join([course_key.org, course_key.course, course_key.run])
+        now = timezone.now().replace(microsecond=0)
+        return u'{0}--{1}--{2}.csv'.format(course_id, now.isoformat(), self.filename_slug)
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        """
+        Append Content-Disposition header to CSV requests.
+        """
+        if request.META.get('HTTP_ACCEPT') == u'text/csv':
+            response['Content-Disposition'] = u'attachment; filename={}'.format(self.get_csv_filename())
+        return super(CsvViewMixin, self).finalize_response(request, response, *args, **kwargs)

analytics_data_api/v0/views/learners.py

@@ -23,7 +23,7 @@ from analytics_data_api.v0.serializers import (
     LastUpdatedSerializer,
     LearnerSerializer,
 )
-from analytics_data_api.v0.views import CourseViewMixin
+from analytics_data_api.v0.views import CourseViewMixin, PaginatedHeadersMixin, CsvViewMixin
 from analytics_data_api.v0.views.utils import split_query_argument
 
 
@@ -37,7 +37,7 @@ class LastUpdateMixin(object):
         """ Returns the serialized RosterUpdate last_updated field. """
         roster_update = RosterUpdate.get_last_updated()
         last_updated = {'date': None}
-        if len(roster_update) == 1:
+        if len(roster_update) >= 1:
             last_updated = roster_update[0]
         else:
             logger.warn('RosterUpdate not found.')
@@ -120,7 +120,7 @@ class LearnerView(LastUpdateMixin, CourseViewMixin, generics.RetrieveAPIView):
         raise LearnerNotFoundError(username=self.username, course_id=self.course_id)
 
 
-class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
+class LearnerListView(LastUpdateMixin, CourseViewMixin, PaginatedHeadersMixin, CsvViewMixin, generics.ListAPIView):
     """
     Get a paginated list of data for all learners in a course.
 
@@ -131,18 +131,12 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
     **Response Values**
 
         Returns a paginated list of learner metadata and engagement data.
-        Pagination data is returned in the top level of the returned JSON
-        object.
 
-            * count: The number of learners that match the query.
-            * page: The current one-indexed page number.
-            * next: A hyperlink to the next page if one exists, otherwise null.
-            * previous: A hyperlink to the previous page if one exists,
-              otherwise null.
+        Pagination links, if applicable, are returned in the response's header.
+        e.g.
+            Link: <next_url>; rel="next", <previous_url>; rel="prev";
 
-        The 'results' key in the returned object maps to an array of
-        learners that contains, at most, a full page's worth of learners. For
-        each learner there is an object that contains the following keys.
+        Returned results may contain the following fields:
 
             * username: The username of an enrolled learner.
             * enrollment_mode: The learner's selected learning track (for
@@ -159,9 +153,9 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
             * city: The learner's reported city.
             * country: The learner's reported country.
             * goals: The learner's reported goals.
-            * segments: Classification, based on engagement, of each learner's
-              work in this course (for example, "highly_engaged" or
-              "struggling").
+            * segments: list of classifications, based on engagement, of each
+              learner's work in this course (for example, ["highly_engaged"] or
+              ["struggling"]).
             * engagements: Summary of engagement events for a time span.
                 * videos_viewed: Number of times any course video was played.
                 * problems_completed: Number of unique problems the learner
@@ -173,12 +167,56 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
                 * discussions_contributed: Number of posts, responses, or
                   comments the learner contributed to course discussions.
 
+        JSON:
+
+            The default format is JSON, with pagination data in the top level,
+            e.g.:
+            {
+                "count": 123,               // The number of learners that match the query.
+                "page":  2,                 // The current one-indexed page number.
+                "next":  "http://...",      // A hyperlink to the next page
+                                            //  if one exists, otherwise null.
+                "previous": "http://...",   // A hyperlink to the previous page
+                                            //  if one exists, otherwise null.
+                "results": [                // One results object per learner
+                    {
+                        "username": "user1",
+                        "name": "name1",
+                        ...
+                    },
+                    ...
+                ]
+            }
+
+        CSV:
+
+            If the request Accept header is 'text/csv', then the returned
+            results will be in CSV format.  Field names will be on the first
+            line as column headings, with one learner per row, e.g.:
+
+                username,name,email,segments.0,engagements.videos_viewed,...
+                user1,name1,user1@example.com,"highly engaged",0,...
+                user2,name2,user2@example.com,struggling,1,...
+
+            Use the 'fields' parameter to control the list of fields returned,
+            and the order they appear in.
+
+            Fields containing "list" values, like 'segments', are flattened and
+            returned in order, e.g., segments.0,segments.1,segments.2,...
+
+            Fields containing "dict" values, like 'engagements', are flattened
+            and use the fully-qualified field name in the heading, e.g.,
+            engagements.videos_viewed,engagements.problems_completed,...
+
+            Note that pagination data is not included in the main response body;
+            see above for details on pagination links in the response header.
+
     **Parameters**
 
         You can filter the list of learners by course ID and by other
         parameters, including enrollment mode and text search. You can also
-        control the page size and page number of the response, as well as sort
-        the learners in the response.
+        control the page size and page number of the response, the list of
+        returned fields, and sort the learners in the response.
 
         course_id -- The course identifier for which user data is requested.
             For example, edX/DemoX/Demo_Course.
@@ -201,35 +239,13 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
         order_by -- The field for sorting the response. Defaults to 'username'.
         sort_order -- The sort direction.  One of 'asc' (ascending) or 'desc'
             (descending). Defaults to 'asc'.
-
+        fields -- The list of fields, and their sort order, to return when
+            viewing CSV data.  Defaults to the full list of available fields,
+            in alphabetical order.
     """
     serializer_class = LearnerSerializer
     pagination_class = EdxPaginationSerializer
-    max_paginate_by = 100  # TODO -- tweak during load testing
-
-    def _validate_query_params(self):
-        """Validates various querystring parameters."""
-        query_params = self.request.query_params
-        page = query_params.get('page')
-        if page:
-            try:
-                page = int(page)
-            except ValueError:
-                raise ParameterValueError('Page must be an integer')
-            finally:
-                if page < 1:
-                    raise ParameterValueError(
-                        'Page numbers are one-indexed, therefore the page value must be greater than 0'
-                    )
-        page_size = query_params.get('page_size')
-        if page_size:
-            try:
-                page_size = int(page_size)
-            except ValueError:
-                raise ParameterValueError('Page size must be an integer')
-            finally:
-                if page_size > self.max_paginate_by or page_size < 1:
-                    raise ParameterValueError('Page size must be in the range [1, {}]'.format(self.max_paginate_by))
+    filename_slug = 'learners'
 
     def list(self, request, *args, **kwargs):
         """
@@ -247,7 +263,6 @@ class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
         Fetches the user list and last updated from elasticsearch returned returned
         as a an array of dicts with fields "learner" and "last_updated".
         """
-        self._validate_query_params()
         query_params = self.request.query_params
 
         order_by = query_params.get('order_by')

analyticsdataserver/settings/base.py

@@ -284,7 +284,7 @@ REST_FRAMEWORK = {
     'DEFAULT_RENDERER_CLASSES': (
         'rest_framework.renderers.JSONRenderer',
         'rest_framework.renderers.BrowsableAPIRenderer',
-        'rest_framework_csv.renderers.CSVRenderer',
+        'analytics_data_api.renderers.PaginatedCsvRenderer',
     )
 }
 ########## END REST FRAMEWORK CONFIGURATION
@@ -307,6 +307,10 @@ MEDIA_URL = 'http://localhost:8100/static/reports/'
 COURSE_REPORT_FILE_LOCATION_TEMPLATE = '{course_id}_{report_name}.csv'
 ENABLED_REPORT_IDENTIFIERS = ('problem_response',)
 
+# Warning: using 0 or None for these can alter the structure of the REST response.
+DEFAULT_PAGE_SIZE = 25
+MAX_PAGE_SIZE = 100
+
 ########## END ANALYTICS DATA API CONFIGURATION
 
 

requirements/base.txt

@@ -7,6 +7,7 @@ djangorestframework-csv==1.4.1		# BSD
 django-countries==4.0               # MIT
 edx-django-release-util==0.1.0
 elasticsearch-dsl==0.0.11           # Apache 2.0
+ordered-set==2.0.1                  # MIT
 
 # markdown is used by swagger for rendering the api docs
 Markdown==2.6.6    					# BSD