Implement Learner List endpoint

AN-6158

analytics_data_api/constants/learner.py

+LEARNER_API_DEFAULT_LIST_PAGE_SIZE = 25

analytics_data_api/v0/exceptions.py

@@ -50,3 +50,10 @@ class CourseKeyMalformedError(BaseError):
     @property
     def message_template(self):
         return 'Course id/key {course_id} malformed.'
+
+
+class ParameterValueError(BaseError):
+    """Raise if multiple incompatible parameters were provided."""
+    def __init__(self, message, *args, **kwargs):
+        super(ParameterValueError, self).__init__(*args, **kwargs)
+        self.message = message

analytics_data_api/v0/middleware.py

@@ -3,9 +3,10 @@ from django.http.response import JsonResponse
 from rest_framework import status
 
 from analytics_data_api.v0.exceptions import (
-    LearnerNotFoundError,
+    CourseKeyMalformedError,
     CourseNotSpecifiedError,
-    CourseKeyMalformedError
+    ParameterValueError,
+    LearnerNotFoundError,
 )
 
 
@@ -91,3 +92,21 @@ class CourseKeyMalformedErrorMiddleware(BaseProcessErrorMiddleware):
     @property
     def status_code(self):
         return status.HTTP_400_BAD_REQUEST
+
+
+class ParameterValueErrorMiddleware(BaseProcessErrorMiddleware):
+    """
+    Raise 400 if illegal parameter values are provided.
+    """
+
+    @property
+    def error(self):
+        return ParameterValueError
+
+    @property
+    def error_code(self):
+        return 'illegal_parameter_values'
+
+    @property
+    def status_code(self):
+        return status.HTTP_400_BAD_REQUEST

analytics_data_api/v0/models.py

 from django.conf import settings
 from django.db import models
-from elasticsearch_dsl import DocType
+from elasticsearch_dsl import DocType, Q
 
 from analytics_data_api.constants import country, genders
 
@@ -220,3 +220,55 @@ class RosterEntry(DocType):
     def get_course_user(cls, course_id, username):
         return cls.search().query('term', course_id=course_id).query(
             'term', username=username).execute()
+
+    @classmethod
+    def get_users_in_course(
+            cls,
+            course_id,
+            segments=None,
+            ignore_segments=None,
+            # TODO: enable during https://openedx.atlassian.net/browse/AN-6319
+            # cohort=None,
+            enrollment_mode=None,
+            text_search=None,
+            order_by='username',
+            sort_order='asc'
+    ):
+        """
+        Construct a search query for all users in `course_id` and return
+        the Search object.  Raises `ValueError` if both `segments` and
+        `ignore_segments` are provided.
+        """
+        if segments and ignore_segments:
+            raise ValueError('Cannot combine `segments` and `ignore_segments` parameters.')
+
+        search = cls.search()
+        search.query = Q('bool', must=[Q('term', course_id=course_id)])
+
+        # Filtering/Search
+        if segments:
+            search.query.must.append(Q('bool', should=[Q('term', segments=segment) for segment in segments]))
+        elif ignore_segments:
+            for segment in ignore_segments:
+                search = search.query(~Q('term', segments=segment))
+        # TODO: enable during https://openedx.atlassian.net/browse/AN-6319
+        # if cohort:
+        #     search = search.query('term', cohort=cohort)
+        if enrollment_mode:
+            search = search.query('term', enrollment_mode=enrollment_mode)
+        if text_search:
+            search.query.must.append(Q('multi_match', query=text_search, fields=['name', 'username', 'email']))
+
+        # Sorting
+        order_by_options = (
+            'username', 'email', 'discussions_contributed', 'problems_attempted', 'problems_completed', 'videos_viewed'
+        )
+        sort_order_options = ('asc', 'desc')
+        if order_by not in order_by_options:
+            raise ValueError('order_by value must be one of: {}'.format(', '.join(order_by_options)))
+        if sort_order not in sort_order_options:
+            raise ValueError('sort_order value must be one of: {}'.format(', '.join(sort_order_options)))
+        sort_term = order_by if sort_order == 'asc' else '-{}'.format(order_by)
+        search = search.sort(sort_term)
+
+        return search

analytics_data_api/v0/serializers.py

 from urlparse import urljoin
 from django.conf import settings
-from rest_framework import serializers
+from rest_framework import pagination, serializers
 
 from analytics_data_api.constants import (
     engagement_entity_types,
     engagement_events,
     enrollment_modes,
-    genders,)
+    genders,
+)
 from analytics_data_api.v0 import models
 
 
@@ -343,3 +344,24 @@ class LearnerSerializer(serializers.Serializer):
                 metric = '{0}_{1}'.format(entity_type, event)
                 engagements[metric] = getattr(obj, metric, 0)
         return engagements
+
+
+class EdxPaginationSerializer(pagination.PaginationSerializer):
+    """
+    Adds values to the response according to edX REST API Conventions.
+    """
+    count = serializers.Field(source='paginator.count')
+    num_pages = serializers.Field(source='paginator.num_pages')
+
+
+class ElasticsearchDSLSearchSerializer(EdxPaginationSerializer):
+    def __init__(self, *args, **kwargs):
+        """Make sure that the elasticsearch query is executed."""
+        # Because the elasticsearch-dsl search object has a different
+        # API from the queryset object that's expected by the django
+        # Paginator object, we have to manually execute the query.
+        # Note that the `kwargs['instance']` is the Page object, and
+        # `kwargs['instance'].object_list` is actually an
+        # elasticsearch-dsl search object.
+        kwargs['instance'].object_list = kwargs['instance'].object_list.execute()
+        super(ElasticsearchDSLSearchSerializer, self).__init__(*args, **kwargs)

analytics_data_api/v0/urls/__init__.py

@@ -7,7 +7,7 @@ urlpatterns = patterns(
     url(r'^courses/', include('analytics_data_api.v0.urls.courses', namespace='courses')),
     url(r'^problems/', include('analytics_data_api.v0.urls.problems', namespace='problems')),
     url(r'^videos/', include('analytics_data_api.v0.urls.videos', namespace='videos')),
-    url(r'^learners/', include('analytics_data_api.v0.urls.learners', namespace='learners')),
+    url('^learners/', include('analytics_data_api.v0.urls.learners', namespace='learners')),
 
     # pylint: disable=no-value-for-parameter
     url(r'^authenticated/$', RedirectView.as_view(url=reverse_lazy('authenticated')), name='authenticated'),

analytics_data_api/v0/urls/learners.py

@@ -2,13 +2,11 @@ from django.conf.urls import patterns, url
 
 from analytics_data_api.v0.views import learners as views
 
-USERNAME_PATTERN = r'(?P<username>.+)'
-LEARNERS_URLS = [
-    ('', views.LearnerView, 'learner')
-]
 
-urlpatterns = []
+USERNAME_PATTERN = r'(?P<username>.+)'
 
-for path, view, name in LEARNERS_URLS:
-    regex = r'^{0}/$'.format(USERNAME_PATTERN)
-    urlpatterns += patterns('', url(regex, view.as_view(), name=name))
+urlpatterns = patterns(
+    '',
+    url(r'^$', views.LearnerListView.as_view(), name='learners'),
+    url(r'^{}/$'.format(USERNAME_PATTERN), views.LearnerView.as_view(), name='learner'),
+)

analytics_data_api/v0/views/learners.py

@@ -7,9 +7,15 @@ from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 
 from analytics_data_api.v0.exceptions import (
-    CourseNotSpecifiedError, CourseKeyMalformedError, LearnerNotFoundError)
+    CourseKeyMalformedError,
+    CourseNotSpecifiedError,
+    LearnerNotFoundError,
+    ParameterValueError,
+)
+from analytics_data_api.constants import learner
 from analytics_data_api.v0.models import RosterEntry
-from analytics_data_api.v0.serializers import LearnerSerializer
+from analytics_data_api.v0.serializers import ElasticsearchDSLSearchSerializer, LearnerSerializer
+from analytics_data_api.v0.views.utils import split_query_argument
 
 
 class CourseViewMixin(object):
@@ -40,31 +46,22 @@ class LearnerView(CourseViewMixin, generics.RetrieveAPIView):
 
     **Response Values**
 
-        Returns viewing data for each segment of a video.  For each segment,
-        the collection contains the following data.
+        Returns the learner metadata and engagement data:
 
-            * segment: The order of the segment in the video timeline.
-            * num_users: The number of unique users who viewed this segment.
-            * num_views: The number of views for this segment.
-            * created: The date the segment data was computed.
-
-        Returns the user metadata and engagement data:
-
-            * username: User name.
+            * username: User's username.
             * enrollment_mode: Enrollment mode (e.g. "honor).
-            * name: User name.
-            * email: User email.
+            * name: User's full name.
+            * email: User's email.
             * segments: Classification for this course based on engagement, (e.g. "has_potential").
             * engagements: Summary of engagement events for a time span.
                 * videos_viewed: Number of times a video was played.
                 * problems_completed: Unique number of problems completed.
                 * problems_attempted: Unique number of problems attempted.
-                * problem_attempts: Number of attempts of problems.
                 * discussions_contributed: Number of discussions (e.g. forum posts).
 
     **Parameters**
 
-        You can specify course ID for which you want data.
+        You can specify the course ID for which you want data.
 
         course_id -- The course within which user data is requested.
 
@@ -85,3 +82,122 @@ class LearnerView(CourseViewMixin, generics.RetrieveAPIView):
         if len(queryset) == 1:
             return queryset[0]
         raise LearnerNotFoundError(username=self.username, course_id=self.course_id)
+
+
+class LearnerListView(CourseViewMixin, generics.ListAPIView):
+    """
+    Get a paginated list of student data for a particular course.
+
+    **Example Request**
+
+        GET /api/v0/learners/?course_id={course_id}
+
+    **Response Values**
+
+        Returns a paginated list of learner metadata and engagement data.
+        Pagination data is returned in the top-level of the returned JSON
+        object:
+
+            * count: The number of learners matching the query.
+            * page: The current one-indexed page number.
+            * next: A hyperlink to the next page if one exists, otherwise null.
+            * previous: A hyperlink to the previous page if one exists, otherwise null.
+
+        The 'results' key in the returned JSON object maps to an array of
+        learners which contains at most a full page's worth of learners.  Each
+        learner is a JSON object containing the following keys:
+
+            * username: User's username.
+            * enrollment_mode: Enrollment mode (e.g. "honor).
+            * name: User's full name.
+            * email: User's email.
+            * segments: Classification for this course based on engagement, (e.g. "has_potential").
+            * engagements: Summary of engagement events for a time span.
+                * videos_viewed: Number of times a video was played.
+                * problems_completed: Unique number of problems completed.
+                * problems_attempted: Unique number of problems attempted.
+                * discussions_contributed: Number of discussions (e.g. forum posts).
+
+    **Parameters**
+
+        You can filter the list of learners by course ID and other parameters
+        such as enrollment mode and text search.  You can also control the
+        page size and page number of the response, as well as sort the learners
+        in the response.
+
+        course_id -- The course within which user data is requested.
+        page -- The page of results which should be returned.
+        page_size -- The maximum number of results which should be returned per page.
+        text_search -- A string to search over the name, username, and email of learners.
+        segments -- A comma-separated string of segments to which
+            learners should belong.  Semgents are "OR"-ed together.
+            Cannot use in combination with `ignore_segments`
+            argument.
+        ignore_segments -- A comma-separated string of segments to
+            which learners should NOT belong.  Semgents are "OR"-ed
+            together.  Cannot use in combination with `segments`
+            argument.
+        cohort -- The cohort to which all returned learners must
+            belong.
+        enrollment_mode -- The enrollment mode to which all returned
+            learners must belong.
+        order_by -- The field for sorting the response.  Defaults to 'username'.
+        sort_order -- The sort direction.  One of 'asc' or 'desc'.
+            Defaults to 'asc'.
+
+    """
+    serializer_class = LearnerSerializer
+    pagination_serializer_class = ElasticsearchDSLSearchSerializer
+    paginate_by_param = 'page_size'
+    paginate_by = learner.LEARNER_API_DEFAULT_LIST_PAGE_SIZE
+    max_paginate_by = 100  # TODO -- tweak during load testing
+
+    def _validate_query_params(self):
+        """Validates various querystring parameters."""
+        query_params = self.request.QUERY_PARAMS
+        page = query_params.get('page')
+        if page:
+            try:
+                page = int(page)
+            except ValueError:
+                raise ParameterValueError('Page must be an integer')
+            finally:
+                if page < 1:
+                    raise ParameterValueError(
+                        'Page numbers are one-indexed, therefore the page value must be greater than 0'
+                    )
+        page_size = query_params.get('page_size')
+        if page_size:
+            try:
+                page_size = int(page_size)
+            except ValueError:
+                raise ParameterValueError('Page size must be an integer')
+            finally:
+                if page_size > self.max_paginate_by or page_size < 1:
+                    raise ParameterValueError('Page size must be in the range [1, {}]'.format(self.max_paginate_by))
+
+    def get_queryset(self):
+        """
+        Fetches the user list from elasticsearch.  Note that an
+        elasticsearch_dsl `Search` object is returned, not an actual
+        queryset.
+        """
+        self._validate_query_params()
+        query_params = self.request.QUERY_PARAMS
+        params = {
+            'segments': split_query_argument(query_params.get('segments')),
+            'ignore_segments': split_query_argument(query_params.get('ignore_segments')),
+            # TODO: enable during https://openedx.atlassian.net/browse/AN-6319
+            # 'cohort': query_params.get('cohort'),
+            'enrollment_mode': query_params.get('enrollment_mode'),
+            'text_search': query_params.get('text_search'),
+            'order_by': query_params.get('order_by'),
+            'sort_order': query_params.get('sort_order')
+        }
+        # Remove None values from `params` so that we don't overwrite default
+        # parameter values in `get_users_in_course`.
+        params = {key: val for key, val in params.items() if val is not None}
+        try:
+            return RosterEntry.get_users_in_course(self.course_id, **params)
+        except ValueError as e:
+            raise ParameterValueError(e.message)

analytics_data_api/v0/views/utils.py

+"""Utilities for view-level API logic."""
+
+
+def split_query_argument(argument):
+    """
+    Splits a comma-separated querystring argument into a list.
+    Returns None if the argument is empty.
+    """
+    if argument:
+        return argument.split(',')
+    else:
+        return None

analyticsdataserver/settings/base.py

@@ -56,7 +56,6 @@ ELASTICSEARCH_LEARNERS_HOST = environ.get('ELASTICSEARCH_LEARNERS_HOST', None)
 ELASTICSEARCH_LEARNERS_INDEX = environ.get('ELASTICSEARCH_LEARNERS_INDEX', None)
 ########## END ELASTICSEARCH CONFIGURATION
 
-
 ########## GENERAL CONFIGURATION
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone
 TIME_ZONE = 'UTC'
@@ -168,6 +167,7 @@ MIDDLEWARE_CLASSES = (
     'analytics_data_api.v0.middleware.LearnerNotFoundErrorMiddleware',
     'analytics_data_api.v0.middleware.CourseNotSpecifiedErrorMiddleware',
     'analytics_data_api.v0.middleware.CourseKeyMalformedErrorMiddleware',
+    'analytics_data_api.v0.middleware.ParameterValueErrorMiddleware',
 )
 ########## END MIDDLEWARE CONFIGURATION
 

requirements/test.txt

 # Test dependencies go here.
 -r base.txt
 coverage==3.7.1
+ddt==1.0.1
 diff-cover >= 0.2.1
 django-dynamic-fixture==1.8.1
 django-nose==1.4.1