Added last_updated field to learner endpoints.

analytics_data_api/v0/models.py

@@ -6,7 +6,6 @@ from django.db.models import Sum
 # some fields (e.g. Float, Integer) are dynamic and your IDE may highlight them as unavailable
 from elasticsearch_dsl import Date, DocType, Float, Integer, Q, String
 
-
 from analytics_data_api.constants import country, engagement_entity_types, genders, learner
 
 
@@ -215,6 +214,20 @@ class Video(BaseVideo):
         db_table = 'video'
 
 
+class RosterUpdate(DocType):
+
+    date = Date()
+
+    # pylint: disable=old-style-class
+    class Meta:
+        index = settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX
+        doc_type = 'marker'
+
+    @classmethod
+    def get_last_updated(cls):
+        return cls.search().query('term', target_index=settings.ELASTICSEARCH_LEARNERS_INDEX).execute()
+
+
 class RosterEntry(DocType):
 
     course_id = String()

analytics_data_api/v0/serializers.py

@@ -317,17 +317,20 @@ class VideoTimelineSerializer(ModelSerializerWithCreatedField):
         )
 
 
+class LastUpdatedSerializer(serializers.Serializer):
+    last_updated = serializers.DateField(source='date', format=settings.DATE_FORMAT)
+
+
 class LearnerSerializer(serializers.Serializer, DefaultIfNoneMixin):
-    username = serializers.CharField()
-    enrollment_mode = serializers.CharField()
-    name = serializers.CharField()
+    username = serializers.CharField(source='username')
+    enrollment_mode = serializers.CharField(source='enrollment_mode')
+    name = serializers.CharField(source='name')
     account_url = serializers.SerializerMethodField('get_account_url')
-    email = serializers.CharField()
+    email = serializers.CharField(source='email')
     segments = serializers.Field(source='segments')
     engagements = serializers.SerializerMethodField('get_engagements')
-    enrollment_date = serializers.DateField(format=settings.DATE_FORMAT)
-    last_updated = serializers.DateField(format=settings.DATE_FORMAT)
-    cohort = serializers.CharField()
+    enrollment_date = serializers.DateField(source='enrollment_date', format=settings.DATE_FORMAT)
+    cohort = serializers.CharField(source='cohort')
 
     def get_account_url(self, obj):
         if settings.LMS_USER_ACCOUNT_BASE_URL:
@@ -426,20 +429,11 @@ class EnagementRangeMetricSerializer(serializers.Serializer):
 
 
 class CourseLearnerMetadataSerializer(serializers.Serializer):
-    enrollment_modes = serializers.SerializerMethodField('get_enrollment_modes')
-    segments = serializers.SerializerMethodField('get_segments')
-    cohorts = serializers.SerializerMethodField('get_cohorts')
+    enrollment_modes = serializers.Field(source='es_data.enrollment_modes')
+    segments = serializers.Field(source='es_data.segments')
+    cohorts = serializers.Field(source='es_data.cohorts')
     engagement_ranges = serializers.SerializerMethodField('get_engagement_ranges')
 
-    def get_enrollment_modes(self, obj):
-        return obj['es_data']['enrollment_modes']
-
-    def get_segments(self, obj):
-        return obj['es_data']['segments']
-
-    def get_cohorts(self, obj):
-        return obj['es_data']['cohorts']
-
     def get_engagement_ranges(self, obj):
         query_set = obj['engagement_ranges']
         engagement_ranges = {

analytics_data_api/v0/tests/views/test_learners.py

@@ -25,10 +25,15 @@ class LearnerAPITestMixin(object):
         """Creates the index and defines a mapping."""
         super(LearnerAPITestMixin, self).setUp()
         self._es = Elasticsearch([settings.ELASTICSEARCH_LEARNERS_HOST])
-        # delete index if for some reason the index wasn't deleted in tearDown
-        if self._es.indices.exists(index=settings.ELASTICSEARCH_LEARNERS_INDEX):
-            self._es.indices.delete(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
-        self._es.indices.create(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
+
+        for index in [settings.ELASTICSEARCH_LEARNERS_INDEX, settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX]:
+            # ensure the test index is deleted
+            def delete_index(to_delete):
+                if self._es.indices.exists(index=to_delete):
+                    self._es.indices.delete(index=to_delete)
+            self.addCleanup(delete_index, index)
+            self._es.indices.create(index=index)
+
         self._es.indices.put_mapping(
             index=settings.ELASTICSEARCH_LEARNERS_INDEX,
             doc_type='roster_entry',
@@ -76,18 +81,25 @@ class LearnerAPITestMixin(object):
                     'enrollment_date': {
                         'type': 'date', 'doc_values': True
                     },
-                    'last_updated': {
+                }
+            }
+        )
+
+        self._es.indices.put_mapping(
+            index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX,
+            doc_type='marker',
+            body={
+                'properties': {
+                    'date': {
                         'type': 'date', 'doc_values': True
                     },
+                    'target_index': {
+                        'type': 'string'
+                    },
                 }
             }
         )
 
-    def tearDown(self):
-        """Remove the index after every test."""
-        super(LearnerAPITestMixin, self).tearDown()
-        self._es.indices.delete(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
-
     def _create_learner(
             self,
             username,
@@ -104,7 +116,6 @@ class LearnerAPITestMixin(object):
             attempt_ratio_order=0,
             videos_viewed=0,
             enrollment_date='2015-01-28',
-            last_updated='2015-01-28'
     ):
         """Create a single learner roster entry in the elasticsearch index."""
         self._es.create(
@@ -125,7 +136,6 @@ class LearnerAPITestMixin(object):
                 'attempt_ratio_order': attempt_ratio_order,
                 'videos_viewed': videos_viewed,
                 'enrollment_date': enrollment_date,
-                'last_updated': last_updated
             }
         )
 
@@ -141,6 +151,20 @@ class LearnerAPITestMixin(object):
             self._create_learner(**learner)
         self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
 
+    def create_update_index(self, date=None):
+        """
+        Created an index with the date of when the learner index was updated.
+        """
+        self._es.create(
+            index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX,
+            doc_type='marker',
+            body={
+                'date': date,
+                'target_index': settings.ELASTICSEARCH_LEARNERS_INDEX,
+            }
+        )
+        self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX)
+
 
 @ddt.ddt
 class LearnerTests(VerifyCourseIdMixin, LearnerAPITestMixin, TestCaseWithAuthentication):
@@ -172,8 +196,8 @@ class LearnerTests(VerifyCourseIdMixin, LearnerAPITestMixin, TestCaseWithAuthent
             "problem_attempts_per_completed": problem_attempts_per_completed,
             "attempt_ratio_order": attempt_ratio_order,
             "enrollment_date": enrollment_date,
-            "last_updated": last_updated,
         }])
+        self.create_update_index(last_updated)
 
         response = self.authenticated_get(self.path_template.format(username, course_id))
         self.assertEquals(response.status_code, 200)
@@ -227,6 +251,7 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
     def setUp(self):
         super(LearnerListTests, self).setUp()
         self.course_id = 'edX/DemoX/Demo_Course'
+        self.create_update_index('2015-09-28')
 
     def _get(self, course_id, **query_params):
         """Helper to send a GET request to the API."""
@@ -291,7 +316,8 @@ class LearnerListTests(LearnerAPITestMixin, VerifyCourseIdMixin, TestCaseWithAut
                 "videos_viewed": 6,
                 "discussions_contributed": 0,
                 "problem_attempts_per_completed": 23.14,
-            }
+            },
+            'last_updated': '2015-09-28',
         }])
 
     @ddt.data(

analytics_data_api/v0/views/learners.py

 """
 API methods for module level data.
 """
+import logging
+
 from rest_framework import generics, status
 
 from analytics_data_api.constants import (
@@ -14,19 +16,38 @@ from analytics_data_api.v0.exceptions import (
 from analytics_data_api.v0.models import (
     ModuleEngagement,
     ModuleEngagementMetricRanges,
-    RosterEntry
+    RosterEntry,
+    RosterUpdate,
 )
 from analytics_data_api.v0.serializers import (
     CourseLearnerMetadataSerializer,
     ElasticsearchDSLSearchSerializer,
     EngagementDaySerializer,
+    LastUpdatedSerializer,
     LearnerSerializer,
 )
 from analytics_data_api.v0.views import CourseViewMixin
 from analytics_data_api.v0.views.utils import split_query_argument
 
 
-class LearnerView(CourseViewMixin, generics.RetrieveAPIView):
+logger = logging.getLogger(__name__)
+
+
+class LastUpdateMixin(object):
+
+    @classmethod
+    def get_last_updated(cls):
+        """ Returns the serialized RosterUpdate last_updated field. """
+        roster_update = RosterUpdate.get_last_updated()
+        last_updated = {'date': None}
+        if len(roster_update) == 1:
+            last_updated = roster_update[0]
+        else:
+            logger.warn('RosterUpdate not found.')
+        return LastUpdatedSerializer(last_updated).data
+
+
+class LearnerView(LastUpdateMixin, CourseViewMixin, generics.RetrieveAPIView):
     """
     Get data for a particular learner in a particular course.
 
@@ -73,6 +94,14 @@ class LearnerView(CourseViewMixin, generics.RetrieveAPIView):
         self.username = self.kwargs.get('username')
         return super(LearnerView, self).get(request, *args, **kwargs)
 
+    def retrieve(self, request, *args, **kwargs):
+        """
+        Adds the last_updated field to the result.
+        """
+        response = super(LearnerView, self).retrieve(request, args, kwargs)
+        response.data.update(self.get_last_updated())
+        return response
+
     def get_queryset(self):
         return RosterEntry.get_course_user(self.course_id, self.username)
 
@@ -83,7 +112,7 @@ class LearnerView(CourseViewMixin, generics.RetrieveAPIView):
         raise LearnerNotFoundError(username=self.username, course_id=self.course_id)
 
 
-class LearnerListView(CourseViewMixin, generics.ListAPIView):
+class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
     """
     Get a paginated list of data for all learners in a course.
 
@@ -186,11 +215,20 @@ class LearnerListView(CourseViewMixin, generics.ListAPIView):
                 if page_size > self.max_paginate_by or page_size < 1:
                     raise ParameterValueError('Page size must be in the range [1, {}]'.format(self.max_paginate_by))
 
+    def list(self, request, *args, **kwargs):
+        """
+        Adds the last_updated field to the results.
+        """
+        response = super(LearnerListView, self).list(request, args, kwargs)
+        last_updated = self.get_last_updated()
+        for result in response.data['results']:
+            result.update(last_updated)
+        return response
+
     def get_queryset(self):
         """
-        Fetches the user list from elasticsearch.  Note that an
-        elasticsearch_dsl `Search` object is returned, not an actual
-        queryset.
+        Fetches the user list and last updated from elasticsearch returned returned
+        as a an array of dicts with fields "learner" and "last_updated".
         """
         self._validate_query_params()
         query_params = self.request.QUERY_PARAMS

analyticsdataserver/settings/base.py

@@ -54,6 +54,7 @@ DATABASES = {
 ########## ELASTICSEARCH CONFIGURATION
 ELASTICSEARCH_LEARNERS_HOST = environ.get('ELASTICSEARCH_LEARNERS_HOST', None)
 ELASTICSEARCH_LEARNERS_INDEX = environ.get('ELASTICSEARCH_LEARNERS_INDEX', None)
+ELASTICSEARCH_LEARNERS_UPDATE_INDEX = environ.get('ELASTICSEARCH_LEARNERS_UPDATE_INDEX', None)
 ########## END ELASTICSEARCH CONFIGURATION
 
 ########## GENERAL CONFIGURATION

analyticsdataserver/settings/test.py

@@ -25,3 +25,4 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 # Default elasticsearch port when running locally
 ELASTICSEARCH_LEARNERS_HOST = 'http://localhost:9200/'
 ELASTICSEARCH_LEARNERS_INDEX = 'roster_test'
+ELASTICSEARCH_LEARNERS_UPDATE_INDEX = 'index_update_test'