Fill in missing dates for engagement timeline

AN-6960

analytics_data_api/constants/engagement_types.py

@@ -9,6 +9,15 @@ class EngagementType(object):
         - The internal question of whether the metric should be counted in terms
           of the entity type or the raw number of events.
     """
+    # Defines the current canonical set of engagement types used in the Learner
+    # Analytics API.
+    ALL_TYPES = (
+        'problems_attempted',
+        'problems_completed',
+        'videos_viewed',
+        'discussion_contributions',
+    )
+
     def __init__(self, entity_type, event_type):
         """
         Initializes an EngagementType for a particular entity and event type.

analytics_data_api/tests.py

+import datetime
+
 from django.contrib.auth.models import User
 from django.core.management import call_command, CommandError
 from django.test import TestCase
@@ -6,7 +8,7 @@ from rest_framework.authtoken.models import Token
 
 from analytics_data_api.constants.country import get_country, UNKNOWN_COUNTRY
 
-from analytics_data_api.utils import delete_user_auth_token, set_user_auth_token
+from analytics_data_api.utils import date_range, delete_user_auth_token, set_user_auth_token
 
 
 class UtilsTests(TestCase):
@@ -91,3 +93,30 @@ class CountryTests(TestCase):
         # Return unknown country if code is invalid
         self.assertEqual(get_country('A1'), UNKNOWN_COUNTRY)
         self.assertEqual(get_country(None), UNKNOWN_COUNTRY)
+
+
+class DateRangeTests(TestCase):
+    def test_empty_range(self):
+        date = datetime.datetime(2016, 1, 1)
+        self.assertEqual([date for date in date_range(date, date)], [])
+
+    def test_range_exclusive(self):
+        start_date = datetime.datetime(2016, 1, 1)
+        end_date = datetime.datetime(2016, 1, 2)
+        self.assertEqual([date for date in date_range(start_date, end_date)], [start_date])
+
+    def test_delta_goes_past_end_date(self):
+        start_date = datetime.datetime(2016, 1, 1)
+        end_date = datetime.datetime(2016, 1, 3)
+        time_delta = datetime.timedelta(days=5)
+        self.assertEqual([date for date in date_range(start_date, end_date, time_delta)], [start_date])
+
+    def test_general_range(self):
+        start_date = datetime.datetime(2016, 1, 1)
+        end_date = datetime.datetime(2016, 1, 5)
+        self.assertEqual([date for date in date_range(start_date, end_date)], [
+            datetime.datetime(2016, 1, 1),
+            datetime.datetime(2016, 1, 2),
+            datetime.datetime(2016, 1, 3),
+            datetime.datetime(2016, 1, 4),
+        ])

analytics_data_api/utils.py

+import datetime
 from importlib import import_module
 
 from django.db.models import Q
@@ -60,3 +61,26 @@ def load_fully_qualified_definition(definition):
     module_name, class_name = definition.rsplit('.', 1)
     module = import_module(module_name)
     return getattr(module, class_name)
+
+
+def date_range(start_date, end_date, delta=datetime.timedelta(days=1)):
+    """
+    Returns a generator that iterates over the date range [start_date, end_date)
+    (start_date inclusive, end_date exclusive).  Each date in the range is
+    offset from the previous date by a change of `delta`, which defaults
+    to one day.
+
+    Arguments:
+        start_date (datetime.datetime): The start date of the range, inclusive
+        end_date (datetime.datetime): The end date of the range, exclusive
+        delta (datetime.timedelta): The change in time between dates in the
+            range.
+
+    Returns:
+        Generator: A generator which iterates over all dates in the specified
+        range.
+    """
+    cur_date = start_date
+    while cur_date < end_date:
+        yield cur_date
+        cur_date += delta

analytics_data_api/v0/models.py

+import datetime
 from itertools import groupby
 
 from django.conf import settings
@@ -8,6 +9,7 @@ from elasticsearch_dsl import Date, DocType, Float, Integer, Q, String  # pylint
 
 from analytics_data_api.constants import country, genders, learner
 from analytics_data_api.constants.engagement_types import EngagementType
+from analytics_data_api.utils import date_range
 
 
 class CourseActivityWeekly(models.Model):
@@ -392,7 +394,7 @@ class ModuleEngagementTimelineManager(models.Manager):
     Modifies the ModuleEngagement queryset to aggregate engagement data for
     the learner engagement timeline.
     """
-    def get_timelines(self, course_id, username):
+    def get_timeline(self, course_id, username):
         queryset = ModuleEngagement.objects.all().filter(course_id=course_id, username=username) \
             .values('date', 'entity_type', 'event') \
             .annotate(total_count=Sum('count')) \
@@ -418,7 +420,24 @@ class ModuleEngagementTimelineManager(models.Manager):
                 day[engagement_type.name] = day.get(engagement_type.name, 0) + count_delta
             timelines.append(day)
 
-        return timelines
+        # Fill in dates that may be missing, since the result store doesn't
+        # store empty engagement entries.
+        full_timeline = []
+        default_timeline_entry = {engagement_type: 0 for engagement_type in EngagementType.ALL_TYPES}
+        for index, current_date in enumerate(timelines):
+            full_timeline.append(current_date)
+            try:
+                next_date = timelines[index + 1]
+            except IndexError:
+                continue
+            one_day = datetime.timedelta(days=1)
+            if next_date['date'] > current_date['date'] + one_day:
+                full_timeline += [
+                    dict(date=date, **default_timeline_entry)
+                    for date in date_range(current_date['date'] + one_day, next_date['date'])
+                ]
+
+        return full_timeline
 
 
 class ModuleEngagement(models.Model):

analytics_data_api/v0/tests/views/test_engagement_timelines.py

@@ -126,6 +126,13 @@ class EngagementTimelineTests(DemoCourseMixin, VerifyCourseIdMixin, TestCaseWith
                     'videos_viewed': 1
                 },
                 {
+                    'date': '2015-05-27',
+                    'discussion_contributions': 0,
+                    'problems_attempted': 0,
+                    'problems_completed': 0,
+                    'videos_viewed': 0
+                },
+                {
                     'date': '2015-05-28',
                     'discussion_contributions': 0,
                     'problems_attempted': 1,

analytics_data_api/v0/views/learners.py

@@ -317,7 +317,7 @@ class EngagementTimelineView(CourseViewMixin, generics.ListAPIView):
         return super(EngagementTimelineView, self).get(request, *args, **kwargs)
 
     def get_queryset(self):
-        queryset = ModuleEngagement.objects.get_timelines(self.course_id, self.username)
+        queryset = ModuleEngagement.objects.get_timeline(self.course_id, self.username)
         if len(queryset) == 0:
             raise LearnerEngagementTimelineNotFoundError(username=self.username, course_id=self.course_id)
         return queryset