initial changes

added tests

added users_completed

added users_not_started

lms/djangoapps/api_manager/courses/tests.py

@@ -3,12 +3,14 @@
 Run these tests @ Devstack:
     rake fasttest_lms[common/djangoapps/api_manager/courses/tests.py]
 """
-from datetime import datetime, timedelta
+from datetime import datetime
 import json
 import uuid
 import mock
 from random import randint
 from urllib import urlencode
+from freezegun import freeze_time
+from dateutil.relativedelta import relativedelta
 
 from django.contrib.auth.models import Group
 from django.core.cache import cache
@@ -1968,6 +1970,144 @@ class CoursesApiTests(ModuleStoreTestCase):
         response = self.do_get(course_metrics_uri)
         self.assertEqual(response.status_code, 404)
 
+    @mock.patch.dict("django.conf.settings.FEATURES", {'MARK_PROGRESS_ON_GRADING_EVENT': True,
+                                                       'SIGNAL_ON_SCORE_CHANGED': True,
+                                                       'STUDENT_GRADEBOOK': True,
+                                                       'STUDENT_PROGRESS': True})
+    def test_courses_data_time_series_metrics(self):
+        course = CourseFactory.create(
+            number='3033',
+            name='metrics_in_timeseries',
+            start=datetime(2014, 9, 16, 14, 30),
+            end=datetime(2015, 1, 16)
+        )
+
+        chapter = ItemFactory.create(
+            category="chapter",
+            parent_location=course.location,
+            data=self.test_data,
+            due=datetime(2015, 5, 16, 14, 30),
+            display_name="Overview"
+        )
+
+        sub_section = ItemFactory.create(
+            parent_location=chapter.location,
+            category="sequential",
+            display_name=u"test subsection",
+        )
+        unit = ItemFactory.create(
+            parent_location=sub_section.location,
+            category="vertical",
+            metadata={'graded': True, 'format': 'Homework'},
+            display_name=u"test unit",
+        )
+
+        item = ItemFactory.create(
+            parent_location=unit.location,
+            category='problem',
+            data=StringResponseXMLFactory().build_xml(answer='bar'),
+            display_name='Problem to test timeseries',
+            metadata={'rerandomize': 'always', 'graded': True, 'format': 'Midterm Exam'}
+        )
+
+        # create 10 users
+        USER_COUNT = 25
+        users = [UserFactory.create(username="testuser_tstest" + str(__), profile='test') for __ in xrange(USER_COUNT)]
+
+        # enroll users with time set to 28 days ago
+        enrolled_time = timezone.now() + relativedelta(days=-28)
+        with freeze_time(enrolled_time):
+            for user in users:
+                CourseEnrollmentFactory.create(user=user, course_id=course.id)
+
+        # Mark users as those who have started course
+        for j, user in enumerate(users):
+            complete_time = timezone.now() + relativedelta(days=-(USER_COUNT - j))
+            with freeze_time(complete_time):
+                points_scored = .25
+                points_possible = 1
+                module = self.get_module_for_user(user, course, item)
+                grade_dict = {'value': points_scored, 'max_value': points_possible, 'user_id': user.id}
+                module.system.publish(module, 'grade', grade_dict)
+
+                # Last 2 users as those who have completed
+                if j >= USER_COUNT - 2:
+                    try:
+                        sg_entry = StudentGradebook.objects.get(user=user, course_id=course.id)
+                        sg_entry.grade = 0.9
+                        sg_entry.proforma_grade = 0.91
+                        sg_entry.save()
+                    except StudentGradebook.DoesNotExist:
+                        StudentGradebook.objects.create(user=user, course_id=course.id, grade=0.9,
+                                                        proforma_grade=0.91)
+
+        test_course_id = unicode(course.id)
+        # get course metrics in time series format
+        end_date = datetime.now().date()
+        start_date = end_date + relativedelta(days=-4)
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}'.format(self.base_courses_uri,
+                                                                                           test_course_id,
+                                                                                           start_date,
+                                                                                           end_date)
+
+        response = self.do_get(course_metrics_uri)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data['users_not_started']), 5)
+        total_not_started = sum([not_started[1] for not_started in response.data['users_not_started']])
+        self.assertEqual(total_not_started, 6)
+        self.assertEqual(len(response.data['users_started']), 5)
+        total_started = sum([started[1] for started in response.data['users_started']])
+        self.assertEqual(total_started, 4)
+        self.assertEqual(len(response.data['users_completed']), 5)
+        total_completed = sum([completed[1] for completed in response.data['users_completed']])
+        self.assertEqual(total_completed, 2)
+
+        # metrics with weeks as interval
+        start_date = end_date + relativedelta(days=-10)
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}&' \
+                             'interval=weeks'.format(self.base_courses_uri,
+                                                     test_course_id,
+                                                     start_date,
+                                                     end_date)
+
+        response = self.do_get(course_metrics_uri)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data['users_not_started']), 2)
+        self.assertEqual(len(response.data['users_started']), 2)
+        self.assertEqual(len(response.data['users_completed']), 2)
+
+        # metrics with months as interval
+        start_date = end_date + relativedelta(months=-3)
+        end_date = datetime.now().date() + relativedelta(months=1)
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}&' \
+                             'interval=months'.format(self.base_courses_uri,
+                                                      test_course_id,
+                                                      start_date,
+                                                      end_date)
+
+        response = self.do_get(course_metrics_uri)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data['users_not_started']), 5)
+        self.assertEqual(len(response.data['users_started']), 5)
+        self.assertEqual(len(response.data['users_completed']), 5)
+
+        # test without end_date
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}'.format(self.base_courses_uri,
+                                                                                           test_course_id,
+                                                                                           start_date)
+        response = self.do_get(course_metrics_uri)
+        self.assertEqual(response.status_code, 400)
+
+        # test with unsupported interval
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}&interval=hours'\
+            .format(self.base_courses_uri,
+                    test_course_id,
+                    start_date,
+                    end_date)
+
+        response = self.do_get(course_metrics_uri)
+        self.assertEqual(response.status_code, 400)
+
     def test_course_workgroups_list(self):
         projects_uri = self.base_projects_uri
         data = {

lms/djangoapps/api_manager/courses/urls.py

@@ -24,6 +24,7 @@ urlpatterns = patterns(
     url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/completions/*$', courses_views.CourseModuleCompletionList.as_view(), name='completion-list'),
     url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/projects/*$', courses_views.CoursesProjectList.as_view(), name='courseproject-list'),
     url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/metrics/*$', courses_views.CoursesMetrics.as_view(), name='course-metrics'),
+    url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/time-series-metrics/*$', courses_views.CoursesTimeSeriesMetrics.as_view(), name='course-time-series-metrics'),
     url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/metrics/cities/$', courses_views.CoursesMetricsCities.as_view(), name='courses-cities-metrics'),
     url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/metrics/completions/leaders/*$', courses_views.CoursesMetricsCompletionsLeadersList.as_view(), name='course-metrics-completions-leaders'),
     url(r'^(?P<course_id>[a-zA-Z0-9_+\/:-]+)/metrics/grades/*$', courses_views.CoursesMetricsGradesList.as_view()),

lms/djangoapps/api_manager/courses/views.py

@@ -13,7 +13,7 @@ from django.db.models import Avg, Count, Max, Min
 from django.http import Http404
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from django.db.models import Q
+from django.db.models import Q, F
 
 from rest_framework import status
 from rest_framework.response import Response
@@ -36,7 +36,7 @@ from api_manager.models import CourseGroupRelationship, CourseContentGroupRelati
     CourseModuleCompletion
 from api_manager.permissions import SecureAPIView, SecureListAPIView
 from api_manager.users.serializers import UserSerializer, UserCountByCitySerializer
-from api_manager.utils import generate_base_uri, str2bool
+from api_manager.utils import generate_base_uri, str2bool, get_time_series_data, parse_datetime
 from projects.models import Project, Workgroup
 from projects.serializers import ProjectSerializer, BasicWorkgroupSerializer
 from .serializers import CourseModuleCompletionSerializer, CourseSerializer
@@ -1539,6 +1539,80 @@ class CoursesMetrics(SecureAPIView):
         data.update(thread_stats)
         return Response(data, status=status.HTTP_200_OK)
 
+class CoursesTimeSeriesMetrics(SecureAPIView):
+    """
+    ### The CoursesTimeSeriesMetrics view allows clients to retrieve a list of Metrics for the specified Course
+    in time series format.
+    - URI: ```/api/courses/{course_id}/time-series-metrics/?start_date={date}&end_date={date}&interval={interval}&organization={organization_id}```
+    - interval can be `days`, `weeks` or `months`
+    - GET: Returns a JSON representation with three metrics
+    {
+        "users_not_started": [[datetime-1, count-1], [datetime-2, count-2], ........ [datetime-n, count-n]],
+        "users_started": [[datetime-1, count-1], [datetime-2, count-2], ........ [datetime-n, count-n]],
+        "users_completed": [[datetime-1, count-1], [datetime-2, count-2], ........ [datetime-n, count-n]]
+    }
+    - metrics can be filtered by organization by adding organization parameter to GET request
+    ### Use Cases/Notes:
+    * Example: Display number of users completed, started or not started in a given course for a given time period
+    """
+
+    def get(self, request, course_id):  # pylint: disable=W0613
+        """
+        GET /api/courses/{course_id}/time-series-metrics/
+        """
+        if not course_exists(request, request.user, course_id):
+            return Response({}, status=status.HTTP_404_NOT_FOUND)
+
+        start = request.QUERY_PARAMS.get('start_date', None)
+        end = request.QUERY_PARAMS.get('end_date', None)
+        interval = request.QUERY_PARAMS.get('interval', 'days')
+        if not start or not end:
+            return Response({"message": _("Both start_date and end_date parameters are required")}
+                            , status=status.HTTP_400_BAD_REQUEST)
+        if interval not in ['days', 'weeks', 'months']:
+            return Response({"message": _("Interval parameter is not valid. It should be one of these "
+                                          "'days', 'weeks', 'months'")}, status=status.HTTP_400_BAD_REQUEST)
+        start_dt = parse_datetime(start)
+        end_dt = parse_datetime(end)
+        course_key = get_course_key(course_id)
+        exclude_users = _get_aggregate_exclusion_user_ids(course_key)
+        grade_complete_match_range = getattr(settings, 'GRADEBOOK_GRADE_COMPLETE_PROFORMA_MATCH_RANGE', 0.01)
+        grades_qs = StudentGradebook.objects.filter(course_id__exact=course_key, user__is_active=True).\
+            exclude(user_id__in=exclude_users)
+        grades_complete_qs = grades_qs.filter(proforma_grade__lte=F('grade') + grade_complete_match_range,
+                                              proforma_grade__gt=0)
+        enrolled_qs = CourseEnrollment.objects.filter(course_id__exact=course_key, user__is_active=True)\
+            .exclude(id__in=exclude_users)
+        users_started_qs = StudentProgress.objects.filter(course_id__exact=course_key, user__is_active=True)\
+            .exclude(user_id__in=exclude_users)
+        organization = request.QUERY_PARAMS.get('organization', None)
+        if organization:
+            enrolled_qs = enrolled_qs.filter(user__organizations=organization)
+            grades_complete_qs = grades_complete_qs.filter(user__organizations=organization)
+            users_started_qs = users_started_qs.filter(user__organizations=organization)
+
+        total_enrolled = enrolled_qs.filter(created__lt=start_dt).count()
+        total_started = users_started_qs.filter(created__lt=start_dt).count()
+        enrolled_series = get_time_series_data(enrolled_qs, start_dt, end_dt, interval=interval,
+                                               date_field='created', aggregate=Count('id'))
+
+        started_series = get_time_series_data(users_started_qs, start_dt, end_dt, interval=interval,
+                                              date_field='created', aggregate=Count('id'))
+        completed_series = get_time_series_data(grades_complete_qs, start_dt, end_dt, interval=interval,
+                                                date_field='modified', aggregate=Count('id'))
+        not_started_series = []
+        for enrolled, started in zip(enrolled_series, started_series):
+            not_started_series.append((started[0], (total_enrolled + enrolled[1]) - (total_started + started[1])))
+            total_started += started[1]
+            total_enrolled += enrolled[1]
+
+        data = {
+            'users_not_started': not_started_series,
+            'users_started': started_series,
+            'users_completed': completed_series
+        }
+        return Response(data, status=status.HTTP_200_OK)
+
 
 class CoursesMetricsGradesLeadersList(SecureListAPIView):
     """

lms/djangoapps/api_manager/utils.py

@@ -3,6 +3,11 @@
 import socket
 import struct
 import json
+import datetime
+from django.utils.timezone import now
+from dateutil.parser import parse
+from dateutil.relativedelta import relativedelta, MO
+from django.conf import settings
 
 
 def address_exists_in_network(ip_address, net_n_bits):
@@ -87,3 +92,90 @@ def extract_data_params(request):
         if key.startswith('data__'):
             data_params.append({key[6:]: val})
     return data_params
+
+
+def strip_time(dt):
+    """
+    Removes time part of datetime
+    """
+    tzinfo = getattr(dt, 'tzinfo', now().tzinfo) or now().tzinfo
+    return datetime.datetime(dt.year, dt.month, dt.day, tzinfo=tzinfo)
+
+
+def parse_datetime(date_val, defaultdt=None):
+    """
+    Parses datetime value from string
+    """
+    if isinstance(date_val, basestring):
+        return parse(date_val, yearfirst=True, default=defaultdt)
+    return date_val
+
+
+def get_interval_bounds(date_val, interval):
+    """
+    Returns interval bounds the datetime is in.
+    """
+
+    day = strip_time(date_val)
+
+    if interval == 'day':
+        begin = day
+        end = day + relativedelta(days=1)
+    elif interval == 'week':
+        begin = day - relativedelta(weekday=MO(-1))
+        end = begin + datetime.timedelta(days=7)
+    elif interval == 'month':
+        begin = strip_time(datetime.datetime(date_val.year, date_val.month, 1, tzinfo=date_val.tzinfo))
+        end = begin + relativedelta(months=1)
+    end = end - relativedelta(microseconds=1)
+    return begin, end
+
+
+def detect_db_engine():
+    """
+    detects database engine used
+    """
+    engine = 'mysql'
+    backend = settings.DATABASES['default']['ENGINE']
+    if 'sqlite' in backend:
+        engine = 'sqlite'
+    return engine
+
+
+def get_time_series_data(queryset, start, end, interval='days', date_field='created', aggregate=None):
+    """
+    Aggregate over time intervals to compute time series representation of data
+    """
+    engine = detect_db_engine()
+    start, _ = get_interval_bounds(start, interval.rstrip('s'))
+    _, end = get_interval_bounds(end, interval.rstrip('s'))
+
+    sql = {
+        'mysql': {
+            'days': "DATE_FORMAT(`{}`, '%%Y-%%m-%%d')".format(date_field),
+            'weeks': "DATE_FORMAT(DATE_SUB(`{}`, INTERVAL(WEEKDAY(`{}`)) DAY), '%%Y-%%m-%%d')".\
+                format(date_field, date_field),
+            'months': "DATE_FORMAT(`{}`, '%%Y-%%m-01')".format(date_field)
+        },
+        'sqlite': {
+            'days': "strftime('%%Y-%%m-%%d', `{}`)".format(date_field),
+            'weeks': "strftime('%%Y-%%m-%%d', julianday(`{}`) - strftime('%%w', `{}`) + 1)".format(date_field,
+                                                                                                   date_field),
+            'months': "strftime('%%Y-%%m-01', `{}`)".format(date_field)
+        }
+    }
+    interval_sql = sql[engine][interval]
+    kwargs = {'{}__range'.format(date_field): (start, end)}
+    aggregate_data = queryset.extra(select={'d': interval_sql}).filter(**kwargs).order_by().values('d').\
+        annotate(agg=aggregate)
+
+    today = strip_time(now())
+    data = dict((strip_time(parse_datetime(item['d'], today)), item['agg']) for item in aggregate_data)
+
+    series = []
+    dt_key = start
+    while dt_key < end:
+        value = data.get(dt_key, 0)
+        series.append((dt_key, value,))
+        dt_key += relativedelta(**{interval: 1})
+    return series