Merge pull request #272 from edx-solutions/ziafazal/api-course-time-series-metrics-am

added modules_completed metric

lms/djangoapps/api_manager/courses/tests.py

@@ -2014,9 +2014,28 @@ class CoursesApiTests(TestCase):
             metadata={'rerandomize': 'always', 'graded': True, 'format': 'Midterm Exam'}
         )
 
+        item2 = ItemFactory.create(
+            parent_location=unit.location,
+            category='problem 2',
+            data=StringResponseXMLFactory().build_xml(answer='bar'),
+            display_name='Problem 2 for test timeseries',
+            metadata={'rerandomize': 'always', 'graded': True, 'format': 'Final Exam'}
+        )
+
         # create 10 users
         USER_COUNT = 25
         users = [UserFactory.create(username="testuser_tstest" + str(__), profile='test') for __ in xrange(USER_COUNT)]
+        user_ids = [user.id for user in users]
+
+        #create an organization
+        data = {
+            'name': 'Test Organization',
+            'display_name': 'Test Org Display Name',
+            'users': user_ids
+        }
+        response = self.do_post(self.base_organizations_uri, data)
+        self.assertEqual(response.status_code, 201)
+        org_id = response.data['id']
 
         # enroll users with time set to 28 days ago
         enrolled_time = timezone.now() + relativedelta(days=-28)
@@ -2024,14 +2043,16 @@ class CoursesApiTests(TestCase):
             for user in users:
                 CourseEnrollmentFactory.create(user=user, course_id=course.id)
 
+        points_scored = .25
+        points_possible = 1
+        grade_dict = {'value': points_scored, 'max_value': points_possible}
+
         # Mark users as those who have started course
         for j, user in enumerate(users):
             complete_time = timezone.now() + relativedelta(days=-(USER_COUNT - j))
             with freeze_time(complete_time):
-                points_scored = .25
-                points_possible = 1
                 module = self.get_module_for_user(user, course, item)
-                grade_dict = {'value': points_scored, 'max_value': points_possible, 'user_id': user.id}
+                grade_dict['user_id'] = user.id
                 module.system.publish(module, 'grade', grade_dict)
 
                 # Last 2 users as those who have completed
@@ -2045,14 +2066,24 @@ class CoursesApiTests(TestCase):
                         StudentGradebook.objects.create(user=user, course_id=course.id, grade=0.9,
                                                         proforma_grade=0.91)
 
+        # make more completions
+        for j, user in enumerate(users[:5]):
+            complete_time = timezone.now() + relativedelta(days=-(USER_COUNT - j))
+            with freeze_time(complete_time):
+                module = self.get_module_for_user(user, course, item2)
+                grade_dict['user_id'] = user.id
+                module.system.publish(module, 'grade', grade_dict)
+
         test_course_id = unicode(course.id)
         # get course metrics in time series format
         end_date = datetime.now().date()
         start_date = end_date + relativedelta(days=-4)
-        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}'.format(self.base_courses_uri,
-                                                                                           test_course_id,
-                                                                                           start_date,
-                                                                                           end_date)
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}&organization={}'\
+            .format(self.base_courses_uri,
+                    test_course_id,
+                    start_date,
+                    end_date,
+                    org_id)
 
         response = self.do_get(course_metrics_uri)
         self.assertEqual(response.status_code, 200)
@@ -2065,8 +2096,26 @@ class CoursesApiTests(TestCase):
         self.assertEqual(len(response.data['users_completed']), 5)
         total_completed = sum([completed[1] for completed in response.data['users_completed']])
         self.assertEqual(total_completed, 2)
+        self.assertEqual(len(response.data['modules_completed']), 5)
+        total_modules_completed = sum([completed[1] for completed in response.data['modules_completed']])
+        self.assertEqual(total_modules_completed, 4)
+
+        # get modules completed for first 5 days
+        start_date = datetime.now().date() + relativedelta(days=-USER_COUNT)
+        end_date = datetime.now().date() + relativedelta(days=-(USER_COUNT - 4))
+        course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}'.format(self.base_courses_uri,
+                                                                                           test_course_id,
+                                                                                           start_date,
+                                                                                           end_date)
+
+        response = self.do_get(course_metrics_uri)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data['modules_completed']), 5)
+        total_modules_completed = sum([completed[1] for completed in response.data['modules_completed']])
+        self.assertEqual(total_modules_completed, 10)
 
         # metrics with weeks as interval
+        end_date = datetime.now().date()
         start_date = end_date + relativedelta(days=-10)
         course_metrics_uri = '{}/{}/time-series-metrics/?start_date={}&end_date={}&' \
                              'interval=weeks'.format(self.base_courses_uri,

lms/djangoapps/api_manager/courses/views.py

@@ -23,7 +23,7 @@ from courseware.models import StudentModule
 from courseware.views import get_static_tab_contents
 from django_comment_common.models import FORUM_ROLE_MODERATOR
 from gradebook.models import StudentGradebook
-from progress.models import StudentProgress
+from progress.models import StudentProgress, StudentProgressHistory
 from instructor.access import revoke_access, update_forum_role
 from student.models import CourseEnrollment, CourseEnrollmentAllowed
 from student.roles import CourseRole, CourseAccessRole, CourseInstructorRole, CourseStaffRole, CourseObserverRole, CourseAssistantRole, UserBasedRole
@@ -1614,6 +1614,90 @@ class CoursesTimeSeriesMetrics(SecureAPIView):
         return Response(data, status=status.HTTP_200_OK)
 
 
+class CoursesTimeSeriesMetrics(SecureAPIView):
+    """
+    ### The CoursesTimeSeriesMetrics view allows clients to retrieve a list of Metrics for the specified Course
+    in time series format.
+    - URI: ```/api/courses/{course_id}/time-series-metrics/?start_date={date}&end_date={date}&interval={interval}&organization={organization_id}```
+    - interval can be `days`, `weeks` or `months`
+    - GET: Returns a JSON representation with three metrics
+    {
+        "users_not_started": [[datetime-1, count-1], [datetime-2, count-2], ........ [datetime-n, count-n]],
+        "users_started": [[datetime-1, count-1], [datetime-2, count-2], ........ [datetime-n, count-n]],
+        "users_completed": [[datetime-1, count-1], [datetime-2, count-2], ........ [datetime-n, count-n]]
+    }
+    - metrics can be filtered by organization by adding organization parameter to GET request
+    ### Use Cases/Notes:
+    * Example: Display number of users completed, started or not started in a given course for a given time period
+    """
+
+    def get(self, request, course_id):  # pylint: disable=W0613
+        """
+        GET /api/courses/{course_id}/time-series-metrics/
+        """
+        if not course_exists(request, request.user, course_id):
+            return Response({}, status=status.HTTP_404_NOT_FOUND)
+
+        start = request.QUERY_PARAMS.get('start_date', None)
+        end = request.QUERY_PARAMS.get('end_date', None)
+        interval = request.QUERY_PARAMS.get('interval', 'days')
+        if not start or not end:
+            return Response({"message": _("Both start_date and end_date parameters are required")}
+                            , status=status.HTTP_400_BAD_REQUEST)
+        if interval not in ['days', 'weeks', 'months']:
+            return Response({"message": _("Interval parameter is not valid. It should be one of these "
+                                          "'days', 'weeks', 'months'")}, status=status.HTTP_400_BAD_REQUEST)
+        start_dt = parse_datetime(start)
+        end_dt = parse_datetime(end)
+        course_key = get_course_key(course_id)
+        exclude_users = _get_aggregate_exclusion_user_ids(course_key)
+        grade_complete_match_range = getattr(settings, 'GRADEBOOK_GRADE_COMPLETE_PROFORMA_MATCH_RANGE', 0.01)
+        grades_qs = StudentGradebook.objects.filter(course_id__exact=course_key, user__is_active=True).\
+            exclude(user_id__in=exclude_users)
+        grades_complete_qs = grades_qs.filter(proforma_grade__lte=F('grade') + grade_complete_match_range,
+                                              proforma_grade__gt=0)
+        enrolled_qs = CourseEnrollment.objects.filter(course_id__exact=course_key, user__is_active=True)\
+            .exclude(id__in=exclude_users)
+        users_started_qs = StudentProgressHistory.objects.filter(course_id__exact=course_key, user__is_active=True)\
+            .exclude(user_id__in=exclude_users)
+        modules_completed_qs = CourseModuleCompletion.get_actual_completions().filter(course_id__exact=course_key,
+                                                                                      user__is_active=True)\
+            .exclude(id__in=exclude_users)
+        organization = request.QUERY_PARAMS.get('organization', None)
+        if organization:
+            enrolled_qs = enrolled_qs.filter(user__organizations=organization)
+            grades_complete_qs = grades_complete_qs.filter(user__organizations=organization)
+            users_started_qs = users_started_qs.filter(user__organizations=organization)
+            modules_completed_qs = modules_completed_qs.filter(user__organizations=organization)
+
+        total_enrolled = enrolled_qs.filter(created__lt=start_dt).count()
+        total_started_count = users_started_qs.filter(created__lt=start_dt).aggregate(Count('user', distinct=True))
+        total_started = total_started_count['user__count'] or 0
+        enrolled_series = get_time_series_data(enrolled_qs, start_dt, end_dt, interval=interval,
+                                               date_field='created', aggregate=Count('id'))
+
+        started_series = get_time_series_data(users_started_qs, start_dt, end_dt, interval=interval,
+                                              date_field='created', aggregate=Count('user', distinct=True))
+        completed_series = get_time_series_data(grades_complete_qs, start_dt, end_dt, interval=interval,
+                                                date_field='modified', aggregate=Count('id'))
+        modules_completed_series = get_time_series_data(modules_completed_qs, start_dt, end_dt, interval=interval,
+                                                        date_field='created', aggregate=Count('id'))
+
+        not_started_series = []
+        for enrolled, started in zip(enrolled_series, started_series):
+            not_started_series.append((started[0], (total_enrolled + enrolled[1]) - (total_started + started[1])))
+            total_started += started[1]
+            total_enrolled += enrolled[1]
+
+        data = {
+            'users_not_started': not_started_series,
+            'users_started': started_series,
+            'users_completed': completed_series,
+            'modules_completed': modules_completed_series
+        }
+        return Response(data, status=status.HTTP_200_OK)
+
+
 class CoursesMetricsGradesLeadersList(SecureListAPIView):
     """
     ### The CoursesMetricsGradesLeadersList view allows clients to retrieve top 3 users who are leading

lms/djangoapps/api_manager/models.py

@@ -3,6 +3,8 @@
 """ Database ORM models managed by this Django app """
 from django.contrib.auth.models import Group, User
 from django.db import models
+from django.db.models import Q
+from django.conf import settings
 
 from model_utils.models import TimeStampedModel
 from .utils import is_int
@@ -159,6 +161,16 @@ class CourseModuleCompletion(TimeStampedModel):
     content_id = models.CharField(max_length=255, db_index=True)
     stage = models.CharField(max_length=255, null=True, blank=True)
 
+    @classmethod
+    def get_actual_completions(cls):
+        """
+        This would skip those modules with ignorable categories
+        """
+        detached_categories = getattr(settings, 'PROGRESS_DETACHED_CATEGORIES', [])
+        cat_list = [Q(content_id__contains=item.strip()) for item in detached_categories]
+        cat_list = reduce(lambda a, b: a | b, cat_list)
+        return cls.objects.all().exclude(cat_list)
+
 
 class APIUserQuerySet(models.query.QuerySet):  # pylint: disable=R0924
     """ Custom QuerySet to modify id based lookup """

lms/djangoapps/api_manager/utils.py

@@ -153,8 +153,8 @@ def get_time_series_data(queryset, start, end, interval='days', date_field='crea
     sql = {
         'mysql': {
             'days': "DATE_FORMAT(`{}`, '%%Y-%%m-%%d')".format(date_field),
-            'weeks': "DATE_FORMAT(DATE_SUB(`{}`, INTERVAL(WEEKDAY(`{}`)) DAY), '%%Y-%%m-%%d')".\
-                format(date_field, date_field),
+            'weeks': "DATE_FORMAT(DATE_SUB(`{}`, INTERVAL(WEEKDAY(`{}`)) DAY), '%%Y-%%m-%%d')".format(date_field,
+                                                                                                      date_field),
             'months': "DATE_FORMAT(`{}`, '%%Y-%%m-01')".format(date_field)
         },
         'sqlite': {