merged with master

lms/djangoapps/api_manager/courses/serializers.py

@@ -48,7 +48,7 @@ class CourseCompletionsLeadersSerializer(serializers.Serializer):
         completions = obj['completions'] or 0
         completion_percentage = 0
         if total_completions > 0:
-            completion_percentage = 100 * completions / float(total_completions)
+            completion_percentage = 100 * (completions / float(total_completions))
         return completion_percentage
 
 

lms/djangoapps/api_manager/courses/tests.py

@@ -1707,26 +1707,62 @@ class CoursesApiTests(TestCase):
         self.assertEqual(len(response.data['leaders']), 0)
 
     def test_courses_completions_leaders_list_get(self):
-        completion_uri = '{}/{}/completions/'.format(self.base_courses_uri, unicode(self.course.id))
+        course = CourseFactory.create(
+            number='4033',
+            name='leaders_by_completions',
+            start=datetime(2014, 9, 16, 14, 30),
+            end=datetime(2015, 1, 16)
+        )
+
+        chapter = ItemFactory.create(
+            category="chapter",
+            parent_location=course.location,
+            data=self.test_data,
+            due=datetime(2014, 5, 16, 14, 30),
+            display_name="Overview"
+        )
+
+        sub_section = ItemFactory.create(
+            parent_location=chapter.location,
+            category="sequential",
+            display_name=u"test subsection",
+        )
+        unit = ItemFactory.create(
+            parent_location=sub_section.location,
+            category="vertical",
+            metadata={'graded': True, 'format': 'Homework'},
+            display_name=u"test unit",
+        )
+
+        # create 5 users
+        USER_COUNT = 5
+        users = [UserFactory.create(username="testuser_cctest" + str(__), profile='test') for __ in xrange(USER_COUNT)]
+
+        for user in users:
+            CourseEnrollmentFactory.create(user=user, course_id=course.id)
+
+        test_course_id = unicode(course.id)
+        completion_uri = '{}/{}/completions/'.format(self.base_courses_uri, test_course_id)
+        leaders_uri = '{}/{}/metrics/completions/leaders/'.format(self.base_courses_uri, test_course_id)
         # Make last user as observer to make sure that data is being filtered out
-        allow_access(self.course, self.users[USER_COUNT-1], 'observer')
+        allow_access(course, users[USER_COUNT-1], 'observer')
 
         for i in xrange(1, 26):
             local_content_name = 'Video_Sequence{}'.format(i)
             local_content = ItemFactory.create(
                 category="videosequence",
-                parent_location=self.unit.location,
+                parent_location=unit.location,
                 data=self.test_data,
                 display_name=local_content_name
             )
             if i < 3:
-                user_id = self.users[0].id
+                user_id = users[0].id
             elif i < 10:
-                user_id = self.users[1].id
+                user_id = users[1].id
             elif i < 17:
-                user_id = self.users[2].id
+                user_id = users[2].id
             else:
-                user_id = self.users[3].id
+                user_id = users[3].id
 
             content_id = unicode(local_content.scope_ids.usage_id)
             completions_data = {'content_id': content_id, 'user_id': user_id}
@@ -1735,40 +1771,62 @@ class CoursesApiTests(TestCase):
 
             # observer should complete everything, so we can assert that it is filtered out
             response = self.do_post(completion_uri, {
-                'content_id': content_id, 'user_id': self.users[USER_COUNT-1].id
+                'content_id': content_id, 'user_id': users[USER_COUNT-1].id
             })
             self.assertEqual(response.status_code, 201)
 
-        test_uri = '{}/{}/metrics/completions/leaders/?{}'.format(self.base_courses_uri, self.test_course_id, 'count=6')
+        expected_course_avg = '25.000'
+        test_uri = '{}?count=6'.format(leaders_uri)
         response = self.do_get(test_uri)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(response.data['leaders']), 4)
-        self.assertEqual(response.data['course_avg'], 14)
+        self.assertEqual('{0:.3f}'.format(response.data['course_avg']), expected_course_avg)
 
         # without count filter and user_id
-        test_uri = '{}/{}/metrics/completions/leaders/?user_id={}'.format(self.base_courses_uri, self.test_course_id,
-                                                                          self.users[1].id)
+        test_uri = '{}?user_id={}'.format(leaders_uri, users[1].id)
         response = self.do_get(test_uri)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.data['leaders']), 3)
+        self.assertEqual(len(response.data['leaders']), 4)
         self.assertEqual(response.data['position'], 2)
-        self.assertEqual(response.data['completions'], 19)
+        self.assertEqual('{0:.3f}'.format(response.data['completions']), '28.000')
 
         # with skipleaders filter
-        test_uri = '{}/{}/metrics/completions/leaders/?user_id={}&skipleaders=true'.format(self.base_courses_uri,
-                                                                                           self.test_course_id,
-                                                                                           self.users[1].id)
+        test_uri = '{}?user_id={}&skipleaders=true'.format(leaders_uri, users[1].id)
         response = self.do_get(test_uri)
         self.assertEqual(response.status_code, 200)
         self.assertIsNone(response.data.get('leaders', None))
-        self.assertIsNone(response.data.get('position', None))
-        self.assertEqual(response.data['completions'], 19)
+        self.assertEqual('{0:.3f}'.format(response.data['course_avg']), expected_course_avg)
+        self.assertEqual('{0:.3f}'.format(response.data['completions']), '28.000')
 
         # test with bogus course
         test_uri = '{}/{}/metrics/completions/leaders/'.format(self.base_courses_uri, self.test_bogus_course_id)
         response = self.do_get(test_uri)
         self.assertEqual(response.status_code, 404)
 
+        #filter course module completion by organization
+        data = {
+            'name': 'Test Organization',
+            'display_name': 'Test Org Display Name',
+            'users': [users[1].id]
+        }
+        response = self.do_post(self.base_organizations_uri, data)
+        self.assertEqual(response.status_code, 201)
+        test_uri = '{}?organizations={}'.format(leaders_uri, response.data['id'])
+        response = self.do_get(test_uri)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data['leaders']), 1)
+        self.assertEqual(response.data['leaders'][0]['id'], users[1].id)
+        self.assertEqual('{0:.3f}'.format(response.data['leaders'][0]['completions']), '28.000')
+        self.assertEqual('{0:.3f}'.format(response.data['course_avg']), '28.000')
+
+        # test with unknown user
+        test_uri = '{}?user_id={}&skipleaders=true'.format(leaders_uri, '909999')
+        response = self.do_get(test_uri)
+        self.assertEqual(response.status_code, 200)
+        self.assertIsNone(response.data.get('leaders', None))
+        self.assertEqual(response.data['position'], 0)
+        self.assertEqual(response.data['completions'], 0)
+
     def test_courses_metrics_grades_list_get(self):
         # Retrieve the list of grades for this course
         # All the course/item/user scaffolding was handled in Setup

lms/djangoapps/api_manager/courses/views.py

@@ -23,6 +23,7 @@ from courseware.models import StudentModule
 from courseware.views import get_static_tab_contents
 from django_comment_common.models import FORUM_ROLE_MODERATOR
 from gradebook.models import StudentGradebook
+from progress.models import StudentProgress
 from instructor.access import revoke_access, update_forum_role
 from student.models import CourseEnrollment, CourseEnrollmentAllowed
 from student.roles import CourseRole, CourseAccessRole, CourseInstructorRole, CourseStaffRole, CourseObserverRole, CourseAssistantRole, UserBasedRole
@@ -1515,15 +1516,15 @@ class CoursesMetrics(SecureAPIView):
         exclude_users = _get_aggregate_exclusion_user_ids(course_key)
         users_enrolled_qs = CourseEnrollment.users_enrolled_in(course_key).exclude(id__in=exclude_users)
         organization = request.QUERY_PARAMS.get('organization', None)
+        org_ids = None
         if organization:
             users_enrolled_qs = users_enrolled_qs.filter(organizations=organization)
+            org_ids = [organization]
 
-        users_started_qs = CourseModuleCompletion.objects.filter(course_id=course_id).exclude(user_id__in=exclude_users)
-        if organization:
-            users_started_qs = users_started_qs.filter(user__organizations=organization)
+        users_started = StudentProgress.get_num_users_started(course_key, exclude_users=exclude_users, org_ids=org_ids)
         data = {
             'users_enrolled': users_enrolled_qs.count(),
-            'users_started': users_started_qs.values('user').distinct().count(),
+            'users_started': users_started,
             'grade_cutoffs': course_descriptor.grading_policy['GRADE_CUTOFFS']
         }
 
@@ -1601,50 +1602,43 @@ class CoursesMetricsCompletionsLeadersList(SecureAPIView):
         GET /api/courses/{course_id}/metrics/completions/leaders/
         """
         user_id = self.request.QUERY_PARAMS.get('user_id', None)
-        count = self.request.QUERY_PARAMS.get('count', 3)
+        count = self.request.QUERY_PARAMS.get('count', None)
         skipleaders = str2bool(self.request.QUERY_PARAMS.get('skipleaders', 'false'))
         data = {}
         course_avg = 0
         if not course_exists(request, request.user, course_id):
             return Response({}, status=status.HTTP_404_NOT_FOUND)
         course_key = get_course_key(course_id)
-        detached_categories = ['discussion-course', 'group-project', 'discussion-forum']
-        total_possible_completions = float(len(get_course_leaf_nodes(course_key, detached_categories)))
+        total_possible_completions = float(len(get_course_leaf_nodes(course_key)))
         exclude_users = _get_aggregate_exclusion_user_ids(course_key)
-        cat_list = [Q(content_id__contains=item.strip()) for item in detached_categories]
-        cat_list = reduce(lambda a, b: a | b, cat_list)
+        orgs_filter = self.request.QUERY_PARAMS.get('organizations', None)
+        if orgs_filter:
+            upper_bound = getattr(settings, 'API_LOOKUP_UPPER_BOUND', 100)
+            orgs_filter = orgs_filter.split(",")[:upper_bound]
 
-        queryset = CourseModuleCompletion.objects.filter(course_id=course_key)\
-            .exclude(user__in=exclude_users).exclude(cat_list)
-        total_actual_completions = queryset.filter(user__is_active=True).count()
+        total_actual_completions = StudentProgress.get_total_completions(course_key, exclude_users=exclude_users,
+                                                                         org_ids=orgs_filter)
         if user_id:
-            user_queryset = CourseModuleCompletion.objects.filter(course_id=course_key, user__id=user_id)\
-                .exclude(cat_list)
-            user_completions = user_queryset.count()
-            if not skipleaders:
-                user_time_completed = user_queryset.aggregate(time_completed=Max('created'))
-                user_time_completed = user_time_completed['time_completed'] or timezone.now()
-                completions_above_user = queryset.filter(user__is_active=True).values('user__id')\
-                    .annotate(completions=Count('content_id')).annotate(time_completed=Max('created'))\
-                    .filter(Q(completions__gt=user_completions) | Q(completions=user_completions,
-                                                                    time_completed__lt=user_time_completed)).count()
-                data['position'] = completions_above_user + 1
+            user_data = StudentProgress.get_user_position(course_key, user_id, exclude_users=exclude_users)
+            data['position'] = user_data['position']
+            user_completions = user_data['completions']
             completion_percentage = 0
             if total_possible_completions > 0:
-                completion_percentage = 100 * user_completions/total_possible_completions
+                completion_percentage = 100 * (user_completions/total_possible_completions)
             data['completions'] = completion_percentage
 
-        total_users = CourseEnrollment.users_enrolled_in(course_key).exclude(id__in=exclude_users).count()
+        total_users_qs = CourseEnrollment.users_enrolled_in(course_key).exclude(id__in=exclude_users)
+        if orgs_filter:
+            total_users_qs = total_users_qs.filter(organizations__in=orgs_filter)
+        total_users = total_users_qs.count()
         if total_users and total_actual_completions:
             course_avg = total_actual_completions / float(total_users)
-            course_avg = 100 * course_avg / total_possible_completions  # avg in percentage
+            course_avg = 100 * (course_avg / total_possible_completions)  # avg in percentage
         data['course_avg'] = course_avg
+
         if not skipleaders:
-            queryset = queryset.filter(user__is_active=True).values('user__id', 'user__username',
-                                                                    'user__profile__title',
-                                                                    'user__profile__avatar_url')\
-                           .annotate(completions=Count('content_id')).annotate(time_completed=Max('created'))\
-                           .order_by('-completions', 'time_completed')[:count]
+            queryset = StudentProgress.generate_leaderboard(course_key, count=count, exclude_users=exclude_users,
+                                                            org_ids=orgs_filter)
             serializer = CourseCompletionsLeadersSerializer(queryset, many=True,
                                                             context={'total_completions': total_possible_completions})
             data['leaders'] = serializer.data  # pylint: disable=E1101

lms/djangoapps/api_manager/courseware_access.py

 """ Centralized access to LMS courseware app """
 from django.contrib.auth.models import AnonymousUser
 from django.utils import timezone
+from django.conf import settings
 
 from courseware import courses, module_render
 from courseware.model_data import FieldDataCache
@@ -62,12 +63,13 @@ def get_course_total_score(course_summary):
     return score
 
 
-def get_course_leaf_nodes(course_key, detached_categories):
+def get_course_leaf_nodes(course_key):
     """
     Get count of the leaf nodes with ability to exclude some categories
     """
     nodes = []
-    verticals = get_modulestore().get_items(course_key, category='vertical')
+    detached_categories = getattr(settings, 'PROGRESS_DETACHED_CATEGORIES', [])
+    verticals = get_modulestore().get_items(course_key,  qualifiers={'category': 'vertical'})
     for vertical in verticals:
         if hasattr(vertical, 'children'):
             nodes.extend([unit for unit in vertical.children

lms/djangoapps/api_manager/models.py

 # pylint: disable=E1101
 
 """ Database ORM models managed by this Django app """
-
 from django.contrib.auth.models import Group, User
 from django.db import models
 
 from model_utils.models import TimeStampedModel
 from .utils import is_int
-
 from projects.models import Workgroup
 
 

lms/djangoapps/progress/__init__.py

+"""
+Initialization module for progress djangoapp
+"""
+import progress.signals

lms/djangoapps/progress/management/commands/generate_progress_entries.py

+"""
+One-time data migration script -- shoulen't need to run it again
+"""
+import logging
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.db.models import Q
+
+from progress.models import StudentProgress
+from api_manager.models import CourseModuleCompletion
+from student.models import CourseEnrollment
+from xmodule.modulestore.django import modulestore
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Creates (or updates) progress entries for the specified course(s) and/or user(s)
+    """
+
+    def handle(self, *args, **options):
+        help = "Command to creaete or update progress entries"
+        option_list = BaseCommand.option_list + (
+            make_option(
+                "-c",
+                "--course_ids",
+                dest="course_ids",
+                help="List of courses for which to generate progress",
+                metavar="slashes:first+course+id,slashes:second+course+id"
+            ),
+            make_option(
+                "-u",
+                "--user_ids",
+                dest="user_ids",
+                help="List of users for which to generate progress",
+                metavar="1234,2468,3579"
+            ),
+        )
+
+        course_ids = options.get('course_ids')
+        user_ids = options.get('user_ids')
+
+        detached_categories = getattr(settings, 'PROGRESS_DETACHED_CATEGORIES', [])
+        cat_list = [Q(content_id__contains=item.strip()) for item in detached_categories]
+        cat_list = reduce(lambda a, b: a | b, cat_list)
+
+        # Get the list of courses from the system
+        courses = modulestore().get_courses()
+
+        # If one or more courses were specified by the caller, just use those ones...
+        if course_ids is not None:
+            filtered_courses = []
+            for course in courses:
+                if unicode(course.id) in course_ids.split(','):
+                    filtered_courses.append(course)
+            courses = filtered_courses
+
+        for course in courses:
+            users = CourseEnrollment.users_enrolled_in(course.id)
+            # If one or more users were specified by the caller, just use those ones...
+            if user_ids is not None:
+                filtered_users = []
+                for user in users:
+                    if str(user.id) in user_ids.split(','):
+                        filtered_users.append(user)
+                users = filtered_users
+
+            # For each user...
+            for user in users:
+                completions = CourseModuleCompletion.objects.filter(course_id=course.id, user_id=user.id)\
+                    .exclude(cat_list).count()
+                progress, created = StudentProgress.objects.get_or_create(user=user,
+                                                                          course_id=course.id,
+                                                                          completions=completions)
+                log_msg = 'Progress entry created -- Course: {}, User: {}  (completions: {})'.format(course.id, user.id
+                                                                                                     , completions)
+                print log_msg
+                log.info(log_msg)

lms/djangoapps/progress/management/commands/tests/test_generate_progress_entries.py

+"""
+Run these tests @ Devstack:
+    paver test_system -t lms/djangoapps/progress/management/commands/tests/test_generate_progress_entries.py --fasttest
+"""
+from datetime import datetime
+import uuid
+import time
+
+from django.db.models.signals import post_save
+
+from capa.tests.response_xml_factory import StringResponseXMLFactory
+from progress.management.commands import generate_progress_entries
+from progress.models import StudentProgress, StudentProgressHistory
+from progress.signals import handle_cmc_post_save_signal
+from student.tests.factories import UserFactory, CourseEnrollmentFactory
+from api_manager.models import CourseModuleCompletion
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+
+
+class GenerateProgressEntriesTests(ModuleStoreTestCase):
+    """
+    Test suite for progress generation script
+    """
+
+    def setUp(self):
+
+        # Create a couple courses to work with
+        self.course = CourseFactory.create(
+            start=datetime(2014, 6, 16, 14, 30),
+            end=datetime(2015, 1, 16)
+        )
+        self.test_data = '<html>{}</html>'.format(str(uuid.uuid4()))
+
+        chapter1 = ItemFactory.create(
+            category="chapter",
+            parent_location=self.course.location,
+            data=self.test_data,
+            due=datetime(2014, 5, 16, 14, 30),
+            display_name="Overview"
+        )
+
+        chapter2 = ItemFactory.create(
+            category="chapter",
+            parent_location=self.course.location,
+            data=self.test_data,
+            due=datetime(2014, 5, 16, 14, 30),
+            display_name="Overview"
+        )
+        self.problem = ItemFactory.create(
+            parent_location=chapter1.location,
+            category='problem',
+            data=StringResponseXMLFactory().build_xml(answer='bar'),
+            display_name="homework problem 1",
+            metadata={'rerandomize': 'always', 'graded': True, 'format': "Homework"}
+        )
+        self.problem2 = ItemFactory.create(
+            parent_location=chapter2.location,
+            category='problem',
+            data=StringResponseXMLFactory().build_xml(answer='bar'),
+            display_name="homework problem 2",
+            metadata={'rerandomize': 'always', 'graded': True, 'format': "Homework"}
+        )
+        self.problem3 = ItemFactory.create(
+            parent_location=chapter2.location,
+            category='problem',
+            data=StringResponseXMLFactory().build_xml(answer='bar'),
+            display_name="lab problem 1",
+            metadata={'rerandomize': 'always', 'graded': True, 'format': "Lab"}
+        )
+
+        # Create some users and enroll them
+        self.users = [UserFactory.create(username="testuser" + str(__), profile='test') for __ in xrange(3)]
+        for user in self.users:
+            CourseEnrollmentFactory.create(user=user, course_id=self.course.id)
+        # Turn off the signalling mechanism temporarily
+        post_save.disconnect(receiver=handle_cmc_post_save_signal,
+                             sender=CourseModuleCompletion, dispatch_uid='edxapp.api_manager.post_save_cms')
+        self._generate_course_completion_test_entries()
+        post_save.connect(receiver=handle_cmc_post_save_signal,
+                          sender=CourseModuleCompletion, dispatch_uid='edxapp.api_manager.post_save_cms')
+
+    def _generate_course_completion_test_entries(self):
+        """
+        Clears existing CourseModuleCompletion entries and creates 3 for each user
+        """
+        CourseModuleCompletion.objects.all().delete()
+        for user in self.users:
+            completion, created = CourseModuleCompletion.objects.get_or_create(user=user,
+                                                                               course_id=self.course.id,
+                                                                               content_id=unicode(self.problem.location),
+                                                                               stage=None)
+
+            completion, created = CourseModuleCompletion.objects.get_or_create(user=user,
+                                                                               course_id=self.course.id,
+                                                                               content_id=unicode(self.problem2.location),
+                                                                               stage=None)
+
+            completion, created = CourseModuleCompletion.objects.get_or_create(user=user,
+                                                                               course_id=self.course.id,
+                                                                               content_id=unicode(self.problem3.location),
+                                                                               stage=None)
+
+    def test_generate_progress_entries_command(self):
+        """
+        Test the progress entry generator
+        """
+        # Set up the command context
+        course_ids = '{},slashes:bogus+course+id'.format(self.course.id)
+        user_ids = '{}'.format(self.users[0].id)
+        current_entries = StudentProgress.objects.all()
+        self.assertEqual(len(current_entries), 0)
+        current_entries = StudentProgressHistory.objects.all()
+        self.assertEqual(len(current_entries), 0)
+
+        # Run the command just for one user
+        generate_progress_entries.Command().handle(user_ids=user_ids)
+
+        # Confirm the progress has been properly updated
+        current_entries = StudentProgress.objects.all()
+        self.assertEqual(len(current_entries), 1)
+        current_entries = StudentProgressHistory.objects.all()
+        self.assertEqual(len(current_entries), 1)
+        user0_entry = StudentProgress.objects.get(user=self.users[0])
+        self.assertEqual(user0_entry.completions, 3)
+
+        # Run the command across all users, but just for the specified course
+        generate_progress_entries.Command().handle(course_ids=course_ids)
+
+        # Confirm that the progress has been properly updated
+        current_entries = StudentProgress.objects.all()
+        self.assertEqual(len(current_entries), 3)
+        current_entries = StudentProgressHistory.objects.all()
+        self.assertEqual(len(current_entries), 3)
+        user0_entry = StudentProgress.objects.get(user=self.users[0])
+        self.assertEqual(user0_entry.completions, 3)
+        user1_entry = StudentProgress.objects.get(user=self.users[1])
+        self.assertEqual(user1_entry.completions, 3)
+        user2_entry = StudentProgress.objects.get(user=self.users[2])
+        self.assertEqual(user2_entry.completions, 3)
+
+    def test_progress_history(self):
+        """
+        Test the progress, and history
+        """
+        # Clear enteries
+        StudentProgress.objects.all().delete()
+        StudentProgressHistory.objects.all().delete()
+        self._generate_course_completion_test_entries()
+
+        #let single bindings to complete their work
+        time.sleep(2)
+        current_entries = StudentProgress.objects.all()
+        self.assertEqual(len(current_entries), 3)
+        current_entries = StudentProgressHistory.objects.all()
+        self.assertEqual(len(current_entries), 9)
+        user0_entry = StudentProgress.objects.get(user=self.users[0])
+        self.assertEqual(user0_entry.completions, 3)

lms/djangoapps/progress/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'StudentProgress'
+        db.create_table('progress_studentprogress', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created', self.gf('model_utils.fields.AutoCreatedField')(default=datetime.datetime.now)),
+            ('modified', self.gf('model_utils.fields.AutoLastModifiedField')(default=datetime.datetime.now)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('course_id', self.gf('xmodule_django.models.CourseKeyField')(db_index=True, max_length=255, blank=True)),
+            ('completions', self.gf('django.db.models.fields.IntegerField')(default=0)),
+        ))
+        db.send_create_signal('progress', ['StudentProgress'])
+
+        # Adding unique constraint on 'StudentProgress', fields ['user', 'course_id']
+        db.create_unique('progress_studentprogress', ['user_id', 'course_id'])
+
+        # Adding model 'StudentProgressHistory'
+        db.create_table('progress_studentprogresshistory', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created', self.gf('model_utils.fields.AutoCreatedField')(default=datetime.datetime.now)),
+            ('modified', self.gf('model_utils.fields.AutoLastModifiedField')(default=datetime.datetime.now)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
+            ('course_id', self.gf('xmodule_django.models.CourseKeyField')(db_index=True, max_length=255, blank=True)),
+            ('completions', self.gf('django.db.models.fields.IntegerField')()),
+        ))
+        db.send_create_signal('progress', ['StudentProgressHistory'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'StudentProgress', fields ['user', 'course_id']
+        db.delete_unique('progress_studentprogress', ['user_id', 'course_id'])
+
+        # Deleting model 'StudentProgress'
+        db.delete_table('progress_studentprogress')
+
+        # Deleting model 'StudentProgressHistory'
+        db.delete_table('progress_studentprogresshistory')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'progress.studentprogress': {
+            'Meta': {'unique_together': "(('user', 'course_id'),)", 'object_name': 'StudentProgress'},
+            'completions': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'course_id': ('xmodule_django.models.CourseKeyField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        },
+        'progress.studentprogresshistory': {
+            'Meta': {'object_name': 'StudentProgressHistory'},
+            'completions': ('django.db.models.fields.IntegerField', [], {}),
+            'course_id': ('xmodule_django.models.CourseKeyField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['progress']
\ No newline at end of file

lms/djangoapps/progress/models.py

+"""
+Django database models supporting the progress app
+"""
+
+from django.contrib.auth.models import User
+from django.db import models
+from django.db.models import Sum, Q
+
+from model_utils.models import TimeStampedModel
+from xmodule_django.models import CourseKeyField
+
+
+class StudentProgress(TimeStampedModel):
+    """
+    StudentProgress is essentially a container used to store calculated progress of user
+    """
+    user = models.ForeignKey(User, db_index=True)
+    course_id = CourseKeyField(db_index=True, max_length=255, blank=True)
+    completions = models.IntegerField(default=0)
+
+    class Meta:
+        """
+        Meta information for this Django model
+        """
+        unique_together = (('user', 'course_id'),)
+
+    @classmethod
+    def get_total_completions(cls, course_key, exclude_users=None, org_ids=None):
+        """
+        Returns count of completions for a given course.
+        """
+        queryset = cls.objects.filter(course_id__exact=course_key, user__is_active=True)\
+            .exclude(user__id__in=exclude_users)
+        if org_ids:
+            queryset = queryset.filter(user__organizations__in=org_ids)
+        completions = queryset.aggregate(total=Sum('completions'))
+        completions = completions['total'] or 0
+        return completions
+
+    @classmethod
+    def get_num_users_started(cls, course_key, exclude_users=None, org_ids=None):
+        """
+        Returns count of users who completed at least one module.
+        """
+        queryset = cls.objects.filter(course_id__exact=course_key, user__is_active=True)\
+            .exclude(user__id__in=exclude_users)
+        if org_ids:
+            queryset = queryset.filter(user__organizations__in=org_ids)
+        return queryset.count()
+
+    @classmethod
+    def get_user_position(cls, course_key, user_id, exclude_users=None):
+        """
+        Returns user's progress position and completions for a given course.
+        data = {"completions": 22, "position": 4}
+        """
+        data = {"completions": 0, "position": 0}
+        try:
+            queryset = cls.objects.get(course_id__exact=course_key, user__id=user_id)
+        except cls.DoesNotExist:
+            queryset = None
+
+        if queryset:
+            user_completions = queryset.completions
+            user_time_completed = queryset.modified
+
+            users_above = cls.objects.filter(Q(completions__gt=user_completions) | Q(completions=user_completions,
+                                                                                     modified__lt=user_time_completed),
+                                             course_id__exact=course_key, user__is_active=True)\
+                .exclude(user__id__in=exclude_users)\
+                .count()
+            data['position'] = users_above + 1
+            data['completions'] = user_completions
+        return data
+
+    @classmethod
+    def generate_leaderboard(cls, course_key, count=3, exclude_users=None, org_ids=None):
+        """
+        Assembles a data set representing the Top N users, by progress, for a given course.
+
+        data = [
+                {'id': 123, 'username': 'testuser1', 'title', 'Engineer', 'avatar_url': 'http://gravatar.com/123/', 'completions': 0.92},
+                {'id': 983, 'username': 'testuser2', 'title', 'Analyst', 'avatar_url': 'http://gravatar.com/983/', 'completions': 0.91},
+                {'id': 246, 'username': 'testuser3', 'title', 'Product Owner', 'avatar_url': 'http://gravatar.com/246/', 'completions': 0.90},
+                {'id': 357, 'username': 'testuser4', 'title', 'Director', 'avatar_url': 'http://gravatar.com/357/', 'completions': 0.89},
+        ]
+
+        """
+        queryset = cls.objects\
+            .filter(course_id__exact=course_key, user__is_active=True).exclude(user__id__in=exclude_users)
+        if org_ids:
+            queryset = queryset.filter(user__organizations__in=org_ids)
+        queryset = queryset.values(
+            'user__id',
+            'user__username',
+            'user__profile__title',
+            'user__profile__avatar_url',
+            'completions')\
+            .order_by('-completions', 'modified')[:count]
+        return queryset
+
+
+class StudentProgressHistory(TimeStampedModel):
+    """
+    A running audit trail for the StudentProgress model.  Listens for
+    post_save events and creates/stores copies of progress entries.
+    """
+    user = models.ForeignKey(User, db_index=True)
+    course_id = CourseKeyField(db_index=True, max_length=255, blank=True)
+    completions = models.IntegerField()

lms/djangoapps/progress/signals.py

+"""
+Signal handlers supporting various progress use cases
+"""
+import sys
+import logging
+from django.dispatch import receiver
+from django.db.models.signals import post_save
+from django.core.exceptions import ObjectDoesNotExist
+from django.conf import settings
+
+from progress.models import StudentProgress, StudentProgressHistory
+from api_manager.models import CourseModuleCompletion
+log = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=CourseModuleCompletion, dispatch_uid='edxapp.api_manager.post_save_cms')
+def handle_cmc_post_save_signal(sender, instance, created, **kwargs):
+    """
+    Broadcast the progress change event
+    """
+    content_id = unicode(instance.content_id)
+    detached_categories = getattr(settings, 'PROGRESS_DETACHED_CATEGORIES', [])
+    if created and not any(category in content_id for category in detached_categories):
+        try:
+            progress = StudentProgress.objects.get(user=instance.user, course_id=instance.course_id)
+            progress.completions += 1
+            progress.save()
+        except ObjectDoesNotExist:
+            progress = StudentProgress(user=instance.user, course_id=instance.course_id, completions=1)
+            progress.save()
+        except Exception:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            logging.error("Exception type: {} with value: {}".format(exc_type, exc_value))
+
+
+@receiver(post_save, sender=StudentProgress)
+def save_history(sender, instance, **kwargs):  # pylint: disable=no-self-argument, unused-argument
+    """
+    Event hook for creating progress entry copies
+    """
+    history_entry = StudentProgressHistory(
+        user=instance.user,
+        course_id=instance.course_id,
+        completions=instance.completions
+    )
+    history_entry.save()

lms/envs/aws.py

@@ -708,6 +708,13 @@ if FEATURES.get('ENABLE_LTI_PROVIDER'):
 if FEATURES.get('STUDENT_GRADEBOOK') and "'gradebook'" not in INSTALLED_APPS:
     INSTALLED_APPS += ('gradebook',)
 
+############# Student Progress #################
+if FEATURES.get('STUDENT_PROGRESS') and "progress" not in INSTALLED_APPS:
+    INSTALLED_APPS += ('progress',)
+
+##### SET THE LIST OF ALLOWED IP ADDRESSES FOR THE API ######
+API_ALLOWED_IP_ADDRESSES = ENV_TOKENS.get('API_ALLOWED_IP_ADDRESSES')
+
 EXCLUDE_MIDDLEWARE_CLASSES = ENV_TOKENS.get('EXCLUDE_MIDDLEWARE_CLASSES', [])
 
 MIDDLEWARE_CLASSES = tuple(_class for _class in MIDDLEWARE_CLASSES if _class not in EXCLUDE_MIDDLEWARE_CLASSES)

lms/envs/common.py

@@ -432,6 +432,11 @@ FEATURES = {
     # In order to use the gradebook, you must add it to the list of INSTALLED_APPS in
     # addition to setting the flag to True here.  A reference is available in aws.py
     'STUDENT_GRADEBOOK': False,
+
+    # Enable the Student Progress, which is essentially a cache of module completions
+    # In order to use the "progress", you must add it to the list of INSTALLED_APPS in
+    # addition to setting the flag to True here.  A reference is available in aws.py
+    'STUDENT_PROGRESS': False,
 }
 
 # Ignore static asset files on import which match this pattern
@@ -612,7 +617,8 @@ USAGE_KEY_PATTERN = r'(?P<usage_key_string>(?:i4x://?[^/]+/[^/]+/[^/]+/[^@]+(?:@
 ASSET_KEY_PATTERN = r'(?P<asset_key_string>(?:/?c4x(:/)?/[^/]+/[^/]+/[^/]+/[^@]+(?:@[^/]+)?)|(?:[^/]+))'
 USAGE_ID_PATTERN = r'(?P<usage_id>(?:i4x://?[^/]+/[^/]+/[^/]+/[^@]+(?:@[^/]+)?)|(?:[^/]+))'
 
-
+# Modules having these categories would be excluded from progress calculations
+PROGRESS_DETACHED_CATEGORIES = ['discussion-course', 'group-project', 'discussion-forum']
 ############################## EVENT TRACKING #################################
 
 # FIXME: Should we be doing this truncation?

lms/envs/test.py

@@ -523,3 +523,9 @@ FEATURES['STUDENT_GRADEBOOK'] = True
 if FEATURES.get('STUDENT_GRADEBOOK', False) and "'gradebook'" not in INSTALLED_APPS:
     INSTALLED_APPS += ('gradebook',)
 
+
+############# Student Progress #################
+FEATURES['STUDENT_PROGRESS'] = True
+if FEATURES.get('STUDENT_PROGRESS', False) and "'progress'" not in INSTALLED_APPS:
+    INSTALLED_APPS += ('progress',)
+