Use cached data to find completed programs on LMS

These changes improve the performance of the LMS' program certification task. They also reduce strain on the programs service, especially when attempting to award program certificates to large numbers of students. ECOM-4490.

lms/djangoapps/learner_dashboard/tests/test_programs.py

@@ -56,8 +56,6 @@ class TestProgramListing(
     def _create_course_and_enroll(self, student, org, course, run):
         """
         Creates a course and associated enrollment.
-
-        TODO: Use CourseEnrollmentFactory to avoid course creation.
         """
         course_location = locator.CourseLocator(org, course, run)
         course = CourseFactory.create(

lms/djangoapps/learner_dashboard/views.py

@@ -11,7 +11,6 @@ from edxmako.shortcuts import render_to_response
 from openedx.core.djangoapps.credentials.utils import get_programs_credentials
 from openedx.core.djangoapps.programs.models import ProgramsApiConfig
 from openedx.core.djangoapps.programs import utils
-from student.views import get_course_enrollments
 
 
 @login_required
@@ -22,8 +21,7 @@ def view_programs(request):
     if not show_program_listing:
         raise Http404
 
-    enrollments = list(get_course_enrollments(request.user, None, []))
-    meter = utils.ProgramProgressMeter(request.user, enrollments)
+    meter = utils.ProgramProgressMeter(request.user)
     programs = meter.engaged_programs
 
     # TODO: Pull 'xseries' string from configuration model.

openedx/core/djangoapps/programs/tasks/v1/tasks.py

@@ -10,7 +10,7 @@ from edx_rest_api_client.client import EdxRestApiClient
 from openedx.core.djangoapps.credentials.models import CredentialsApiConfig
 from openedx.core.djangoapps.credentials.utils import get_user_credentials
 from openedx.core.djangoapps.programs.models import ProgramsApiConfig
-from openedx.core.djangoapps.programs.utils import get_completed_courses
+from openedx.core.djangoapps.programs.utils import ProgramProgressMeter
 from openedx.core.lib.token_utils import get_id_token
 
 
@@ -35,21 +35,20 @@ def get_api_client(api_config, student):
     return EdxRestApiClient(api_config.internal_api_url, jwt=id_token)
 
 
-def get_completed_programs(client, course_certificates):
+def get_completed_programs(student):
     """
     Given a set of completed courses, determine which programs are completed.
 
     Args:
-        client:
-            programs API client (EdxRestApiClient)
-        course_certificates:
-            iterable of dicts with structure {'course_id': course_key, 'mode': cert_type}
+        student (User): Representing the student whose completed programs to check for.
 
     Returns:
         list of program ids
 
     """
-    return client.programs.complete.post({'completed_courses': course_certificates})['program_ids']
+    meter = ProgramProgressMeter(student)
+
+    return meter.completed_programs
 
 
 def get_awarded_certificate_programs(student):
@@ -147,29 +146,9 @@ def award_program_certificates(self, username):
             # Don't retry for this case - just conclude the task.
             return
 
-        # Fetch the set of all course runs for which the user has earned a
-        # certificate.
-        course_certs = get_completed_courses(student)
-        if not course_certs:
-            # Highly unlikely, since at present the only trigger for this task
-            # is the earning of a new course certificate.  However, it could be
-            # that the transaction in which a course certificate was awarded
-            # was subsequently rolled back, which could lead to an empty result
-            # here, so we'll at least log that this happened before exiting.
-            #
-            # If this task is ever updated to support revocation of program
-            # certs, this branch should be removed, since it could make sense
-            # in that case to call this task for a user without any (valid)
-            # course certs.
-            LOGGER.warning('Task award_program_certificates was called for user %s with no completed courses', username)
-            return
-
-        # Invoke the Programs API completion check endpoint to identify any
-        # programs that are satisfied by these course completions.
-        programs_client = get_api_client(config, student)
-        program_ids = get_completed_programs(programs_client, course_certs)
+        program_ids = get_completed_programs(student)
         if not program_ids:
-            # Again, no reason to continue beyond this point unless/until this
+            # No reason to continue beyond this point unless/until this
             # task gets updated to support revocation of program certs.
             LOGGER.info('Task award_program_certificates was called for user %s with no completed programs', username)
             return

openedx/core/djangoapps/programs/utils.py

@@ -5,6 +5,7 @@ import logging
 
 from django.core.urlresolvers import reverse
 from django.utils import timezone
+from django.utils.functional import cached_property
 from opaque_keys.edx.keys import CourseKey
 import pytz
 
@@ -170,29 +171,27 @@ class ProgramProgressMeter(object):
 
     Arguments:
         user (User): The user for which to find programs.
-        enrollments (list): The user's active enrollments.
     """
-    def __init__(self, user, enrollments):
+    def __init__(self, user):
         self.user = user
+        self.course_ids = None
 
-        enrollments = sorted(enrollments, key=lambda e: e.created, reverse=True)
-        # enrollment.course_id is really a course key ಠ_ಠ
-        self.course_ids = [unicode(e.course_id) for e in enrollments]
-
-        self.engaged_programs = self._find_engaged_programs(self.user)
-        self.course_certs = None
+        self.programs = get_programs(self.user)
+        self.course_certs = get_completed_courses(self.user)
 
-    def _find_engaged_programs(self, user):
+    @cached_property
+    def engaged_programs(self):
         """Derive a list of programs in which the given user is engaged.
 
-        Arguments:
-            user (User): The user for which to find engaged programs.
-
         Returns:
             list of program dicts, ordered by most recent enrollment.
         """
-        programs = get_programs(user)
-        flattened = flatten_programs(programs, self.course_ids)
+        enrollments = CourseEnrollment.enrollments_for_user(self.user)
+        enrollments = sorted(enrollments, key=lambda e: e.created, reverse=True)
+        # enrollment.course_id is really a course key ಠ_ಠ
+        self.course_ids = [unicode(e.course_id) for e in enrollments]
+
+        flattened = flatten_programs(self.programs, self.course_ids)
 
         engaged_programs = []
         for course_id in self.course_ids:
@@ -210,8 +209,6 @@ class ProgramProgressMeter(object):
             list of dict, each containing information about a user's progress
                 towards completing a program.
         """
-        self.course_certs = get_completed_courses(self.user)
-
         progress = []
         for program in self.engaged_programs:
             completed, in_progress, not_started = [], [], []
@@ -219,9 +216,9 @@ class ProgramProgressMeter(object):
             for course_code in program['course_codes']:
                 name = course_code['display_name']
 
-                if self._is_complete(course_code):
+                if self._is_course_code_complete(course_code):
                     completed.append(name)
-                elif self._is_in_progress(course_code):
+                elif self._is_course_code_in_progress(course_code):
                     in_progress.append(name)
                 else:
                     not_started.append(name)
@@ -235,11 +232,33 @@ class ProgramProgressMeter(object):
 
         return progress
 
-    def _is_complete(self, course_code):
+    @property
+    def completed_programs(self):
+        """Identify programs completed by the student.
+
+        Returns:
+            list of int, each the ID of a completed program.
+        """
+        return [program['id'] for program in self.programs if self._is_program_complete(program)]
+
+    def _is_program_complete(self, program):
+        """Check if a user has completed a program.
+
+        A program is completed if the user has completed all nested course codes.
+
+        Arguments:
+            program (dict): Representing the program whose completion to assess.
+
+        Returns:
+            bool, whether the program is complete.
+        """
+        return all(self._is_course_code_complete(course_code) for course_code in program['course_codes'])
+
+    def _is_course_code_complete(self, course_code):
         """Check if a user has completed a course code.
 
-        A course code qualifies as completed if the user has earned a
-        certificate in the right mode for any nested run.
+        A course code is completed if the user has earned a certificate
+        in the right mode for any nested run.
 
         Arguments:
             course_code (dict): Containing nested run modes.
@@ -247,12 +266,9 @@ class ProgramProgressMeter(object):
         Returns:
             bool, whether the course code is complete.
         """
-        return any([
-            self._parse(run_mode) in self.course_certs
-            for run_mode in course_code['run_modes']
-        ])
+        return any(self._parse(run_mode) in self.course_certs for run_mode in course_code['run_modes'])
 
-    def _is_in_progress(self, course_code):
+    def _is_course_code_in_progress(self, course_code):
         """Check if a user is in the process of completing a course code.
 
         A user is in the process of completing a course code if they're
@@ -264,10 +280,7 @@ class ProgramProgressMeter(object):
         Returns:
             bool, whether the course code is in progress.
         """
-        return any([
-            run_mode['course_key'] in self.course_ids
-            for run_mode in course_code['run_modes']
-        ])
+        return any(run_mode['course_key'] in self.course_ids for run_mode in course_code['run_modes'])
 
     def _parse(self, run_mode):
         """Modify the structure of a run mode dict.