Merge pull request #11341 from edx/jsa/ecom-3354

Implement celery task to award program certs.

cms/envs/common.py

@@ -1113,3 +1113,8 @@ OAUTH_ID_TOKEN_EXPIRATION = 5 * 60
 
 # Partner support link for CMS footer
 PARTNER_SUPPORT_EMAIL = ''
+
+
+################################ Settings for Credentials Service ################################
+
+CREDENTIALS_SERVICE_USERNAME = 'credentials_service_user'

lms/djangoapps/certificates/api.py

@@ -34,6 +34,15 @@ from branding import api as branding_api
 log = logging.getLogger("edx.certificate")
 
 
+def is_passing_status(cert_status):
+    """
+    Given the status of a certificate, return a boolean indicating whether
+    the student passed the course.  This just proxies to the classmethod
+    defined in models.py
+    """
+    return CertificateStatuses.is_passing_status(cert_status)
+
+
 def get_certificates_for_user(username):
     """
     Retrieve certificate information for a particular user.
@@ -116,7 +125,7 @@ def generate_user_certificates(student, course_key, course=None, insecure=False,
         generate_pdf=generate_pdf,
         forced_grade=forced_grade
     )
-    if cert.status in [CertificateStatuses.generating, CertificateStatuses.downloadable]:
+    if CertificateStatuses.is_passing_status(cert.status):
         emit_certificate_event('created', student, course_key, course, {
             'user_id': student.id,
             'course_id': unicode(course_key),

lms/djangoapps/certificates/models.py

@@ -97,6 +97,14 @@ class CertificateStatuses(object):
         error: "error states"
     }
 
+    @classmethod
+    def is_passing_status(cls, status):
+        """
+        Given the status of a certificate, return a boolean indicating whether
+        the student passed the course.
+        """
+        return status in [cls.downloadable, cls.generating]
+
 
 class CertificateSocialNetworks(object):
     """
@@ -297,13 +305,10 @@ class GeneratedCertificate(models.Model):
     def save(self, *args, **kwargs):
         """
         After the base save() method finishes, fire the COURSE_CERT_AWARDED
-        signal iff we have stored a record of a learner passing the course.
-
-        The learner is assumed to have passed the course if certificate status
-        is either 'generating' or 'downloadable'.
+        signal iff we are saving a record of a learner passing the course.
         """
         super(GeneratedCertificate, self).save(*args, **kwargs)
-        if self.status in [CertificateStatuses.generating, CertificateStatuses.downloadable]:
+        if CertificateStatuses.is_passing_status(self.status):
             COURSE_CERT_AWARDED.send_robust(
                 sender=self.__class__,
                 user=self.user,

lms/djangoapps/instructor_task/tasks_helper.py

@@ -1448,7 +1448,7 @@ def generate_students_certificates(
             course=course
         )
 
-        if status in [CertificateStatuses.generating, CertificateStatuses.downloadable]:
+        if CertificateStatuses.is_passing_status(status):
             task_progress.succeeded += 1
         else:
             task_progress.failed += 1

lms/envs/common.py

@@ -2750,3 +2750,8 @@ REGISTRATION_EXTENSION_FORM = None
 MOBILE_APP_USER_AGENT_REGEXES = [
     r'edX/org.edx.mobile',
 ]
+
+
+################################ Settings for Credentials Service ################################
+
+CREDENTIALS_SERVICE_USERNAME = 'credentials_service_user'

openedx/core/djangoapps/programs/signals.py

@@ -47,5 +47,5 @@ def handle_course_cert_awarded(sender, user, course_key, mode, status, **kwargs)
         status,
     )
     # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
-    from openedx.core.djangoapps.programs import tasks
-    tasks.award_program_certificates.delay(user.username)
+    from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates
+    award_program_certificates.delay(user.username)

openedx/core/djangoapps/programs/tasks.py

-"""
-This file contains celery tasks for programs-related functionality.
-"""
-
-from celery import task
-from celery.utils.log import get_task_logger  # pylint: disable=no-name-in-module, import-error
-
-from lms.djangoapps.certificates.api import get_certificates_for_user
-
-LOGGER = get_task_logger(__name__)
-
-
-@task
-def award_program_certificates(username):
-    """
-    This task is designed to be called whenever a user's completion status
-    changes with respect to one or more courses (primarily, when a course
-    certificate is awarded).
-
-    It will consult with a variety of APIs to determine whether or not the
-    specified user should be awarded a certificate in one or more programs, and
-    use the credentials service to create said certificates if so.
-
-    This task may also be invoked independently of any course completion status
-    change - for example, to backpopulate missing program credentials for a
-    user.
-
-    TODO: this is shelled out and incomplete for now.
-    """
-
-    # fetch the set of all course runs for which the user has earned a certificate
-    LOGGER.debug('fetching all completed courses for user %s', username)
-    user_certs = get_certificates_for_user(username)
-    course_certs = [
-        {'course_id': uc['course_id'], 'mode': uc['mode']}
-        for uc in user_certs
-        if uc['status'] in ('downloadable', 'generating')
-    ]
-
-    # invoke the Programs API completion check endpoint to identify any programs
-    # that are satisfied by these course completions
-    LOGGER.debug('determining completed programs for courses: %r', course_certs)
-    program_ids = []  # TODO
-
-    # determine which program certificates the user has already been awarded, if
-    # any, and remove those, since they already exist.
-    LOGGER.debug('fetching existing program certificates for %s', username)
-    existing_program_ids = []  # TODO
-    new_program_ids = list(set(program_ids) - set(existing_program_ids))
-
-    # generate a new certificate for each of the remaining programs.
-    LOGGER.debug('generating new program certificates for %s in programs: %r', username, new_program_ids)
-    for program_id in new_program_ids:
-        LOGGER.debug('calling credentials service to issue certificate for user %s in program %s', username, program_id)
-        # TODO

openedx/core/djangoapps/programs/tasks/v1/tasks.py

+"""
+This file contains celery tasks for programs-related functionality.
+"""
+
+from celery import task
+from celery.utils.log import get_task_logger  # pylint: disable=no-name-in-module, import-error
+from django.conf import settings
+from django.contrib.auth.models import User
+from edx_rest_api_client.client import EdxRestApiClient
+
+from lms.djangoapps.certificates.api import get_certificates_for_user, is_passing_status
+
+from openedx.core.djangoapps.credentials.models import CredentialsApiConfig
+from openedx.core.djangoapps.credentials.utils import get_user_credentials
+from openedx.core.djangoapps.programs.models import ProgramsApiConfig
+from openedx.core.lib.token_utils import get_id_token
+
+
+LOGGER = get_task_logger(__name__)
+
+
+def get_api_client(api_config, student):
+    """
+    Create and configure an API client for authenticated HTTP requests.
+
+    Args:
+        api_config: ProgramsApiConfig or CredentialsApiConfig object
+        student: User object as whom to authenticate to the API
+
+    Returns:
+        EdxRestApiClient
+
+    """
+    id_token = get_id_token(student, api_config.OAUTH2_CLIENT_NAME)
+    return EdxRestApiClient(api_config.internal_api_url, jwt=id_token)
+
+
+def get_completed_courses(student):
+    """
+    Determine which courses have been completed by the user.
+
+    Args:
+        student:
+            User object representing the student
+
+    Returns:
+        iterable of dicts with structure {'course_id': course_key, 'mode': cert_type}
+
+    """
+    all_certs = get_certificates_for_user(student.username)
+    return [
+        {'course_id': cert['course_key'], 'mode': cert['type']}
+        for cert in all_certs
+        if is_passing_status(cert['status'])
+    ]
+
+
+def get_completed_programs(client, course_certificates):
+    """
+    Given a set of completed courses, determine which programs are completed.
+
+    Args:
+        client:
+            programs API client (EdxRestApiClient)
+        course_certificates:
+            iterable of dicts with structure {'course_id': course_key, 'mode': cert_type}
+
+    Returns:
+        list of program ids
+
+    """
+    return client.programs.complete.post({'completed_courses': course_certificates})['program_ids']
+
+
+def get_awarded_certificate_programs(student):
+    """
+    Find the ids of all the programs for which the student has already been awarded
+    a certificate.
+
+    Args:
+        student:
+            User object representing the student
+
+    Returns:
+        ids of the programs for which the student has been awarded a certificate
+
+    """
+    return [
+        credential['credential']['program_id']
+        for credential in get_user_credentials(student)
+        if 'program_id' in credential['credential'] and credential['status'] == 'awarded'
+    ]
+
+
+def award_program_certificate(client, username, program_id):
+    """
+    Issue a new certificate of completion to the given student for the given program.
+
+    Args:
+        client:
+            credentials API client (EdxRestApiClient)
+        username:
+            The username of the student
+        program_id:
+            id of the completed program
+
+    Returns:
+        None
+
+    """
+    client.user_credentials.post({'program_id': program_id, 'username': username})
+
+
+@task(bind=True, ignore_result=True)
+def award_program_certificates(self, username):
+    """
+    This task is designed to be called whenever a student's completion status
+    changes with respect to one or more courses (primarily, when a course
+    certificate is awarded).
+
+    It will consult with a variety of APIs to determine whether or not the
+    specified user should be awarded a certificate in one or more programs, and
+    use the credentials service to create said certificates if so.
+
+    This task may also be invoked independently of any course completion status
+    change - for example, to backpopulate missing program credentials for a
+    student.
+
+    Args:
+        username:
+            The username of the student
+
+    Returns:
+        None
+
+    """
+    LOGGER.info('Running task award_program_certificates for username %s', username)
+
+    # If either programs or credentials config models are disabled for this
+    # feature, this task should not have been invoked in the first place, and
+    # an error somewhere is likely (though a race condition is also possible).
+    # In either case, the task should not be executed nor should it be retried.
+    if not ProgramsApiConfig.current().is_certification_enabled:
+        LOGGER.warning(
+            'Task award_program_certificates cannot be executed when program certification is disabled in API config',
+        )
+        return
+
+    if not CredentialsApiConfig.current().is_learner_issuance_enabled:
+        LOGGER.warning(
+            'Task award_program_certificates cannot be executed when credentials issuance is disabled in API config',
+        )
+        return
+
+    try:
+        try:
+            student = User.objects.get(username=username)
+        except User.DoesNotExist:
+            LOGGER.exception('Task award_program_certificates was called with invalid username %s', username)
+            # Don't retry for this case - just conclude the task.
+            return
+
+        # Fetch the set of all course runs for which the user has earned a
+        # certificate.
+        course_certs = get_completed_courses(student)
+        if not course_certs:
+            # Highly unlikely, since at present the only trigger for this task
+            # is the earning of a new course certificate.  However, it could be
+            # that the transaction in which a course certificate was awarded
+            # was subsequently rolled back, which could lead to an empty result
+            # here, so we'll at least log that this happened before exiting.
+            #
+            # If this task is ever updated to support revocation of program
+            # certs, this branch should be removed, since it could make sense
+            # in that case to call this task for a user without any (valid)
+            # course certs.
+            LOGGER.warning('Task award_program_certificates was called for user %s with no completed courses', username)
+            return
+
+        # Invoke the Programs API completion check endpoint to identify any
+        # programs that are satisfied by these course completions.
+        programs_client = get_api_client(ProgramsApiConfig.current(), student)
+        program_ids = get_completed_programs(programs_client, course_certs)
+        if not program_ids:
+            # Again, no reason to continue beyond this point unless/until this
+            # task gets updated to support revocation of program certs.
+            return
+
+        # Determine which program certificates the user has already been
+        # awarded, if any.
+        existing_program_ids = get_awarded_certificate_programs(student)
+
+    except Exception, exc:  # pylint: disable=broad-except
+        LOGGER.exception('Failed to determine program certificates to be awarded for user %s', username)
+        raise self.retry(exc=exc)
+
+    # For each completed program for which the student doesn't already have a
+    # certificate, award one now.
+    #
+    # N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
+    new_program_ids = sorted(list(set(program_ids) - set(existing_program_ids)))
+    if new_program_ids:
+        try:
+            credentials_client = get_api_client(
+                CredentialsApiConfig.current(),
+                User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME)  # pylint: disable=no-member
+            )
+        except Exception, exc:  # pylint: disable=broad-except
+            LOGGER.exception('Failed to create a credentials API client to award program certificates')
+            # Retry because a misconfiguration could be fixed
+            raise self.retry(exc=exc)
+
+        for program_id in new_program_ids:
+            try:
+                award_program_certificate(credentials_client, username, program_id)
+                LOGGER.info('Awarded certificate for program %s to user %s', program_id, username)
+            except Exception:  # pylint: disable=broad-except
+                # keep trying to award other certs.
+                LOGGER.exception('Failed to award certificate for program %s to user %s', program_id, username)

openedx/core/djangoapps/programs/tests/test_signals.py

@@ -14,7 +14,7 @@ from openedx.core.djangoapps.programs.signals import handle_course_cert_awarded
 TEST_USERNAME = 'test-user'
 
 
-@mock.patch('openedx.core.djangoapps.programs.tasks.award_program_certificates.delay')
+@mock.patch('openedx.core.djangoapps.programs.tasks.v1.tasks.award_program_certificates.delay')
 @mock.patch(
     'openedx.core.djangoapps.programs.models.ProgramsApiConfig.is_certification_enabled',
     new_callable=mock.PropertyMock,