Implement celery task to award program certs.

ECOM-3354

cms/envs/common.py

@@ -1113,3 +1113,8 @@ OAUTH_ID_TOKEN_EXPIRATION = 5 * 60
 
 # Partner support link for CMS footer
 PARTNER_SUPPORT_EMAIL = ''
+
+
+################################ Settings for Credentials Service ################################
+
+CREDENTIALS_SERVICE_USERNAME = 'credentials_service_user'

lms/djangoapps/certificates/api.py

@@ -34,6 +34,15 @@ from branding import api as branding_api
 log = logging.getLogger("edx.certificate")
 
 
+def is_passing_status(cert_status):
+    """
+    Given the status of a certificate, return a boolean indicating whether
+    the student passed the course.  This just proxies to the classmethod
+    defined in models.py
+    """
+    return CertificateStatuses.is_passing_status(cert_status)
+
+
 def get_certificates_for_user(username):
     """
     Retrieve certificate information for a particular user.
@@ -116,7 +125,7 @@ def generate_user_certificates(student, course_key, course=None, insecure=False,
         generate_pdf=generate_pdf,
         forced_grade=forced_grade
     )
-    if cert.status in [CertificateStatuses.generating, CertificateStatuses.downloadable]:
+    if CertificateStatuses.is_passing_status(cert.status):
         emit_certificate_event('created', student, course_key, course, {
             'user_id': student.id,
             'course_id': unicode(course_key),

lms/djangoapps/certificates/models.py

@@ -97,6 +97,14 @@ class CertificateStatuses(object):
         error: "error states"
     }
 
+    @classmethod
+    def is_passing_status(cls, status):
+        """
+        Given the status of a certificate, return a boolean indicating whether
+        the student passed the course.
+        """
+        return status in [cls.downloadable, cls.generating]
+
 
 class CertificateSocialNetworks(object):
     """
@@ -297,13 +305,10 @@ class GeneratedCertificate(models.Model):
     def save(self, *args, **kwargs):
         """
         After the base save() method finishes, fire the COURSE_CERT_AWARDED
-        signal iff we have stored a record of a learner passing the course.
-
-        The learner is assumed to have passed the course if certificate status
-        is either 'generating' or 'downloadable'.
+        signal iff we are saving a record of a learner passing the course.
         """
         super(GeneratedCertificate, self).save(*args, **kwargs)
-        if self.status in [CertificateStatuses.generating, CertificateStatuses.downloadable]:
+        if CertificateStatuses.is_passing_status(self.status):
             COURSE_CERT_AWARDED.send_robust(
                 sender=self.__class__,
                 user=self.user,

lms/djangoapps/instructor_task/tasks_helper.py

@@ -1448,7 +1448,7 @@ def generate_students_certificates(
             course=course
         )
 
-        if status in [CertificateStatuses.generating, CertificateStatuses.downloadable]:
+        if CertificateStatuses.is_passing_status(status):
             task_progress.succeeded += 1
         else:
             task_progress.failed += 1

lms/envs/common.py

@@ -2750,3 +2750,8 @@ REGISTRATION_EXTENSION_FORM = None
 MOBILE_APP_USER_AGENT_REGEXES = [
     r'edX/org.edx.mobile',
 ]
+
+
+################################ Settings for Credentials Service ################################
+
+CREDENTIALS_SERVICE_USERNAME = 'credentials_service_user'

openedx/core/djangoapps/programs/signals.py

@@ -47,5 +47,5 @@ def handle_course_cert_awarded(sender, user, course_key, mode, status, **kwargs)
         status,
     )
     # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
-    from openedx.core.djangoapps.programs import tasks
-    tasks.award_program_certificates.delay(user.username)
+    from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates
+    award_program_certificates.delay(user.username)

openedx/core/djangoapps/programs/tasks.py

-"""
-This file contains celery tasks for programs-related functionality.
-"""
-
-from celery import task
-from celery.utils.log import get_task_logger  # pylint: disable=no-name-in-module, import-error
-
-from lms.djangoapps.certificates.api import get_certificates_for_user
-
-LOGGER = get_task_logger(__name__)
-
-
-@task
-def award_program_certificates(username):
-    """
-    This task is designed to be called whenever a user's completion status
-    changes with respect to one or more courses (primarily, when a course
-    certificate is awarded).
-
-    It will consult with a variety of APIs to determine whether or not the
-    specified user should be awarded a certificate in one or more programs, and
-    use the credentials service to create said certificates if so.
-
-    This task may also be invoked independently of any course completion status
-    change - for example, to backpopulate missing program credentials for a
-    user.
-
-    TODO: this is shelled out and incomplete for now.
-    """
-
-    # fetch the set of all course runs for which the user has earned a certificate
-    LOGGER.debug('fetching all completed courses for user %s', username)
-    user_certs = get_certificates_for_user(username)
-    course_certs = [
-        {'course_id': uc['course_id'], 'mode': uc['mode']}
-        for uc in user_certs
-        if uc['status'] in ('downloadable', 'generating')
-    ]
-
-    # invoke the Programs API completion check endpoint to identify any programs
-    # that are satisfied by these course completions
-    LOGGER.debug('determining completed programs for courses: %r', course_certs)
-    program_ids = []  # TODO
-
-    # determine which program certificates the user has already been awarded, if
-    # any, and remove those, since they already exist.
-    LOGGER.debug('fetching existing program certificates for %s', username)
-    existing_program_ids = []  # TODO
-    new_program_ids = list(set(program_ids) - set(existing_program_ids))
-
-    # generate a new certificate for each of the remaining programs.
-    LOGGER.debug('generating new program certificates for %s in programs: %r', username, new_program_ids)
-    for program_id in new_program_ids:
-        LOGGER.debug('calling credentials service to issue certificate for user %s in program %s', username, program_id)
-        # TODO

openedx/core/djangoapps/programs/tasks/v1/tasks.py

+"""
+This file contains celery tasks for programs-related functionality.
+"""
+
+from celery import task
+from celery.utils.log import get_task_logger  # pylint: disable=no-name-in-module, import-error
+from django.conf import settings
+from django.contrib.auth.models import User
+from edx_rest_api_client.client import EdxRestApiClient
+
+from lms.djangoapps.certificates.api import get_certificates_for_user, is_passing_status
+
+from openedx.core.djangoapps.credentials.models import CredentialsApiConfig
+from openedx.core.djangoapps.credentials.utils import get_user_credentials
+from openedx.core.djangoapps.programs.models import ProgramsApiConfig
+from openedx.core.lib.token_utils import get_id_token
+
+
+LOGGER = get_task_logger(__name__)
+
+
+def get_api_client(api_config, student):
+    """
+    Create and configure an API client for authenticated HTTP requests.
+
+    Args:
+        api_config: ProgramsApiConfig or CredentialsApiConfig object
+        student: User object as whom to authenticate to the API
+
+    Returns:
+        EdxRestApiClient
+
+    """
+    id_token = get_id_token(student, api_config.OAUTH2_CLIENT_NAME)
+    return EdxRestApiClient(api_config.internal_api_url, jwt=id_token)
+
+
+def get_completed_courses(student):
+    """
+    Determine which courses have been completed by the user.
+
+    Args:
+        student:
+            User object representing the student
+
+    Returns:
+        iterable of dicts with structure {'course_id': course_key, 'mode': cert_type}
+
+    """
+    all_certs = get_certificates_for_user(student.username)
+    return [
+        {'course_id': cert['course_key'], 'mode': cert['type']}
+        for cert in all_certs
+        if is_passing_status(cert['status'])
+    ]
+
+
+def get_completed_programs(client, course_certificates):
+    """
+    Given a set of completed courses, determine which programs are completed.
+
+    Args:
+        client:
+            programs API client (EdxRestApiClient)
+        course_certificates:
+            iterable of dicts with structure {'course_id': course_key, 'mode': cert_type}
+
+    Returns:
+        list of program ids
+
+    """
+    return client.programs.complete.post({'completed_courses': course_certificates})['program_ids']
+
+
+def get_awarded_certificate_programs(student):
+    """
+    Find the ids of all the programs for which the student has already been awarded
+    a certificate.
+
+    Args:
+        student:
+            User object representing the student
+
+    Returns:
+        ids of the programs for which the student has been awarded a certificate
+
+    """
+    return [
+        credential['credential']['program_id']
+        for credential in get_user_credentials(student)
+        if 'program_id' in credential['credential'] and credential['status'] == 'awarded'
+    ]
+
+
+def award_program_certificate(client, username, program_id):
+    """
+    Issue a new certificate of completion to the given student for the given program.
+
+    Args:
+        client:
+            credentials API client (EdxRestApiClient)
+        username:
+            The username of the student
+        program_id:
+            id of the completed program
+
+    Returns:
+        None
+
+    """
+    client.user_credentials.post({'program_id': program_id, 'username': username})
+
+
+@task(bind=True, ignore_result=True)
+def award_program_certificates(self, username):
+    """
+    This task is designed to be called whenever a student's completion status
+    changes with respect to one or more courses (primarily, when a course
+    certificate is awarded).
+
+    It will consult with a variety of APIs to determine whether or not the
+    specified user should be awarded a certificate in one or more programs, and
+    use the credentials service to create said certificates if so.
+
+    This task may also be invoked independently of any course completion status
+    change - for example, to backpopulate missing program credentials for a
+    student.
+
+    Args:
+        username:
+            The username of the student
+
+    Returns:
+        None
+
+    """
+    LOGGER.info('Running task award_program_certificates for username %s', username)
+
+    # If either programs or credentials config models are disabled for this
+    # feature, this task should not have been invoked in the first place, and
+    # an error somewhere is likely (though a race condition is also possible).
+    # In either case, the task should not be executed nor should it be retried.
+    if not ProgramsApiConfig.current().is_certification_enabled:
+        LOGGER.warning(
+            'Task award_program_certificates cannot be executed when program certification is disabled in API config',
+        )
+        return
+
+    if not CredentialsApiConfig.current().is_learner_issuance_enabled:
+        LOGGER.warning(
+            'Task award_program_certificates cannot be executed when credentials issuance is disabled in API config',
+        )
+        return
+
+    try:
+        try:
+            student = User.objects.get(username=username)
+        except User.DoesNotExist:
+            LOGGER.exception('Task award_program_certificates was called with invalid username %s', username)
+            # Don't retry for this case - just conclude the task.
+            return
+
+        # Fetch the set of all course runs for which the user has earned a
+        # certificate.
+        course_certs = get_completed_courses(student)
+        if not course_certs:
+            # Highly unlikely, since at present the only trigger for this task
+            # is the earning of a new course certificate.  However, it could be
+            # that the transaction in which a course certificate was awarded
+            # was subsequently rolled back, which could lead to an empty result
+            # here, so we'll at least log that this happened before exiting.
+            #
+            # If this task is ever updated to support revocation of program
+            # certs, this branch should be removed, since it could make sense
+            # in that case to call this task for a user without any (valid)
+            # course certs.
+            LOGGER.warning('Task award_program_certificates was called for user %s with no completed courses', username)
+            return
+
+        # Invoke the Programs API completion check endpoint to identify any
+        # programs that are satisfied by these course completions.
+        programs_client = get_api_client(ProgramsApiConfig.current(), student)
+        program_ids = get_completed_programs(programs_client, course_certs)
+        if not program_ids:
+            # Again, no reason to continue beyond this point unless/until this
+            # task gets updated to support revocation of program certs.
+            return
+
+        # Determine which program certificates the user has already been
+        # awarded, if any.
+        existing_program_ids = get_awarded_certificate_programs(student)
+
+    except Exception, exc:  # pylint: disable=broad-except
+        LOGGER.exception('Failed to determine program certificates to be awarded for user %s', username)
+        raise self.retry(exc=exc)
+
+    # For each completed program for which the student doesn't already have a
+    # certificate, award one now.
+    #
+    # N.B. the list is sorted to facilitate deterministic ordering, e.g. for tests.
+    new_program_ids = sorted(list(set(program_ids) - set(existing_program_ids)))
+    if new_program_ids:
+        try:
+            credentials_client = get_api_client(
+                CredentialsApiConfig.current(),
+                User.objects.get(username=settings.CREDENTIALS_SERVICE_USERNAME)  # pylint: disable=no-member
+            )
+        except Exception, exc:  # pylint: disable=broad-except
+            LOGGER.exception('Failed to create a credentials API client to award program certificates')
+            # Retry because a misconfiguration could be fixed
+            raise self.retry(exc=exc)
+
+        for program_id in new_program_ids:
+            try:
+                award_program_certificate(credentials_client, username, program_id)
+                LOGGER.info('Awarded certificate for program %s to user %s', program_id, username)
+            except Exception:  # pylint: disable=broad-except
+                # keep trying to award other certs.
+                LOGGER.exception('Failed to award certificate for program %s to user %s', program_id, username)

openedx/core/djangoapps/programs/tasks/v1/tests/test_tasks.py

+"""
+Tests for programs celery tasks.
+"""
+
+import ddt
+from django.conf import settings
+from django.test import override_settings, TestCase
+from edx_rest_api_client.client import EdxRestApiClient
+import httpretty
+import json
+import mock
+
+from oauth2_provider.tests.factories import ClientFactory
+from openedx.core.djangoapps.credentials.tests.mixins import CredentialsApiConfigMixin
+from openedx.core.djangoapps.programs.tests.mixins import ProgramsApiConfigMixin
+from openedx.core.djangoapps.programs.tasks.v1 import tasks
+from student.tests.factories import UserFactory
+
+
+TASKS_MODULE = 'openedx.core.djangoapps.programs.tasks.v1.tasks'
+
+
+class GetApiClientTestCase(TestCase, ProgramsApiConfigMixin):
+    """
+    Test the get_api_client function
+    """
+
+    @mock.patch(TASKS_MODULE + '.get_id_token')
+    def test_get_api_client(self, mock_get_id_token):
+        """
+        Ensure the function is making the right API calls based on inputs
+        """
+        student = UserFactory()
+        ClientFactory.create(name='programs')
+        api_config = self.create_programs_config(
+            internal_service_url='http://foo',
+            api_version_number=99,
+        )
+        mock_get_id_token.return_value = 'test-token'
+
+        api_client = tasks.get_api_client(api_config, student)
+        self.assertEqual(mock_get_id_token.call_args[0], (student, 'programs'))
+        self.assertEqual(api_client._store['base_url'], 'http://foo/api/v99/')  # pylint: disable=protected-access
+        self.assertEqual(api_client._store['session'].auth.token, 'test-token')  # pylint: disable=protected-access
+
+
+class GetCompletedCoursesTestCase(TestCase):
+    """
+    Test the get_completed_courses function
+    """
+
+    def make_cert_result(self, **kwargs):
+        """
+        Helper to create dummy results from the certificates API
+        """
+        result = {
+            'username': 'dummy-username',
+            'course_key': 'dummy-course',
+            'type': 'dummy-type',
+            'status': 'dummy-status',
+            'download_url': 'http://www.example.com/cert.pdf',
+            'grade': '0.98',
+            'created': '2015-07-31T00:00:00Z',
+            'modified': '2015-07-31T00:00:00Z',
+        }
+        result.update(**kwargs)
+        return result
+
+    @mock.patch(TASKS_MODULE + '.get_certificates_for_user')
+    def test_get_completed_courses(self, mock_get_certs_for_user):
+        """
+        Ensure the function correctly calls to and handles results from the
+        certificates API
+        """
+        student = UserFactory(username='test-username')
+        mock_get_certs_for_user.return_value = [
+            self.make_cert_result(status='downloadable', type='verified', course_key='downloadable-course'),
+            self.make_cert_result(status='generating', type='prof-ed', course_key='generating-course'),
+            self.make_cert_result(status='unknown', type='honor', course_key='unknown-course'),
+        ]
+
+        result = tasks.get_completed_courses(student)
+        self.assertEqual(mock_get_certs_for_user.call_args[0], (student.username, ))
+        self.assertEqual(result, [
+            {'course_id': 'downloadable-course', 'mode': 'verified'},
+            {'course_id': 'generating-course', 'mode': 'prof-ed'},
+        ])
+
+
+class GetCompletedProgramsTestCase(TestCase):
+    """
+    Test the get_completed_programs function
+    """
+
+    @httpretty.activate
+    def test_get_completed_programs(self):
+        """
+        Ensure the correct API call gets made
+        """
+        test_client = EdxRestApiClient('http://test-server', jwt='test-token')
+        httpretty.register_uri(
+            httpretty.POST,
+            'http://test-server/programs/complete/',
+            body='{"program_ids": [1, 2, 3]}',
+            content_type='application/json',
+        )
+        payload = [
+            {'course_id': 'test-course-1', 'mode': 'verified'},
+            {'course_id': 'test-course-2', 'mode': 'prof-ed'},
+        ]
+        result = tasks.get_completed_programs(test_client, payload)
+        self.assertEqual(httpretty.last_request().body, json.dumps({'completed_courses': payload}))
+        self.assertEqual(result, [1, 2, 3])
+
+
+class GetAwardedCertificateProgramsTestCase(TestCase):
+    """
+    Test the get_awarded_certificate_programs function
+    """
+
+    def make_credential_result(self, **kwargs):
+        """
+        Helper to make dummy results from the credentials API
+        """
+        result = {
+            'id': 1,
+            'username': 'dummy-username',
+            'credential': {
+                'credential_id': None,
+                'program_id': None,
+            },
+            'status': 'dummy-status',
+            'uuid': 'dummy-uuid',
+            'certificate_url': 'http://credentials.edx.org/credentials/dummy-uuid/'
+        }
+        result.update(**kwargs)
+        return result
+
+    @mock.patch(TASKS_MODULE + '.get_user_credentials')
+    def test_get_awarded_certificate_programs(self, mock_get_user_credentials):
+        """
+        Ensure the API is called and results handled correctly.
+        """
+        student = UserFactory(username='test-username')
+        mock_get_user_credentials.return_value = [
+            self.make_credential_result(status='awarded', credential={'program_id': 1}),
+            self.make_credential_result(status='awarded', credential={'course_id': 2}),
+            self.make_credential_result(status='revoked', credential={'program_id': 3}),
+        ]
+
+        result = tasks.get_awarded_certificate_programs(student)
+        self.assertEqual(mock_get_user_credentials.call_args[0], (student, ))
+        self.assertEqual(result, [1])
+
+
+class AwardProgramCertificateTestCase(TestCase):
+    """
+    Test the award_program_certificate function
+    """
+
+    @httpretty.activate
+    def test_award_program_certificate(self):
+        """
+        Ensure the correct API call gets made
+        """
+        test_username = 'test-username'
+        test_client = EdxRestApiClient('http://test-server', jwt='test-token')
+        httpretty.register_uri(
+            httpretty.POST,
+            'http://test-server/user_credentials/',
+        )
+        tasks.award_program_certificate(test_client, test_username, 123)
+        self.assertEqual(httpretty.last_request().body, json.dumps({'program_id': 123, 'username': test_username}))
+
+
+@ddt.ddt
+@mock.patch(TASKS_MODULE + '.award_program_certificate')
+@mock.patch(TASKS_MODULE + '.get_awarded_certificate_programs')
+@mock.patch(TASKS_MODULE + '.get_completed_programs')
+@mock.patch(TASKS_MODULE + '.get_completed_courses')
+@override_settings(CREDENTIALS_SERVICE_USERNAME='test-service-username')
+class AwardProgramCertificatesTestCase(TestCase, ProgramsApiConfigMixin, CredentialsApiConfigMixin):
+    """
+    Tests for the 'award_program_certificates' celery task.
+    """
+
+    def setUp(self):
+        super(AwardProgramCertificatesTestCase, self).setUp()
+        self.create_programs_config()
+        self.create_credentials_config()
+        self.student = UserFactory.create(username='test-student')
+
+        ClientFactory.create(name='programs')
+        ClientFactory.create(name='credentials')
+        UserFactory.create(username=settings.CREDENTIALS_SERVICE_USERNAME)  # pylint: disable=no-member
+
+    def test_completion_check(
+            self,
+            mock_get_completed_courses,
+            mock_get_completed_programs,
+            mock_get_awarded_certificate_programs,  # pylint: disable=unused-argument
+            mock_award_program_certificate,  # pylint: disable=unused-argument
+    ):
+        """
+        Checks that the Programs API is used correctly to determine completed
+        programs.
+        """
+        completed_courses = [
+            {'course_id': 'course-1', 'type': 'verified'},
+            {'course_id': 'course-2', 'type': 'prof-ed'},
+        ]
+        mock_get_completed_courses.return_value = completed_courses
+
+        tasks.award_program_certificates.delay(self.student.username).get()
+
+        self.assertEqual(
+            mock_get_completed_programs.call_args[0][1],
+            completed_courses
+        )
+
+    @ddt.data(
+        ([1], [2, 3]),
+        ([], [1, 2, 3]),
+        ([1, 2, 3], []),
+    )
+    @ddt.unpack
+    def test_awarding_certs(
+            self,
+            already_awarded_program_ids,
+            expected_awarded_program_ids,
+            mock_get_completed_courses,  # pylint: disable=unused-argument
+            mock_get_completed_programs,
+            mock_get_awarded_certificate_programs,
+            mock_award_program_certificate,
+    ):
+        """
+        Checks that the Credentials API is used to award certificates for
+        the proper programs.
+        """
+        mock_get_completed_programs.return_value = [1, 2, 3]
+        mock_get_awarded_certificate_programs.return_value = already_awarded_program_ids
+
+        tasks.award_program_certificates.delay(self.student.username).get()
+
+        actual_program_ids = [call[0][2] for call in mock_award_program_certificate.call_args_list]
+        self.assertEqual(actual_program_ids, expected_awarded_program_ids)
+
+    @ddt.data(
+        ('programs', 'enable_certification'),
+        ('credentials', 'enable_learner_issuance'),
+    )
+    @ddt.unpack
+    def test_abort_if_config_disabled(
+            self,
+            disabled_config_type,
+            disabled_config_attribute,
+            *mock_helpers
+    ):
+        """
+        Checks that the task is aborted if any relevant api configs are
+        disabled.
+        """
+        getattr(self, 'create_{}_config'.format(disabled_config_type))(**{disabled_config_attribute: False})
+        with mock.patch(TASKS_MODULE + '.LOGGER.warning') as mock_warning:
+            tasks.award_program_certificates.delay(self.student.username).get()
+            self.assertTrue(mock_warning.called)
+        for mock_helper in mock_helpers:
+            self.assertFalse(mock_helper.called)
+
+    def test_abort_if_invalid_username(self, *mock_helpers):
+        """
+        Checks that the task will be aborted and not retried if the username
+        passed was not found, and that an exception is logged.
+        """
+        with mock.patch(TASKS_MODULE + '.LOGGER.exception') as mock_exception:
+            tasks.award_program_certificates.delay('nonexistent-username').get()
+            self.assertTrue(mock_exception.called)
+        for mock_helper in mock_helpers:
+            self.assertFalse(mock_helper.called)
+
+    def test_abort_if_no_completed_courses(
+            self,
+            mock_get_completed_courses,
+            mock_get_completed_programs,
+            mock_get_awarded_certificate_programs,
+            mock_award_program_certificate,
+    ):
+        """
+        Checks that the task will be aborted without further action if the
+        student does not have any completed courses, but that a warning is
+        logged.
+        """
+        mock_get_completed_courses.return_value = []
+        with mock.patch(TASKS_MODULE + '.LOGGER.warning') as mock_warning:
+            tasks.award_program_certificates.delay(self.student.username).get()
+            self.assertTrue(mock_warning.called)
+        self.assertTrue(mock_get_completed_courses.called)
+        self.assertFalse(mock_get_completed_programs.called)
+        self.assertFalse(mock_get_awarded_certificate_programs.called)
+        self.assertFalse(mock_award_program_certificate.called)
+
+    def test_abort_if_no_completed_programs(
+            self,
+            mock_get_completed_courses,
+            mock_get_completed_programs,
+            mock_get_awarded_certificate_programs,
+            mock_award_program_certificate,
+    ):
+        """
+        Checks that the task will be aborted without further action if there
+        are no programs for which to award a certificate.
+        """
+        mock_get_completed_programs.return_value = []
+        tasks.award_program_certificates.delay(self.student.username).get()
+        self.assertTrue(mock_get_completed_courses.called)
+        self.assertTrue(mock_get_completed_programs.called)
+        self.assertFalse(mock_get_awarded_certificate_programs.called)
+        self.assertFalse(mock_award_program_certificate.called)
+
+    def _make_side_effect(self, side_effects):
+        """
+        DRY helper.  Returns a side effect function for use with mocks that
+        will be called multiple times, permitting Exceptions to be raised
+        (or not) in a specified order.
+
+        See Also:
+            http://www.voidspace.org.uk/python/mock/examples.html#multiple-calls-with-different-effects
+            http://www.voidspace.org.uk/python/mock/mock.html#mock.Mock.side_effect
+
+        """
+        def side_effect(*_a):  # pylint: disable=missing-docstring
+            exc = side_effects.pop(0)
+            if exc:
+                raise exc
+            return mock.DEFAULT
+        return side_effect
+
+    def test_continue_awarding_certs_if_error(
+            self,
+            mock_get_completed_courses,  # pylint: disable=unused-argument
+            mock_get_completed_programs,
+            mock_get_awarded_certificate_programs,
+            mock_award_program_certificate,
+    ):
+        """
+        Checks that a single failure to award one of several certificates
+        does not cause the entire task to fail.  Also ensures that
+        successfully awarded certs are logged as INFO and exceptions
+        that arise are logged also.
+        """
+        mock_get_completed_programs.return_value = [1, 2]
+        mock_get_awarded_certificate_programs.return_value = []
+        mock_award_program_certificate.side_effect = self._make_side_effect([Exception('boom'), None])
+
+        with mock.patch(TASKS_MODULE + '.LOGGER.info') as mock_info, \
+                mock.patch(TASKS_MODULE + '.LOGGER.exception') as mock_exception:
+            tasks.award_program_certificates.delay(self.student.username).get()
+
+        self.assertEqual(mock_award_program_certificate.call_count, 2)
+        mock_exception.assert_called_once_with(mock.ANY, 1, self.student.username)
+        mock_info.assert_called_with(mock.ANY, 2, self.student.username)
+
+    def test_retry_on_certificates_api_errors(
+            self,
+            mock_get_completed_courses,
+            *_mock_helpers  # pylint: disable=unused-argument
+    ):
+        """
+        Ensures that any otherwise-unhandled errors that arise while trying
+        to get existing course certificates (e.g. network issues or other
+        transient API errors) will cause the task to be failed and queued for
+        retry.
+        """
+        mock_get_completed_courses.side_effect = self._make_side_effect([Exception('boom'), None])
+        tasks.award_program_certificates.delay(self.student.username).get()
+        self.assertEqual(mock_get_completed_courses.call_count, 2)
+
+    def test_retry_on_programs_api_errors(
+            self,
+            mock_get_completed_courses,  # pylint: disable=unused-argument
+            mock_get_completed_programs,
+            *_mock_helpers  # pylint: disable=unused-argument
+    ):
+        """
+        Ensures that any otherwise-unhandled errors that arise while trying
+        to get completed programs (e.g. network issues or other
+        transient API errors) will cause the task to be failed and queued for
+        retry.
+        """
+        mock_get_completed_programs.side_effect = self._make_side_effect([Exception('boom'), None])
+        tasks.award_program_certificates.delay(self.student.username).get()
+        self.assertEqual(mock_get_completed_programs.call_count, 2)
+
+    def test_retry_on_credentials_api_errors(
+            self,
+            mock_get_completed_courses,  # pylint: disable=unused-argument
+            mock_get_completed_programs,
+            mock_get_awarded_certificate_programs,
+            mock_award_program_certificate,
+    ):
+        """
+        Ensures that any otherwise-unhandled errors that arise while trying
+        to get existing program credentials (e.g. network issues or other
+        transient API errors) will cause the task to be failed and queued for
+        retry.
+        """
+        mock_get_completed_programs.return_value = [1, 2]
+        mock_get_awarded_certificate_programs.return_value = [1]
+        mock_get_awarded_certificate_programs.side_effect = self._make_side_effect([Exception('boom'), None])
+        tasks.award_program_certificates.delay(self.student.username).get()
+        self.assertEqual(mock_get_awarded_certificate_programs.call_count, 2)
+        self.assertEqual(mock_award_program_certificate.call_count, 1)

openedx/core/djangoapps/programs/tests/test_signals.py

@@ -14,7 +14,7 @@ from openedx.core.djangoapps.programs.signals import handle_course_cert_awarded
 TEST_USERNAME = 'test-user'
 
 
-@mock.patch('openedx.core.djangoapps.programs.tasks.award_program_certificates.delay')
+@mock.patch('openedx.core.djangoapps.programs.tasks.v1.tasks.award_program_certificates.delay')
 @mock.patch(
     'openedx.core.djangoapps.programs.models.ProgramsApiConfig.is_certification_enabled',
     new_callable=mock.PropertyMock,