Merge pull request #12421 from edx/renzo/backpopulate-command

Add management command for backpopulating missing program credentials

openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py

+"""Management command for backpopulating missing program credentials."""
+from collections import namedtuple
+import logging
+
+from django.core.management import BaseCommand, CommandError
+from django.db.models import Q
+from opaque_keys.edx.keys import CourseKey
+from provider.oauth2.models import Client
+
+from certificates.models import GeneratedCertificate  # pylint: disable=import-error
+from openedx.core.djangoapps.programs.models import ProgramsApiConfig
+from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates
+from openedx.core.djangoapps.programs.utils import get_programs
+
+
+# TODO: Log to console, even with debug mode disabled?
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+RunMode = namedtuple('RunMode', ['course_key', 'mode_slug'])
+
+
+class Command(BaseCommand):
+    """Management command for backpopulating missing program credentials.
+
+    The command's goal is to pass a narrow subset of usernames to an idempotent
+    Celery task for further (parallelized) processing.
+    """
+    help = 'Backpopulate missing program credentials.'
+    client = None
+    run_modes = None
+    usernames = None
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-c', '--commit',
+            action='store_true',
+            dest='commit',
+            default=False,
+            help='Submit tasks for processing.'
+        )
+
+    def handle(self, *args, **options):
+        programs_config = ProgramsApiConfig.current()
+        self.client = Client.objects.get(name=programs_config.OAUTH2_CLIENT_NAME)
+
+        if self.client.user is None:
+            msg = (
+                'No user is associated with the {} OAuth2 client. '
+                'A service user is necessary to make requests to the Programs API. '
+                'No tasks have been enqueued. '
+                'Associate a user with the client and try again.'
+            ).format(programs_config.OAUTH2_CLIENT_NAME)
+
+            raise CommandError(msg)
+
+        self._load_run_modes()
+
+        logger.info('Looking for users who may be eligible for a program certificate.')
+
+        self._load_usernames()
+
+        if options.get('commit'):
+            logger.info('Enqueuing program certification tasks for %d candidates.', len(self.usernames))
+        else:
+            logger.info(
+                'Found %d candidates. To enqueue program certification tasks, pass the -c or --commit flags.',
+                len(self.usernames)
+            )
+            return
+
+        succeeded, failed = 0, 0
+        for username in self.usernames:
+            try:
+                award_program_certificates.delay(username)
+            except:  # pylint: disable=bare-except
+                failed += 1
+                logger.exception('Failed to enqueue task for user [%s]', username)
+            else:
+                succeeded += 1
+                logger.debug('Successfully enqueued task for user [%s]', username)
+
+        logger.info(
+            'Done. Successfully enqueued tasks for %d candidates. '
+            'Failed to enqueue tasks for %d candidates.',
+            succeeded,
+            failed
+        )
+
+    def _load_run_modes(self):
+        """Find all run modes which are part of a program."""
+        programs = get_programs(self.client.user)
+        self.run_modes = self._flatten(programs)
+
+    def _flatten(self, programs):
+        """Flatten program dicts into a set of run modes."""
+        run_modes = set()
+        for program in programs:
+            for course_code in program['course_codes']:
+                for run in course_code['run_modes']:
+                    course_key = CourseKey.from_string(run['course_key'])
+                    run_modes.add(
+                        RunMode(course_key, run['mode_slug'])
+                    )
+
+        return run_modes
+
+    def _load_usernames(self):
+        """Identify a subset of users who may be eligible for a program certificate.
+
+        This is done by finding users who have earned a certificate in at least one
+        program course code's run mode.
+        """
+        query = reduce(
+            lambda x, y: x | y,
+            [Q(course_id=r.course_key, mode=r.mode_slug) for r in self.run_modes]
+        )
+
+        # TODO: Filter further, by passing status?
+        username_dicts = GeneratedCertificate.eligible_certificates.filter(query).values('user__username').distinct()
+        self.usernames = [d['user__username'] for d in username_dicts]

openedx/core/djangoapps/programs/tests/test_backpopulate_program_credentials.py

+"""Tests for the backpopulate_program_credentials management command."""
+import json
+from unittest import skipUnless
+
+import ddt
+from django.conf import settings
+from django.core.management import call_command, CommandError
+from django.test import TestCase
+from edx_oauth2_provider.tests.factories import ClientFactory
+import httpretty
+import mock
+from provider.constants import CONFIDENTIAL
+
+from lms.djangoapps.certificates.api import MODES
+from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory
+from openedx.core.djangoapps.programs.models import ProgramsApiConfig
+from openedx.core.djangoapps.programs.tests import factories
+from openedx.core.djangoapps.programs.tests.mixins import ProgramsApiConfigMixin
+from student.tests.factories import UserFactory
+
+
+COMMAND_MODULE = 'openedx.core.djangoapps.programs.management.commands.backpopulate_program_credentials'
+
+
+@ddt.ddt
+@httpretty.activate
+@mock.patch(COMMAND_MODULE + '.award_program_certificates.delay')
+@skipUnless(settings.ROOT_URLCONF == 'lms.urls', 'Test only valid in lms')
+class BackpopulateProgramCredentialsTests(ProgramsApiConfigMixin, TestCase):
+    """Tests for the backpopulate_program_credentials management command."""
+    course_id, alternate_course_id = 'org/course/run', 'org/alternate/run'
+
+    def setUp(self):
+        super(BackpopulateProgramCredentialsTests, self).setUp()
+
+        self.alice = UserFactory()
+        self.bob = UserFactory()
+        self.oauth2_user = UserFactory()
+        self.oauth2_client = ClientFactory(name=ProgramsApiConfig.OAUTH2_CLIENT_NAME, client_type=CONFIDENTIAL)
+
+        self.create_programs_config()
+
+    def _link_oauth2_user(self):
+        """Helper to link user and OAuth2 client."""
+        self.oauth2_client.user = self.oauth2_user
+        self.oauth2_client.save()  # pylint: disable=no-member
+
+    def _mock_programs_api(self, data):
+        """Helper for mocking out Programs API URLs."""
+        self.assertTrue(httpretty.is_enabled(), msg='httpretty must be enabled to mock Programs API calls.')
+
+        url = ProgramsApiConfig.current().internal_api_url.strip('/') + '/programs/'
+        body = json.dumps({'results': data})
+
+        httpretty.register_uri(httpretty.GET, url, body=body, content_type='application/json')
+
+    @ddt.data(True, False)
+    def test_handle(self, commit, mock_task):
+        """Verify that relevant tasks are only enqueued when the commit option is passed."""
+        data = [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=self.course_id),
+                    ]),
+                ]
+            ),
+        ]
+        self._mock_programs_api(data)
+        self._link_oauth2_user()
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        GeneratedCertificateFactory(
+            user=self.bob,
+            course_id=self.alternate_course_id,
+            mode=MODES.verified,
+        )
+
+        call_command('backpopulate_program_credentials', commit=commit)
+
+        if commit:
+            mock_task.assert_called_once_with(self.alice.username)
+        else:
+            mock_task.assert_not_called()
+
+    @ddt.data(
+        [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=course_id),
+                    ]),
+                ]
+            ),
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=alternate_course_id),
+                    ]),
+                ]
+            ),
+        ],
+        [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=course_id),
+                    ]),
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=alternate_course_id),
+                    ]),
+                ]
+            ),
+        ],
+        [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=course_id),
+                        factories.RunMode(course_key=alternate_course_id),
+                    ]),
+                ]
+            ),
+        ],
+    )
+    def test_handle_flatten(self, data, mock_task):
+        """Verify that program structures are flattened correctly."""
+        self._mock_programs_api(data)
+        self._link_oauth2_user()
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        GeneratedCertificateFactory(
+            user=self.bob,
+            course_id=self.alternate_course_id,
+            mode=MODES.verified,
+        )
+
+        call_command('backpopulate_program_credentials', commit=True)
+
+        calls = [
+            mock.call(self.alice.username),
+            mock.call(self.bob.username)
+        ]
+        mock_task.assert_has_calls(calls, any_order=True)
+
+    def test_handle_username_dedup(self, mock_task):
+        """Verify that only one task is enqueued for a user with multiple eligible certs."""
+        data = [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=self.course_id),
+                        factories.RunMode(course_key=self.alternate_course_id),
+                    ]),
+                ]
+            ),
+        ]
+        self._mock_programs_api(data)
+        self._link_oauth2_user()
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.alternate_course_id,
+            mode=MODES.verified,
+        )
+
+        call_command('backpopulate_program_credentials', commit=True)
+
+        mock_task.assert_called_once_with(self.alice.username)
+
+    def test_handle_mode_slugs(self, mock_task):
+        """Verify that mode slugs are taken into account."""
+        data = [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(
+                            course_key=self.course_id,
+                            mode_slug=MODES.honor
+                        ),
+                    ]),
+                ]
+            ),
+        ]
+        self._mock_programs_api(data)
+        self._link_oauth2_user()
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.course_id,
+        )
+
+        GeneratedCertificateFactory(
+            user=self.bob,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        call_command('backpopulate_program_credentials', commit=True)
+
+        mock_task.assert_called_once_with(self.alice.username)
+
+    def test_handle_unlinked_oauth2_user(self, mock_task):
+        """Verify that the command fails when no user is associated with the OAuth2 client."""
+        data = [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=self.course_id),
+                    ]),
+                ]
+            ),
+        ]
+        self._mock_programs_api(data)
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        with self.assertRaises(CommandError):
+            call_command('backpopulate_program_credentials')
+
+        mock_task.assert_not_called()
+
+    @mock.patch(COMMAND_MODULE + '.logger.exception')
+    def test_handle_enqueue_failure(self, mock_log, mock_task):
+        """Verify that failure to enqueue a task doesn't halt execution."""
+        def side_effect(username):
+            """Simulate failure to enqueue a task."""
+            if username == self.alice.username:
+                raise Exception
+
+        mock_task.side_effect = side_effect
+
+        data = [
+            factories.Program(
+                organizations=[factories.Organization()],
+                course_codes=[
+                    factories.CourseCode(run_modes=[
+                        factories.RunMode(course_key=self.course_id),
+                    ]),
+                ]
+            ),
+        ]
+        self._mock_programs_api(data)
+        self._link_oauth2_user()
+
+        GeneratedCertificateFactory(
+            user=self.alice,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        GeneratedCertificateFactory(
+            user=self.bob,
+            course_id=self.course_id,
+            mode=MODES.verified,
+        )
+
+        call_command('backpopulate_program_credentials', commit=True)
+
+        self.assertTrue(mock_log.called)
+
+        calls = [
+            mock.call(self.alice.username),
+            mock.call(self.bob.username)
+        ]
+        mock_task.assert_has_calls(calls, any_order=True)