Skip to content

E
edx-platform
Commit 4027dd84 by Renzo Lucioni
Options

Use the catalog for program certificate backpopulation 

This command is used to backfill missing program certificates. Formerly, it used the programs service. It now uses the catalog service exclusively.

ECOM-4422
parent 327ffa92
Showing with 92 additions and 115 deletions
openedx/core/djangoapps/catalog/models.py
......@@ -27,7 +27,7 @@ class CatalogIntegration(ConfigurationModel):
service_username = models.CharField(
max_length=100,
default="lms_catalog_service_user",
default='lms_catalog_service_user',
null=False,
blank=False,
help_text=_(
......
openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py
......@@ -3,21 +3,18 @@ from collections import namedtuple
import logging
from django.contrib.auth.models import User
from django.core.management import BaseCommand, CommandError
from django.core.management import BaseCommand
from django.db.models import Q
from opaque_keys.edx.keys import CourseKey
from provider.oauth2.models import Client
import waffle
from certificates.models import GeneratedCertificate, CertificateStatuses # pylint: disable=import-error
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.utils import get_programs
from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates
from openedx.core.djangoapps.programs.utils import get_programs
# TODO: Log to console, even with debug mode disabled?
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
RunMode = namedtuple('RunMode', ['course_key', 'mode_slug'])
CourseRun = namedtuple('CourseRun', ['key', 'type'])
class Command(BaseCommand):
......@@ -27,8 +24,7 @@ class Command(BaseCommand):
Celery task for further (parallelized) processing.
"""
help = 'Backpopulate missing program credentials.'
client = None
run_modes = None
course_runs = None
usernames = None
def add_arguments(self, parser):
......@@ -41,20 +37,10 @@ class Command(BaseCommand):
)
def handle(self, *args, **options):
catalog_config = CatalogIntegration.current()
try:
user = User.objects.get(username=catalog_config.service_username)
except:
raise CommandError(
'User with username [{}] not found. '
'A service user is required to run this command.'.format(catalog_config.service_username)
)
self._load_run_modes(user)
logger.info('Loading programs from the catalog.')
self._load_course_runs()
logger.info('Looking for users who may be eligible for a program certificate.')
self._load_usernames()
if options.get('commit'):
......@@ -84,38 +70,44 @@ class Command(BaseCommand):
failed
)
def _load_run_modes(self, user):
"""Find all run modes which are part of a program."""
use_catalog = waffle.switch_is_active('get_programs_from_catalog')
programs = get_programs(user, use_catalog=use_catalog)
self.run_modes = self._flatten(programs)
def _load_course_runs(self):
"""Find all course runs which are part of a program."""
programs = get_programs()
self.course_runs = self._flatten(programs)
def _flatten(self, programs):
"""Flatten program dicts into a set of run modes."""
run_modes = set()
"""Flatten programs into a set of course runs."""
course_runs = set()
for program in programs:
for course_code in program['course_codes']:
for run in course_code['run_modes']:
course_key = CourseKey.from_string(run['course_key'])
run_modes.add(
RunMode(course_key, run['mode_slug'])
for course in program['courses']:
for run in course['course_runs']:
key = CourseKey.from_string(run['key'])
course_runs.add(
CourseRun(key, run['type'])
)
return run_modes
return course_runs
def _load_usernames(self):
"""Identify a subset of users who may be eligible for a program certificate.
This is done by finding users who have earned a certificate in at least one
program course code's run mode.
This is done by finding users who have earned a qualifying certificate in
at least one program course's course run.
"""
status_query = Q(status__in=CertificateStatuses.PASSED_STATUSES)
run_mode_query = reduce(
course_run_query = reduce(
lambda x, y: x | y,
[Q(course_id=r.course_key, mode=r.mode_slug) for r in self.run_modes]
# A course run's type is assumed to indicate which mode must be
# completed in order for the run to count towards program completion.
# This supports the same flexible program construction allowed by the
# old programs service (e.g., completion of an old honor-only run may
# count towards completion of a course in a program). This may change
# in the future to make use of the more rigid set of "applicable seat
# types" associated with each program type in the catalog.
[Q(course_id=run.key, mode=run.type) for run in self.course_runs]
)
query = status_query & run_mode_query
query = status_query & course_run_query
username_dicts = GeneratedCertificate.eligible_certificates.filter(query).values('user__username').distinct()
self.usernames = [d['user__username'] for d in username_dicts]
openedx/core/djangoapps/programs/tests/test_backpopulate_program_credentials.py
"""Tests for the backpopulate_program_credentials management command."""
import json
import ddt
from django.core.management import call_command, CommandError
from django.core.management import call_command
from django.test import TestCase
import httpretty
import mock
from certificates.models import CertificateStatuses # pylint: disable=import-error
from lms.djangoapps.certificates.api import MODES
from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory
from openedx.core.djangoapps.programs.tests import factories
from openedx.core.djangoapps.catalog.tests import factories
from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
from openedx.core.djangoapps.credentials.tests.mixins import CredentialsApiConfigMixin
from openedx.core.djangolib.testing.utils import skip_unless_lms
......@@ -26,7 +23,7 @@ COMMAND_MODULE = 'openedx.core.djangoapps.programs.management.commands.backpopul
@skip_unless_lms
class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsApiConfigMixin, TestCase):
"""Tests for the backpopulate_program_credentials management command."""
course_id, alternate_course_id = 'org/course/run', 'org/alternate/run'
course_run_key, alternate_course_run_key = (factories.generate_course_run_key() for __ in range(2))
def setUp(self):
super(BackpopulateProgramCredentialsTests, self).setUp()
......@@ -44,13 +41,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
@ddt.data(True, False)
def test_handle(self, commit, mock_task, mock_get_programs):
"""Verify that relevant tasks are only enqueued when the commit option is passed."""
"""
Verify that relevant tasks are only enqueued when the commit option is passed.
"""
data = [
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=self.course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=self.course_run_key),
]),
]
),
......@@ -59,14 +57,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
GeneratedCertificateFactory(
user=self.alice,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
GeneratedCertificateFactory(
user=self.bob,
course_id=self.alternate_course_id,
course_id=self.alternate_course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
......@@ -81,42 +79,38 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
@ddt.data(
[
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=course_run_key),
]),
]
),
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=alternate_course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=alternate_course_run_key),
]),
]
),
],
[
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=course_run_key),
]),
factories.CourseCode(run_modes=[
factories.RunMode(course_key=alternate_course_id),
factories.Course(course_runs=[
factories.CourseRun(key=alternate_course_run_key),
]),
]
),
],
[
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=course_id),
factories.RunMode(course_key=alternate_course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=course_run_key),
factories.CourseRun(key=alternate_course_run_key),
]),
]
),
......@@ -128,14 +122,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
GeneratedCertificateFactory(
user=self.alice,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
GeneratedCertificateFactory(
user=self.bob,
course_id=self.alternate_course_id,
course_id=self.alternate_course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
......@@ -149,14 +143,16 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
mock_task.assert_has_calls(calls, any_order=True)
def test_handle_username_dedup(self, mock_task, mock_get_programs):
"""Verify that only one task is enqueued for a user with multiple eligible certs."""
"""
Verify that only one task is enqueued for a user with multiple eligible
course run certificates.
"""
data = [
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=self.course_id),
factories.RunMode(course_key=self.alternate_course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=self.course_run_key),
factories.CourseRun(key=self.alternate_course_run_key),
]),
]
),
......@@ -165,14 +161,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
GeneratedCertificateFactory(
user=self.alice,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
GeneratedCertificateFactory(
user=self.alice,
course_id=self.alternate_course_id,
course_id=self.alternate_course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
......@@ -182,16 +178,15 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
mock_task.assert_called_once_with(self.alice.username)
def test_handle_mode_slugs(self, mock_task, mock_get_programs):
"""Verify that mode slugs are taken into account."""
"""
Verify that course run types are taken into account when identifying
qualifying course run certificates.
"""
data = [
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(
course_key=self.course_id,
mode_slug=MODES.honor
),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=self.course_run_key, type='honor'),
]),
]
),
......@@ -200,13 +195,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
GeneratedCertificateFactory(
user=self.alice,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.honor,
status=CertificateStatuses.downloadable,
)
GeneratedCertificateFactory(
user=self.bob,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
......@@ -216,14 +212,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
mock_task.assert_called_once_with(self.alice.username)
def test_handle_passing_status(self, mock_task, mock_get_programs):
"""Verify that only certificates with a passing status are selected."""
"""
Verify that only course run certificates with a passing status are selected.
"""
data = [
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=self.course_id),
factories.RunMode(course_key=self.alternate_course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=self.course_run_key),
]),
]
),
......@@ -238,16 +234,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
GeneratedCertificateFactory(
user=self.alice,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=passing_status,
)
# The alternate course is used here to verify that the status and run_mode
# queries are being ANDed together correctly.
GeneratedCertificateFactory(
user=self.bob,
course_id=self.alternate_course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=failing_status,
)
......@@ -256,14 +250,6 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
mock_task.assert_called_once_with(self.alice.username)
def test_handle_missing_service_user(self, mock_task, __):
"""Verify that the command fails when no service user exists."""
self.catalog_integration = self.create_catalog_integration(service_username='test')
with self.assertRaises(CommandError):
call_command('backpopulate_program_credentials')
mock_task.assert_not_called()
@mock.patch(COMMAND_MODULE + '.logger.exception')
def test_handle_enqueue_failure(self, mock_log, mock_task, mock_get_programs):
"""Verify that failure to enqueue a task doesn't halt execution."""
......@@ -276,10 +262,9 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
data = [
factories.Program(
organizations=[factories.Organization()],
course_codes=[
factories.CourseCode(run_modes=[
factories.RunMode(course_key=self.course_id),
courses=[
factories.Course(course_runs=[
factories.CourseRun(key=self.course_run_key),
]),
]
),
......@@ -288,14 +273,14 @@ class BackpopulateProgramCredentialsTests(CatalogIntegrationMixin, CredentialsAp
GeneratedCertificateFactory(
user=self.alice,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
GeneratedCertificateFactory(
user=self.bob,
course_id=self.course_id,
course_id=self.course_run_key,
mode=MODES.verified,
status=CertificateStatuses.downloadable,
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment