multitenant Program cache.

Fetched Programs and Program details from Course Discovery service for all sites and stored the uuids in cache with site-specfic keys.
Learner-1146

common/djangoapps/student/views.py

@@ -36,7 +36,6 @@ from django.utils.translation import get_language, ungettext
 from django.views.decorators.csrf import csrf_exempt, ensure_csrf_cookie
 from django.views.decorators.http import require_GET, require_POST
 from django.views.generic import TemplateView
-from eventtracking import tracker
 from ipware.ip import get_ip
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
@@ -46,9 +45,9 @@ from provider.oauth2.models import Client
 from pytz import UTC
 from ratelimitbackend.exceptions import RateLimitException
 from requests import HTTPError
-from social_django import utils as social_utils
 from social_core.backends import oauth as social_oauth
 from social_core.exceptions import AuthAlreadyAssociated, AuthException
+from social_django import utils as social_utils
 
 import dogstats_wrapper as dog_stats_api
 import openedx.core.djangoapps.external_auth.views
@@ -66,6 +65,7 @@ from courseware.access import has_access
 from courseware.courses import get_courses, sort_by_announcement, sort_by_start_date  # pylint: disable=import-error
 from django_comment_common.models import assign_role
 from edxmako.shortcuts import render_to_response, render_to_string
+from eventtracking import tracker
 from lms.djangoapps.commerce.utils import EcommerceService  # pylint: disable=import-error
 from lms.djangoapps.grades.new.course_grade_factory import CourseGradeFactory
 from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification  # pylint: disable=import-error
@@ -174,7 +174,6 @@ def index(request, extra_context=None, user=AnonymousUser()):
     if extra_context is None:
         extra_context = {}
 
-    programs_list = []
     courses = get_courses(user)
 
     if configuration_helpers.get_value(
@@ -208,17 +207,7 @@ def index(request, extra_context=None, user=AnonymousUser()):
     # Insert additional context for use in the template
     context.update(extra_context)
 
-    # Get the active programs of the type configured for the current site from the catalog service. The programs_list
-    # is being added to the context but it's not being used currently in courseware/courses.html. To use this list,
-    # you need to create a custom theme that overrides courses.html. The modifications to courses.html to display the
-    # programs will be done after the support for edx-pattern-library is added.
-    program_types = configuration_helpers.get_value('ENABLED_PROGRAM_TYPES')
-
-    # Do not add programs to the context if there are no program types enabled for the site.
-    if program_types:
-        programs_list = get_programs_with_type(program_types, include_hidden=False)
-
-    context["programs_list"] = programs_list
+    context['programs_list'] = get_programs_with_type(include_hidden=False)
 
     return render_to_response('index.html', context)
 

common/djangoapps/terrain/stubs/catalog.py

@@ -14,6 +14,7 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
         pattern_handlers = {
             r'/api/v1/programs/$': self.program_list,
             r'/api/v1/programs/([0-9a-f-]+)/$': self.program_detail,
+            r'/api/v1/program_types/$': self.program_types,
         }
 
         if self.match_pattern(pattern_handlers):
@@ -42,6 +43,10 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
         program = self.server.config.get('catalog.programs.' + program_uuid)
         self.send_json_response(program)
 
+    def program_types(self):
+        program_types = self.server.config.get('catalog.programs_types', [])
+        self.send_json_response(program_types)
+
 
 class StubCatalogService(StubHttpService):
     HANDLER_CLASS = StubCatalogServiceHandler

common/test/acceptance/fixtures/catalog.py

@@ -29,7 +29,6 @@ class CatalogFixture(object):
             uuids.append(uuid)
 
             program_key = '{base}.{uuid}'.format(base=key, uuid=uuid)
-
             requests.put(
                 '{}/set_config'.format(CATALOG_STUB_URL),
                 data={program_key: json.dumps(program)},
@@ -41,6 +40,18 @@ class CatalogFixture(object):
             data={key: json.dumps(uuids)},
         )
 
+    def install_program_types(self, program_types):
+        """
+        Stub the discovery service's program type list API endpoints.
+
+        Arguments:
+            program_types (list): A list of program types. List endpoint will be stubbed using data from this list.
+        """
+        requests.put(
+            '{}/set_config'.format(CATALOG_STUB_URL),
+            data={'catalog.programs_types': json.dumps(program_types)},
+        )
+
 
 class CatalogIntegrationMixin(object):
     """Mixin providing a method used to configure the catalog integration."""

common/test/acceptance/tests/lms/test_programs.py

@@ -8,7 +8,12 @@ from common.test.acceptance.pages.common.auto_auth import AutoAuthPage
 from common.test.acceptance.pages.lms.catalog import CacheProgramsPage
 from common.test.acceptance.pages.lms.programs import ProgramDetailsPage, ProgramListingPage
 from common.test.acceptance.tests.helpers import UniqueCourseTest
-from openedx.core.djangoapps.catalog.tests.factories import CourseFactory, CourseRunFactory, ProgramFactory
+from openedx.core.djangoapps.catalog.tests.factories import (
+    CourseFactory,
+    CourseRunFactory,
+    ProgramFactory,
+    ProgramTypeFactory
+)
 
 
 class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourseTest):
@@ -36,7 +41,8 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
         course_run = CourseRunFactory(key=self.course_id)
         course = CourseFactory(course_runs=[course_run])
 
-        return ProgramFactory(courses=[course])
+        program_type = ProgramTypeFactory()
+        return ProgramFactory(courses=[course], type=program_type['name'])
 
     def stub_catalog_api(self, programs):
         """
@@ -45,6 +51,9 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
         self.set_catalog_integration(is_enabled=True, service_username=self.username)
         CatalogFixture().install_programs(programs)
 
+        program_types = [program['type'] for program in programs]
+        CatalogFixture().install_program_types(program_types)
+
     def cache_programs(self):
         """
         Populate the LMS' cache of program data.

lms/djangoapps/branding/tests/test_page.py

@@ -299,11 +299,6 @@ class IndexPageProgramsTests(SiteMixin, ModuleStoreTestCase):
     """
     @ddt.data([], ['fake_program_type'])
     def test_get_programs_with_type_called(self, program_types):
-        self.site_configuration.values.update({
-            'ENABLED_PROGRAM_TYPES': program_types
-        })
-        self.site_configuration.save()
-
         views = [
             (reverse('root'), 'student.views.get_programs_with_type'),
             (reverse('branding.views.courses'), 'courseware.views.views.get_programs_with_type'),

lms/djangoapps/branding/tests/test_views.py

@@ -318,12 +318,3 @@ class TestIndex(SiteMixin, TestCase):
         self.client.login(username=self.user.username, password="password")
         response = self.client.get(reverse("dashboard"))
         self.assertIn(self.site_configuration_other.values["MKTG_URLS"]["ROOT"], response.content)
-
-    def test_index_with_enabled_program_types(self):
-        """ Test index view with Enabled Program Types."""
-        self.site_configuration.values.update({'ENABLED_PROGRAM_TYPES': ['TestProgramType']})
-        self.site_configuration.save()
-        with mock.patch('student.views.get_programs_with_type') as patched_get_programs_with_type:
-            patched_get_programs_with_type.return_value = []
-            response = self.client.get(reverse("root"))
-            self.assertEqual(response.status_code, 200)

lms/djangoapps/courseware/views/views.py

@@ -155,15 +155,7 @@ def courses(request):
         else:
             courses_list = sort_by_announcement(courses_list)
 
-    # Get the active programs of the type configured for the current site from the catalog service. The programs_list
-    # is being added to the context but it's not being used currently in courseware/courses.html. To use this list,
-    # you need to create a custom theme that overrides courses.html. The modifications to courses.html to display the
-    # programs will be done after the support for edx-pattern-library is added.
-    program_types = configuration_helpers.get_value('ENABLED_PROGRAM_TYPES')
-
-    # Do not add programs to the context if there are no program types enabled for the site.
-    if program_types:
-        programs_list = get_programs_with_type(program_types, include_hidden=False)
+    programs_list = get_programs_with_type(include_hidden=False)
 
     return render_to_response(
         "courseware/courses.html",

openedx/core/djangoapps/catalog/cache.py

@@ -2,4 +2,9 @@
 PROGRAM_CACHE_KEY_TPL = 'program-{uuid}'
 
 # Cache key used to locate an item containing a list of all program UUIDs.
+# This has to be deleted when removing the waffle flags populate-multitenant-programs and get-multitenant-programs
+# For more, see LEARNER-1146
 PROGRAM_UUIDS_CACHE_KEY = 'program-uuids'
+
+# Cache key used to locate an item containing a list of all program UUIDs for a site.
+SITE_PROGRAM_UUIDS_CACHE_KEY_TPL = 'program-uuids-{domain}'

openedx/core/djangoapps/catalog/management/commands/cache_programs.py

 import logging
 import sys
 
+import waffle
 from django.contrib.auth import get_user_model
+from django.contrib.sites.models import Site
 from django.core.cache import cache
 from django.core.management import BaseCommand
 
-from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
+from openedx.core.djangoapps.catalog.cache import (
+    PROGRAM_CACHE_KEY_TPL,
+    PROGRAM_UUIDS_CACHE_KEY,
+    SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
+)
 from openedx.core.djangoapps.catalog.models import CatalogIntegration
 from openedx.core.djangoapps.catalog.utils import create_catalog_api_client
 
@@ -24,18 +30,111 @@ class Command(BaseCommand):
     help = "Rebuild the LMS' cache of program data."
 
     def handle(self, *args, **options):
-        catalog_integration = CatalogIntegration.current()
-        username = catalog_integration.service_username
+        if waffle.switch_is_active('populate-multitenant-programs'):
+            failure = False
+            logger.info('populate-multitenant-programs switch is ON')
 
-        try:
-            user = User.objects.get(username=username)
-            client = create_catalog_api_client(user)
-        except User.DoesNotExist:
-            logger.error(
-                'Failed to create API client. Service user {username} does not exist.'.format(username)
-            )
-            raise
+            catalog_integration = CatalogIntegration.current()
+            username = catalog_integration.service_username
+
+            try:
+                user = User.objects.get(username=username)
+            except User.DoesNotExist:
+                logger.error(
+                    'Failed to create API client. Service user {username} does not exist.'.format(username)
+                )
+                raise
+
+            programs = {}
+            for site in Site.objects.all():
+                site_config = getattr(site, 'configuration', None)
+                if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
+                    logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
+                    continue
+
+                client = create_catalog_api_client(user, site=site)
+                uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
+                new_programs, program_details_failed = self.fetch_program_details(client, uuids)
+
+                if program_uuids_failed or program_details_failed:
+                    failure = True
+
+                programs.update(new_programs)
+
+                logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
+                    total=len(uuids),
+                    site_name=site.domain,
+                ))
+                cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
+
+            successful = len(programs)
+            logger.info('Caching details for {successful} programs.'.format(successful=successful))
+            cache.set_many(programs, None)
+
+            if failure:
+                # This will fail a Jenkins job running this command, letting site
+                # operators know that there was a problem.
+                sys.exit(1)
+
+        else:
+            catalog_integration = CatalogIntegration.current()
+            username = catalog_integration.service_username
+
+            try:
+                user = User.objects.get(username=username)
+                client = create_catalog_api_client(user)
+            except User.DoesNotExist:
+                logger.error(
+                    'Failed to create API client. Service user {username} does not exist.'.format(username)
+                )
+                raise
+
+            try:
+                querystring = {
+                    'exclude_utm': 1,
+                    'status': ('active', 'retired'),
+                    'uuids_only': 1,
+                }
+
+                logger.info('Requesting program UUIDs.')
+                uuids = client.programs.get(**querystring)
+            except:  # pylint: disable=bare-except
+                logger.error('Failed to retrieve program UUIDs.')
+                raise
+
+            total = len(uuids)
+            logger.info('Received {total} UUIDs.'.format(total=total))
+
+            programs = {}
+            failure = False
+            for uuid in uuids:
+                try:
+                    logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
+                    program = client.programs(uuid).get(exclude_utm=1)
+
+                    cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
+                    programs[cache_key] = program
+                except:  # pylint: disable=bare-except
+                    logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
+                    failure = True
+
+                    continue
 
+            successful = len(programs)
+            logger.info('Caching details for {successful} programs.'.format(successful=successful))
+            cache.set_many(programs, None)
+
+            logger.info('Caching UUIDs for {total} programs.'.format(total=total))
+            cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
+
+            if failure:
+                # This will fail a Jenkins job running this command, letting site
+                # operators know that there was a problem.
+                sys.exit(1)
+
+    def get_site_program_uuids(self, client, site):
+        failure = False
+        uuids = []
         try:
             querystring = {
                 'exclude_utm': 1,
@@ -43,38 +142,29 @@ class Command(BaseCommand):
                 'uuids_only': 1,
             }
 
-            logger.info('Requesting program UUIDs.')
+            logger.info('Requesting program UUIDs for {domain}.'.format(domain=site.domain))
             uuids = client.programs.get(**querystring)
         except:  # pylint: disable=bare-except
-            logger.error('Failed to retrieve program UUIDs.')
-            raise
+            logger.error('Failed to retrieve program UUIDs for site: {domain}.'.format(domain=site.domain))
+            failure = True
 
-        total = len(uuids)
-        logger.info('Received {total} UUIDs.'.format(total=total))
+        logger.info('Received {total} UUIDs for site {domain}'.format(
+            total=len(uuids),
+            domain=site.domain
+        ))
+        return uuids, failure
 
+    def fetch_program_details(self, client, uuids):
         programs = {}
         failure = False
         for uuid in uuids:
             try:
+                cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
                 logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
                 program = client.programs(uuid).get(exclude_utm=1)
-
-                cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
                 programs[cache_key] = program
             except:  # pylint: disable=bare-except
                 logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
                 failure = True
-
                 continue
-
-        successful = len(programs)
-        logger.info('Caching details for {successful} programs.'.format(successful=successful))
-        cache.set_many(programs, None)
-
-        logger.info('Caching UUIDs for {total} programs.'.format(total=total))
-        cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
-
-        if failure:
-            # This will fail a Jenkins job running this command, letting site
-            # operators know that there was a problem.
-            sys.exit(1)
+        return programs, failure

openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py

 import json
 
 import httpretty
+import waffle
 from django.core.cache import cache
 from django.core.management import call_command
 
-from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
+from openedx.core.djangoapps.catalog.cache import (
+    PROGRAM_CACHE_KEY_TPL,
+    PROGRAM_UUIDS_CACHE_KEY,
+    SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
+)
 from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory
 from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
+from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
 from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
 from student.tests.factories import UserFactory
 
 
 @skip_unless_lms
 @httpretty.activate
-class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
+class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMixin):
     ENABLED_CACHES = ['default']
 
     def setUp(self):
         super(TestCachePrograms, self).setUp()
 
         self.catalog_integration = self.create_catalog_integration()
+        self.site_domain = 'testsite.com'
+        self.set_up_site(
+            self.site_domain,
+            {
+                'COURSE_CATALOG_API_URL': self.catalog_integration.get_internal_api_url().rstrip('/')
+            }
+        )
 
         self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/'
         self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/'
@@ -61,6 +74,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
             content_type='application/json'
         )
 
+    @waffle.testutils.override_switch('populate-multitenant-programs', True)
     def test_handle(self):
         """
         Verify that the command requests and caches program UUIDs and details.
@@ -83,7 +97,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
 
         call_command('cache_programs')
 
-        cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
+        cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
         self.assertEqual(
             set(cached_uuids),
             set(self.uuids)
@@ -104,6 +118,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
         for key, program in cached_programs.items():
             self.assertEqual(program, programs[key])
 
+    @waffle.testutils.override_switch('populate-multitenant-programs', True)
     def test_handle_missing_service_user(self):
         """
         Verify that the command raises an exception when run without a service
@@ -112,9 +127,10 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
         with self.assertRaises(Exception):
             call_command('cache_programs')
 
-        cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
+        cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
         self.assertEqual(cached_uuids, None)
 
+    @waffle.testutils.override_switch('populate-multitenant-programs', True)
     def test_handle_missing_uuids(self):
         """
         Verify that the command raises an exception when it fails to retrieve
@@ -122,12 +138,14 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
         """
         UserFactory(username=self.catalog_integration.service_username)
 
-        with self.assertRaises(Exception):
+        with self.assertRaises(SystemExit) as context:
             call_command('cache_programs')
+            self.assertEqual(context.exception.code, 1)
 
-        cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
-        self.assertEqual(cached_uuids, None)
+        cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
+        self.assertEqual(cached_uuids, [])
 
+    @waffle.testutils.override_switch('populate-multitenant-programs', True)
     def test_handle_missing_programs(self):
         """
         Verify that a problem retrieving a program doesn't prevent the command
@@ -154,7 +172,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
 
             self.assertEqual(context.exception.code, 1)
 
-        cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
+        cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
         self.assertEqual(
             set(cached_uuids),
             set(self.uuids)

openedx/core/djangoapps/catalog/utils.py

@@ -2,26 +2,36 @@
 import copy
 import logging
 
+import waffle
 from django.conf import settings
 from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
 from edx_rest_api_client.client import EdxRestApiClient
 
-from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
+from openedx.core.djangoapps.catalog.cache import (
+    PROGRAM_CACHE_KEY_TPL,
+    PROGRAM_UUIDS_CACHE_KEY,
+    SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
+)
 from openedx.core.djangoapps.catalog.models import CatalogIntegration
+from openedx.core.djangoapps.theming.helpers import get_current_site
 from openedx.core.lib.edx_api_utils import get_edx_api_data
 from openedx.core.lib.token_utils import JwtBuilder
 
 logger = logging.getLogger(__name__)
 
 
-def create_catalog_api_client(user):
+def create_catalog_api_client(user, site=None):
     """Returns an API client which can be used to make Catalog API requests."""
     scopes = ['email', 'profile']
     expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
     jwt = JwtBuilder(user).build_token(scopes, expires_in)
 
-    url = CatalogIntegration.current().get_internal_api_url()
+    if site:
+        url = site.configuration.get_value('COURSE_CATALOG_API_URL')
+    else:
+        url = CatalogIntegration.current().get_internal_api_url()
+
     return EdxRestApiClient(url, jwt=jwt)
 
 
@@ -45,8 +55,10 @@ def get_programs(uuid=None):
             logger.warning(missing_details_msg_tpl.format(uuid=uuid))
 
         return program
-
-    uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY, [])
+    if waffle.switch_is_active('get-multitenant-programs'):
+        uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=get_current_site().domain), [])
+    else:
+        uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY, [])
     if not uuids:
         logger.warning('Failed to get program UUIDs from the cache.')
 
@@ -109,16 +121,14 @@ def get_program_types(name=None):
         return []
 
 
-def get_programs_with_type(types=None, include_hidden=True):
+def get_programs_with_type(include_hidden=True):
     """
-    Return the list of programs. You can filter the types of programs returned using the optional
-    types parameter. If no filter is provided, all programs of all types will be returned. In addition,
-    you can specify whether to include hidden programs using the optional include_hidden parameter.
+    Return the list of programs. You can filter the types of programs returned by using the optional
+    include_hidden parameter. By default hidden programs will be included.
 
     The program dict is updated with the fully serialized program type.
 
     Keyword Arguments:
-        types (list): List of program type slugs to filter by.
         include_hidden (bool): whether to include hidden programs
 
     Return:
@@ -130,12 +140,7 @@ def get_programs_with_type(types=None, include_hidden=True):
     if programs:
         program_types = {program_type['name']: program_type for program_type in get_program_types()}
         for program in programs:
-            # This limited type filtering is sufficient for current needs and
-            # helps us avoid additional complexity when caching program data.
-            # However, if the need for additional filtering of programs should
-            # arise, consider using the cache_programs management command to
-            # cache the filtered lists of UUIDs.
-            if types and program['type'] not in types:
+            if program['type'] not in program_types:
                 continue
 
             if program['hidden'] and not include_hidden: