Retrieve marketable MicroMasters from the catalog service

Catalog-based MicroMasters need to be displayed in the LMS. However, the LMS currently retrieves all program data from the soon-to-be-retired programs service. Consuming program data exclusively from the catalog service is out of the question right now; it's too complex to confidently pull off in a week. This is a functional middle ground introduced by ECOM-5460. Cleaning up this debt is tracked by ECOM-4418.

cms/envs/bok_choy.env.json

@@ -82,6 +82,9 @@
     },
     "FEEDBACK_SUBMISSION_EMAIL": "",
     "GITHUB_REPO_ROOT": "** OVERRIDDEN **",
+    "JWT_AUTH": {
+        "JWT_SECRET_KEY": "super-secret-key"
+    },
     "GRADES_DOWNLOAD": {
         "BUCKET": "edx-grades",
         "ROOT_PATH": "/tmp/edx-s3/grades",

common/djangoapps/terrain/stubs/catalog.py

@@ -11,6 +11,7 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):  # pylint: disable=miss
 
     def do_GET(self):  # pylint: disable=invalid-name, missing-docstring
         pattern_handlers = {
+            r'/api/v1/programs/$': self.get_programs,
             r'/api/v1/course_runs/(?P<course_id>[^/+]+(/|\+)[^/+]+(/|\+)[^/?]+)/$': self.get_course_run,
         }
 
@@ -31,9 +32,16 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):  # pylint: disable=miss
                 return True
         return None
 
+    def get_programs(self):
+        """
+        Stubs the catalog's programs endpoint.
+        """
+        programs = self.server.config.get('catalog.programs', [])
+        self.send_json_response(programs)
+
     def get_course_run(self, course_id):
         """
-        Stubs a catalog course run endpoint.
+        Stubs the catalog's course run endpoint.
         """
         course_run = self.server.config.get('course_run.{}'.format(course_id), [])
         self.send_json_response(course_run)

common/test/acceptance/fixtures/catalog.py

@@ -13,6 +13,15 @@ class CatalogFixture(object):
     """
     Interface to set up mock responses from the Catalog stub server.
     """
+    def install_programs(self, programs):
+        """Set response data for the catalog's course run API."""
+        key = 'catalog.programs'
+
+        requests.put(
+            '{}/set_config'.format(CATALOG_STUB_URL),
+            data={key: json.dumps(programs)},
+        )
+
     def install_course_run(self, course_run):
         """Set response data for the catalog's course run API."""
         key = 'catalog.{}'.format(course_run['key'])

common/test/acceptance/tests/lms/test_programs.py

@@ -17,8 +17,8 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogConfigMixin, UniqueCourseTest)
         super(ProgramPageBase, self).setUp()
 
         self.set_programs_api_configuration(is_enabled=True)
-        self.set_catalog_configuration(is_enabled=True)
 
+        self.programs = [catalog_factories.Program() for __ in range(3)]
         self.course_run = catalog_factories.CourseRun(key=self.course_id)
         self.stub_catalog_api()
 
@@ -51,7 +51,9 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogConfigMixin, UniqueCourseTest)
         ProgramsFixture().install_programs(programs, is_list=is_list)
 
     def stub_catalog_api(self):
-        """Stub out the catalog API's course run endpoint."""
+        """Stub out the catalog API's program and course run endpoints."""
+        self.set_catalog_configuration(is_enabled=True)
+        CatalogFixture().install_programs(self.programs)
         CatalogFixture().install_course_run(self.course_run)
 
     def auth(self, enroll=True):

common/test/acceptance/tests/studio/test_studio_home.py

@@ -6,6 +6,7 @@ from flaky import flaky
 from opaque_keys.edx.locator import LibraryLocator
 from uuid import uuid4
 
+from common.test.acceptance.fixtures.catalog import CatalogFixture, CatalogConfigMixin
 from common.test.acceptance.fixtures.programs import ProgramsFixture, ProgramsConfigMixin
 from common.test.acceptance.pages.studio.auto_auth import AutoAuthPage
 from common.test.acceptance.pages.studio.library import LibraryEditPage
@@ -68,18 +69,30 @@ class CreateLibraryTest(WebAppTest):
         self.assertTrue(self.dashboard_page.has_library(name=name, org=org, number=number))
 
 
-class DashboardProgramsTabTest(ProgramsConfigMixin, WebAppTest):
+class DashboardProgramsTabTest(ProgramsConfigMixin, CatalogConfigMixin, WebAppTest):
     """
     Test the programs tab on the studio home page.
     """
 
     def setUp(self):
         super(DashboardProgramsTabTest, self).setUp()
-        ProgramsFixture().install_programs([])
+        self.stub_programs_api()
+        self.stub_catalog_api()
+
         self.auth_page = AutoAuthPage(self.browser, staff=True)
         self.dashboard_page = DashboardPageWithPrograms(self.browser)
         self.auth_page.visit()
 
+    def stub_programs_api(self):
+        """Stub out the programs API with fake data."""
+        self.set_programs_api_configuration(is_enabled=True)
+        ProgramsFixture().install_programs([])
+
+    def stub_catalog_api(self):
+        """Stub out the catalog API's program endpoint."""
+        self.set_catalog_configuration(is_enabled=True)
+        CatalogFixture().install_programs([])
+
     def test_tab_is_disabled(self):
         """
         The programs tab and "new program" button should not appear at all
@@ -96,7 +109,6 @@ class DashboardProgramsTabTest(ProgramsConfigMixin, WebAppTest):
         via config.  When the programs list is empty, a button should appear
         that allows creating a new program.
         """
-        self.set_programs_api_configuration(True)
         self.dashboard_page.visit()
         self.assertTrue(self.dashboard_page.is_programs_tab_present())
         self.assertTrue(self.dashboard_page.is_new_program_button_present())
@@ -129,8 +141,6 @@ class DashboardProgramsTabTest(ProgramsConfigMixin, WebAppTest):
 
         ProgramsFixture().install_programs(programs)
 
-        self.set_programs_api_configuration(True)
-
         self.dashboard_page.visit()
 
         self.assertTrue(self.dashboard_page.is_programs_tab_present())
@@ -145,7 +155,6 @@ class DashboardProgramsTabTest(ProgramsConfigMixin, WebAppTest):
         The programs tab and "new program" button will not be available, even
         when enabled via config, if the user is not global staff.
         """
-        self.set_programs_api_configuration(True)
         AutoAuthPage(self.browser, staff=False).visit()
         self.dashboard_page.visit()
         self.assertFalse(self.dashboard_page.is_programs_tab_present())

lms/djangoapps/learner_dashboard/urls.py

@@ -7,5 +7,6 @@ from . import views
 urlpatterns = [
     url(r'^programs/$', views.program_listing, name='program_listing_view'),
     # Matches paths like 'programs/123/' and 'programs/123/foo/', but not 'programs/123/foo/bar/'.
-    url(r'^programs/(?P<program_id>\d+)/[\w\-]*/?$', views.program_details, name='program_details_view'),
+    # Also accepts strings that look like UUIDs, to support retrieval of catalog-based MicroMasters.
+    url(r'^programs/(?P<program_id>[0-9a-f-]+)/[\w\-]*/?$', views.program_details, name='program_details_view'),
 ]

lms/djangoapps/learner_dashboard/views.py

 """Learner dashboard views"""
+import uuid
+
 from django.contrib.auth.decorators import login_required
 from django.core.urlresolvers import reverse
 from django.http import Http404
@@ -6,6 +8,7 @@ from django.views.decorators.http import require_GET
 
 from edxmako.shortcuts import render_to_response
 from lms.djangoapps.learner_dashboard.utils import strip_course_id, FAKE_COURSE_KEY
+from openedx.core.djangoapps.catalog.utils import get_programs as get_catalog_programs, munge_catalog_program
 from openedx.core.djangoapps.credentials.utils import get_programs_credentials
 from openedx.core.djangoapps.programs.models import ProgramsApiConfig
 from openedx.core.djangoapps.programs import utils
@@ -43,7 +46,15 @@ def program_details(request, program_id):
     if not programs_config.show_program_details:
         raise Http404
 
-    program_data = utils.get_programs(request.user, program_id=program_id)
+    try:
+        # If the ID is a UUID, the requested program resides in the catalog.
+        uuid.UUID(program_id)
+
+        program_data = get_catalog_programs(request.user, uuid=program_id)
+        if program_data:
+            program_data = munge_catalog_program(program_data)
+    except ValueError:
+        program_data = utils.get_programs(request.user, program_id=program_id)
 
     if not program_data:
         raise Http404

openedx/core/djangoapps/catalog/tests/factories.py

 """Factories for generating fake catalog data."""
+from uuid import uuid4
+
 import factory
 from factory.fuzzy import FuzzyText
 
 
+class Organization(factory.Factory):
+    """
+    Factory for stubbing Organization resources from the catalog API.
+    """
+    class Meta(object):
+        model = dict
+
+    name = FuzzyText(prefix='Organization ')
+    key = FuzzyText(suffix='X')
+
+
 class CourseRun(factory.Factory):
     """
     Factory for stubbing CourseRun resources from the catalog API.
@@ -12,3 +25,48 @@ class CourseRun(factory.Factory):
 
     key = FuzzyText(prefix='org/', suffix='/run')
     marketing_url = FuzzyText(prefix='https://www.example.com/marketing/')
+
+
+class Course(factory.Factory):
+    """
+    Factory for stubbing Course resources from the catalog API.
+    """
+    class Meta(object):
+        model = dict
+
+    title = FuzzyText(prefix='Course ')
+    key = FuzzyText(prefix='course+')
+    owners = [Organization()]
+    course_runs = [CourseRun() for __ in range(3)]
+
+
+class BannerImage(factory.Factory):
+    """
+    Factory for stubbing BannerImage resources from the catalog API.
+    """
+    class Meta(object):
+        model = dict
+
+    url = FuzzyText(
+        prefix='https://www.somecdn.com/media/programs/banner_images/',
+        suffix='.jpg'
+    )
+
+
+class Program(factory.Factory):
+    """
+    Factory for stubbing Program resources from the catalog API.
+    """
+    class Meta(object):
+        model = dict
+
+    uuid = str(uuid4())
+    title = FuzzyText(prefix='Program ')
+    subtitle = FuzzyText(prefix='Subtitle ')
+    type = 'FooBar'
+    marketing_slug = FuzzyText(prefix='slug_')
+    authoring_organizations = [Organization()]
+    courses = [Course() for __ in range(3)]
+    banner_image = {
+        size: BannerImage() for size in ['large', 'medium', 'small', 'x-small']
+    }

openedx/core/djangoapps/catalog/tests/test_utils.py

 """Tests covering utilities for integrating with the catalog service."""
+import uuid
+
 import ddt
 from django.test import TestCase
 import mock
@@ -16,6 +18,139 @@ UTILS_MODULE = 'openedx.core.djangoapps.catalog.utils'
 @mock.patch(UTILS_MODULE + '.get_edx_api_data')
 # ConfigurationModels use the cache. Make every cache get a miss.
 @mock.patch('config_models.models.cache.get', return_value=None)
+class TestGetPrograms(mixins.CatalogIntegrationMixin, TestCase):
+    """Tests covering retrieval of programs from the catalog service."""
+    def setUp(self):
+        super(TestGetPrograms, self).setUp()
+
+        self.user = UserFactory()
+        self.uuid = str(uuid.uuid4())
+        self.type = 'FooBar'
+        self.catalog_integration = self.create_catalog_integration(cache_ttl=1)
+
+    def assert_contract(self, call_args, program_uuid=None, type=None):  # pylint: disable=redefined-builtin
+        """Verify that API data retrieval utility is used correctly."""
+        args, kwargs = call_args
+
+        for arg in (self.catalog_integration, self.user, 'programs'):
+            self.assertIn(arg, args)
+
+        self.assertEqual(kwargs['resource_id'], program_uuid)
+
+        cache_key = '{base}.programs{type}'.format(
+            base=self.catalog_integration.CACHE_KEY,
+            type='.' + type if type else ''
+        )
+        self.assertEqual(
+            kwargs['cache_key'],
+            cache_key if self.catalog_integration.is_cache_enabled else None
+        )
+
+        self.assertEqual(kwargs['api']._store['base_url'], self.catalog_integration.internal_api_url)  # pylint: disable=protected-access
+
+        querystring = {'marketable': 1}
+        if type:
+            querystring['type'] = type
+        self.assertEqual(kwargs['querystring'], querystring)
+
+        return args, kwargs
+
+    def test_get_programs(self, _mock_cache, mock_get_catalog_data):
+        programs = [factories.Program() for __ in range(3)]
+        mock_get_catalog_data.return_value = programs
+
+        data = utils.get_programs(self.user)
+
+        self.assert_contract(mock_get_catalog_data.call_args)
+        self.assertEqual(data, programs)
+
+    def test_get_one_program(self, _mock_cache, mock_get_catalog_data):
+        program = factories.Program()
+        mock_get_catalog_data.return_value = program
+
+        data = utils.get_programs(self.user, uuid=self.uuid)
+
+        self.assert_contract(mock_get_catalog_data.call_args, program_uuid=self.uuid)
+        self.assertEqual(data, program)
+
+    def test_get_programs_by_type(self, _mock_cache, mock_get_catalog_data):
+        programs = [factories.Program() for __ in range(2)]
+        mock_get_catalog_data.return_value = programs
+
+        data = utils.get_programs(self.user, type=self.type)
+
+        self.assert_contract(mock_get_catalog_data.call_args, type=self.type)
+        self.assertEqual(data, programs)
+
+    def test_programs_unavailable(self, _mock_cache, mock_get_catalog_data):
+        mock_get_catalog_data.return_value = []
+
+        data = utils.get_programs(self.user)
+
+        self.assert_contract(mock_get_catalog_data.call_args)
+        self.assertEqual(data, [])
+
+    def test_cache_disabled(self, _mock_cache, mock_get_catalog_data):
+        self.catalog_integration = self.create_catalog_integration(cache_ttl=0)
+        utils.get_programs(self.user)
+        self.assert_contract(mock_get_catalog_data.call_args)
+
+    def test_config_missing(self, _mock_cache, _mock_get_catalog_data):
+        """Verify that no errors occur if this method is called when catalog config is missing."""
+        CatalogIntegration.objects.all().delete()
+
+        data = utils.get_programs(self.user)
+        self.assertEqual(data, [])
+
+
+class TestMungeCatalogProgram(TestCase):
+    """Tests covering querystring stripping."""
+    catalog_program = factories.Program()
+
+    def test_munge_catalog_program(self):
+        munged = utils.munge_catalog_program(self.catalog_program)
+        expected = {
+            'id': self.catalog_program['uuid'],
+            'name': self.catalog_program['title'],
+            'subtitle': self.catalog_program['subtitle'],
+            'category': self.catalog_program['type'],
+            'marketing_slug': self.catalog_program['marketing_slug'],
+            'organizations': [
+                {
+                    'display_name': organization['name'],
+                    'key': organization['key']
+                } for organization in self.catalog_program['authoring_organizations']
+            ],
+            'course_codes': [
+                {
+                    'display_name': course['title'],
+                    'key': course['key'],
+                    'organization': {
+                        'display_name': course['owners'][0]['name'],
+                        'key': course['owners'][0]['key']
+                    },
+                    'run_modes': [
+                        {
+                            'course_key': run['key'],
+                            'run_key': CourseKey.from_string(run['key']).run,
+                            'mode_slug': 'verified'
+                        } for run in course['course_runs']
+                    ],
+                } for course in self.catalog_program['courses']
+            ],
+            'banner_image_urls': {
+                'w1440h480': self.catalog_program['banner_image']['large']['url'],
+                'w726h242': self.catalog_program['banner_image']['medium']['url'],
+                'w435h145': self.catalog_program['banner_image']['small']['url'],
+                'w348h116': self.catalog_program['banner_image']['x-small']['url'],
+            },
+        }
+
+        self.assertEqual(munged, expected)
+
+
+@mock.patch(UTILS_MODULE + '.get_edx_api_data')
+@mock.patch('config_models.models.cache.get', return_value=None)
 class TestGetCourseRun(mixins.CatalogIntegrationMixin, TestCase):
     """Tests covering retrieval of course runs from the catalog service."""
     def setUp(self):

openedx/core/djangoapps/catalog/utils.py

@@ -3,12 +3,114 @@ from urlparse import urlparse
 
 from django.conf import settings
 from edx_rest_api_client.client import EdxRestApiClient
+from opaque_keys.edx.keys import CourseKey
 
 from openedx.core.djangoapps.catalog.models import CatalogIntegration
 from openedx.core.lib.edx_api_utils import get_edx_api_data
 from openedx.core.lib.token_utils import JwtBuilder
 
 
+def create_catalog_api_client(user, catalog_integration):
+    """Returns an API client which can be used to make catalog API requests."""
+    scopes = ['email', 'profile']
+    expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
+    jwt = JwtBuilder(user).build_token(scopes, expires_in)
+
+    return EdxRestApiClient(catalog_integration.internal_api_url, jwt=jwt)
+
+
+def get_programs(user, uuid=None, type=None):  # pylint: disable=redefined-builtin
+    """Retrieve marketable programs from the catalog service.
+
+    Keyword Arguments:
+        uuid (string): UUID identifying a specific program.
+        type (string): Filter programs by type (e.g., "MicroMasters" will only return MicroMasters programs).
+
+    Returns:
+        list of dict, representing programs.
+        dict, if a specific program is requested.
+    """
+    catalog_integration = CatalogIntegration.current()
+
+    if catalog_integration.enabled:
+        api = create_catalog_api_client(user, catalog_integration)
+
+        cache_key = '{base}.programs{type}'.format(
+            base=catalog_integration.CACHE_KEY,
+            type='.' + type if type else ''
+        )
+
+        querystring = {'marketable': 1}
+        if type:
+            querystring['type'] = type
+
+        return get_edx_api_data(
+            catalog_integration,
+            user,
+            'programs',
+            resource_id=uuid,
+            cache_key=cache_key if catalog_integration.is_cache_enabled else None,
+            api=api,
+            querystring=querystring,
+        )
+    else:
+        return []
+
+
+def munge_catalog_program(catalog_program):
+    """Make a program from the catalog service look like it came from the programs service.
+
+    Catalog-based MicroMasters need to be displayed in the LMS. However, the LMS
+    currently retrieves all program data from the soon-to-be-retired programs service.
+    Consuming program data exclusively from the catalog service would have taken more time
+    than we had prior to the MicroMasters launch. This is a functional middle ground
+    introduced by ECOM-5460. Cleaning up this debt is tracked by ECOM-4418.
+
+    Arguments:
+        catalog_program (dict): The catalog service's representation of a program.
+
+    Return:
+        dict, imitating the schema used by the programs service.
+    """
+    return {
+        'id': catalog_program['uuid'],
+        'name': catalog_program['title'],
+        'subtitle': catalog_program['subtitle'],
+        'category': catalog_program['type'],
+        'marketing_slug': catalog_program['marketing_slug'],
+        'organizations': [
+            {
+                'display_name': organization['name'],
+                'key': organization['key']
+            } for organization in catalog_program['authoring_organizations']
+        ],
+        'course_codes': [
+            {
+                'display_name': course['title'],
+                'key': course['key'],
+                'organization': {
+                    # The Programs schema only supports one organization here.
+                    'display_name': course['owners'][0]['name'],
+                    'key': course['owners'][0]['key']
+                },
+                'run_modes': [
+                    {
+                        'course_key': run['key'],
+                        'run_key': CourseKey.from_string(run['key']).run,
+                        'mode_slug': 'verified'
+                    } for run in course['course_runs']
+                ],
+            } for course in catalog_program['courses']
+        ],
+        'banner_image_urls': {
+            'w1440h480': catalog_program['banner_image']['large']['url'],
+            'w726h242': catalog_program['banner_image']['medium']['url'],
+            'w435h145': catalog_program['banner_image']['small']['url'],
+            'w348h116': catalog_program['banner_image']['x-small']['url'],
+        },
+    }
+
+
 def get_course_run(course_key, user):
     """Get a course run's data from the course catalog service.
 
@@ -22,10 +124,7 @@ def get_course_run(course_key, user):
     catalog_integration = CatalogIntegration.current()
 
     if catalog_integration.enabled:
-        scopes = ['email', 'profile']
-        expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
-        jwt = JwtBuilder(user).build_token(scopes, expires_in)
-        api = EdxRestApiClient(catalog_integration.internal_api_url, jwt=jwt)
+        api = create_catalog_api_client(user, catalog_integration)
 
         data = get_edx_api_data(
             catalog_integration,

openedx/core/djangoapps/programs/utils.py

@@ -14,7 +14,11 @@ import pytz
 from course_modes.models import CourseMode
 from lms.djangoapps.certificates import api as certificate_api
 from lms.djangoapps.commerce.utils import EcommerceService
-from openedx.core.djangoapps.catalog.utils import get_run_marketing_url
+from openedx.core.djangoapps.catalog.utils import (
+    get_programs as get_catalog_programs,
+    munge_catalog_program,
+    get_run_marketing_url,
+)
 from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
 from openedx.core.djangoapps.programs.models import ProgramsApiConfig
 from openedx.core.lib.edx_api_utils import get_edx_api_data
@@ -31,6 +35,7 @@ DEFAULT_ENROLLMENT_START_DATE = datetime.datetime(1900, 1, 1, tzinfo=pytz.UTC)
 
 def get_programs(user, program_id=None):
     """Given a user, get programs from the Programs service.
+
     Returned value is cached depending on user permissions. Staff users making requests
     against Programs will receive unpublished programs, while regular users will only receive
     published programs.
@@ -43,6 +48,7 @@ def get_programs(user, program_id=None):
 
     Returns:
         list of dict, representing programs returned by the Programs service.
+        dict, if a specific program is requested.
     """
     programs_config = ProgramsApiConfig.current()
 
@@ -50,7 +56,15 @@ def get_programs(user, program_id=None):
     # to see them displayed immediately.
     cache_key = programs_config.CACHE_KEY if programs_config.is_cache_enabled and not user.is_staff else None
 
-    return get_edx_api_data(programs_config, user, 'programs', resource_id=program_id, cache_key=cache_key)
+    programs = get_edx_api_data(programs_config, user, 'programs', resource_id=program_id, cache_key=cache_key)
+
+    # Mix in munged MicroMasters data from the catalog.
+    if not program_id:
+        programs += [
+            munge_catalog_program(micromaster) for micromaster in get_catalog_programs(user, type='MicroMasters')
+        ]
+
+    return programs
 
 
 def get_programs_for_credentials(user, programs_credentials):