Merge pull request #226 from edx/clintonb/program-types

Added edX program types

.travis.yml

@@ -23,7 +23,7 @@ before_install:
 
 install:
     - pip install -U pip wheel codecov
-    - pip install -r requirements/test.txt
+    - make requirements
     - make requirements.js
 
 before_script:
@@ -31,6 +31,11 @@ before_script:
   - sleep 10
 
 script:
+    # Ensure documentation can be compiled
+    - cd docs && make html
+    - cd ..
+
+    # Compile static assets and validate the code
     - make static -e DJANGO_SETTINGS_MODULE="course_discovery.settings.test"
     - make validate
 

course_discovery/apps/api/serializers.py

@@ -10,7 +10,7 @@ from rest_framework.fields import DictField
 
 from course_discovery.apps.catalogs.models import Catalog
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseRun, Image, Organization, Person, Prerequisite, Seat, Subject, Video, Program
+    Course, CourseRun, Image, Organization, Person, Prerequisite, Seat, Subject, Video, Program, ProgramType,
 )
 from course_discovery.apps.course_metadata.search_indexes import CourseIndex, CourseRunIndex, ProgramIndex
 
@@ -50,7 +50,7 @@ PROGRAM_FACET_FIELD_OPTIONS = {
 }
 
 PROGRAM_SEARCH_FIELDS = (
-    'text', 'uuid', 'title', 'subtitle', 'category', 'marketing_url', 'organizations', 'content_type', 'status',
+    'text', 'uuid', 'title', 'subtitle', 'type', 'marketing_url', 'organizations', 'content_type', 'status',
     'card_image_url',
 )
 
@@ -264,10 +264,11 @@ class ContainedCoursesSerializer(serializers.Serializer):
 class ProgramSerializer(serializers.ModelSerializer):
     courses = CourseSerializer(many=True)
     authoring_organizations = OrganizationSerializer(many=True)
+    type = serializers.SlugRelatedField(slug_field='name', queryset=ProgramType.objects.all())
 
     class Meta:
         model = Program
-        fields = ('uuid', 'title', 'subtitle', 'category', 'marketing_slug', 'marketing_url', 'card_image_url',
+        fields = ('uuid', 'title', 'subtitle', 'type', 'marketing_slug', 'marketing_url', 'card_image_url',
                   'banner_image_url', 'authoring_organizations', 'courses',)
         read_only_fields = ('uuid', 'marketing_url',)
 

course_discovery/apps/api/tests/test_serializers.py

@@ -181,7 +181,7 @@ class ProgramSerializerTests(TestCase):
             'uuid': str(program.uuid),
             'title': program.title,
             'subtitle': program.subtitle,
-            'category': program.category,
+            'type': program.type.name,
             'marketing_slug': program.marketing_slug,
             'marketing_url': program.marketing_url,
             'card_image_url': program.card_image_url,
@@ -409,7 +409,7 @@ class ProgramSearchSerializerTests(TestCase):
             'uuid': str(program.uuid),
             'title': program.title,
             'subtitle': program.subtitle,
-            'category': program.category,
+            'type': program.type.name,
             'marketing_url': program.marketing_url,
             'organizations': expected_organizations,
             'content_type': 'program',

course_discovery/apps/course_metadata/data_loaders/api.py

@@ -7,7 +7,7 @@ from course_discovery.apps.core.models import Currency
 from course_discovery.apps.course_metadata.data_loaders import AbstractDataLoader
 from course_discovery.apps.course_metadata.models import (
     Image, Video, Organization, Seat, CourseRun, Program, Course, CourseOrganization,
-)
+    ProgramType)
 
 logger = logging.getLogger(__name__)
 
@@ -240,6 +240,11 @@ class ProgramsApiDataLoader(AbstractDataLoader):
     """ Loads programs from the Programs API. """
     image_width = 1440
     image_height = 480
+    XSERIES = None
+
+    def __init__(self, partner, api_url, access_token=None, token_type=None):
+        super(ProgramsApiDataLoader, self).__init__(partner, api_url, access_token, token_type)
+        self.XSERIES = ProgramType.objects.get(name='XSeries')
 
     def ingest(self):
         api_url = self.partner.programs_api_url
@@ -265,14 +270,17 @@ class ProgramsApiDataLoader(AbstractDataLoader):
 
         logger.info('Retrieved %d programs from %s.', count, api_url)
 
+    def _get_uuid(self, body):
+        return body['uuid']
+
     def update_program(self, body):
-        uuid = body['uuid']
+        uuid = self._get_uuid(body)
 
         try:
             defaults = {
                 'title': body['name'],
                 'subtitle': body['subtitle'],
-                'category': body['category'],
+                'type': self.XSERIES,
                 'status': body['status'],
                 'marketing_slug': body['marketing_slug'],
                 'banner_image_url': self._get_banner_image_url(body),
@@ -280,35 +288,40 @@ class ProgramsApiDataLoader(AbstractDataLoader):
             }
 
             program, __ = Program.objects.update_or_create(uuid=uuid, defaults=defaults)
-
-            org_keys = [org['key'] for org in body['organizations']]
-            organizations = Organization.objects.filter(key__in=org_keys, partner=self.partner)
-
-            if len(org_keys) != organizations.count():
-                logger.error('Organizations for program [%s] are invalid!', uuid)
-
-            program.authoring_organizations.clear()
-            program.authoring_organizations.add(*organizations)
-
-            course_run_keys = set()
-            for course_code in body.get('course_codes', []):
-                course_run_keys.update([course_run['course_key'] for course_run in course_code['run_modes']])
-
-            # The course_code key field is technically useless, so we must build the course list from the
-            # associated course runs.
-            courses = Course.objects.filter(course_runs__key__in=course_run_keys).distinct()
-            program.courses.clear()
-            program.courses.add(*courses)
-
-            excluded_course_runs = CourseRun.objects.filter(course__in=courses). \
-                exclude(key__in=course_run_keys)
-            program.excluded_course_runs.clear()
-            program.excluded_course_runs.add(*excluded_course_runs)
-
+            self._update_program_organizations(body, program)
+            self._update_program_courses_and_runs(body, program)
             program.save()
         except Exception:  # pylint: disable=broad-except
             logger.exception('Failed to load program %s', uuid)
 
+    def _update_program_courses_and_runs(self, body, program):
+        course_run_keys = set()
+        for course_code in body.get('course_codes', []):
+            course_run_keys.update([course_run['course_key'] for course_run in course_code['run_modes']])
+
+        # The course_code key field is technically useless, so we must build the course list from the
+        # associated course runs.
+        courses = Course.objects.filter(course_runs__key__in=course_run_keys).distinct()
+        program.courses.clear()
+        program.courses.add(*courses)
+
+        # Do a diff of all the course runs and the explicitly-associated course runs to determine
+        # which course runs should be explicitly excluded.
+        excluded_course_runs = CourseRun.objects.filter(course__in=courses).exclude(key__in=course_run_keys)
+        program.excluded_course_runs.clear()
+        program.excluded_course_runs.add(*excluded_course_runs)
+
+    def _update_program_organizations(self, body, program):
+        uuid = self._get_uuid(body)
+        org_keys = [org['key'] for org in body['organizations']]
+        organizations = Organization.objects.filter(key__in=org_keys, partner=self.partner)
+
+        if len(org_keys) != organizations.count():
+            logger.error('Organizations for program [%s] are invalid!', uuid)
+
+        program.authoring_organizations.clear()
+        program.authoring_organizations.add(*organizations)
+
     def _get_banner_image_url(self, body):
         image_key = 'w{width}h{height}'.format(width=self.image_width, height=self.image_height)
         image_url = body.get('banner_image_urls', {}).get(image_key)

course_discovery/apps/course_metadata/data_loaders/marketing_site.py

@@ -274,7 +274,6 @@ class XSeriesMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
 
         data = {
             'subtitle': data.get('field_xseries_subtitle_short'),
-            'category': 'XSeries',
             'card_image_url': card_image_url,
             'overview': overview,
             'video': self.get_or_create_video(video_url)

course_discovery/apps/course_metadata/data_loaders/tests/test_api.py

@@ -14,12 +14,12 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
 from course_discovery.apps.course_metadata.data_loaders.tests import JSON
 from course_discovery.apps.course_metadata.data_loaders.tests.mixins import ApiClientTestMixin, DataLoaderTestMixin
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseRun, Image, Organization, Seat, Program
+    Course, CourseRun, Image, Organization, Seat, Program, ProgramType,
 )
 from course_discovery.apps.course_metadata.tests import mock_data
 from course_discovery.apps.course_metadata.tests.factories import (
-    CourseRunFactory, SeatFactory, ImageFactory, PersonFactory, VideoFactory,
-    OrganizationFactory, CourseFactory)
+    CourseRunFactory, SeatFactory, ImageFactory, PersonFactory, VideoFactory, OrganizationFactory, CourseFactory,
+)
 
 LOGGER_PATH = 'course_discovery.apps.course_metadata.data_loaders.api.logger'
 
@@ -392,9 +392,11 @@ class ProgramsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCa
         program = Program.objects.get(uuid=AbstractDataLoader.clean_string(body['uuid']), partner=self.partner)
 
         self.assertEqual(program.title, body['name'])
-        for attr in ('subtitle', 'category', 'status', 'marketing_slug',):
+        for attr in ('subtitle', 'status', 'marketing_slug',):
             self.assertEqual(getattr(program, attr), AbstractDataLoader.clean_string(body[attr]))
 
+        self.assertEqual(program.type, ProgramType.objects.get(name='XSeries'))
+
         keys = [org['key'] for org in body['organizations']]
         expected_organizations = list(Organization.objects.filter(key__in=keys))
         self.assertEqual(keys, [org.key for org in expected_organizations])

course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py

@@ -328,7 +328,6 @@ class XSeriesMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMix
         self.assertEqual(program.overview, overview)
 
         self.assertEqual(program.subtitle, data.get('field_xseries_subtitle_short'))
-        self.assertEqual(program.category, 'XSeries')
 
         card_image_url = data.get('field_card_image', {}).get('url')
         self.assertEqual(program.card_image_url, card_image_url)

course_discovery/apps/course_metadata/models.py

@@ -411,6 +411,9 @@ class SeatType(TimeStampedModel):
     name = models.CharField(max_length=64, unique=True)
     slug = AutoSlugField(populate_from='name')
 
+    def __str__(self):
+        return self.name
+
 
 class Seat(TimeStampedModel):
     """ Seat model. """
@@ -503,6 +506,9 @@ class ProgramType(TimeStampedModel):
                               'of the course counted toward the completion of the program.'),
     )
 
+    def __str__(self):
+        return self.name
+
 
 class Program(TimeStampedModel):
     class ProgramStatus(DjangoChoices):
@@ -555,7 +561,7 @@ class Program(TimeStampedModel):
     @property
     def marketing_url(self):
         if self.marketing_slug:
-            path = '{category}/{slug}'.format(category=self.category, slug=self.marketing_slug)
+            path = '{type}/{slug}'.format(type=self.type.name.lower(), slug=self.marketing_slug)
             return urljoin(self.partner.marketing_site_url_root, path)
 
         return None

course_discovery/apps/course_metadata/search_indexes.py

@@ -112,7 +112,7 @@ class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
     uuid = indexes.CharField(model_attr='uuid')
     title = indexes.CharField(model_attr='title')
     subtitle = indexes.CharField(model_attr='subtitle')
-    category = indexes.CharField(model_attr='category', faceted=True)
+    type = indexes.CharField(model_attr='type__name', faceted=True)
     marketing_url = indexes.CharField(null=True)
     organizations = indexes.MultiValueField(faceted=True)
     authoring_organizations = indexes.MultiValueField(faceted=True)

course_discovery/apps/course_metadata/tests/factories.py

@@ -12,7 +12,7 @@ from course_discovery.apps.core.tests.factories import PartnerFactory
 from course_discovery.apps.core.tests.utils import FuzzyURL
 from course_discovery.apps.course_metadata.models import (
     Course, CourseRun, Organization, Person, Image, Video, Subject, Seat, Prerequisite, LevelType, Program,
-    AbstractSocialNetworkModel, CourseRunSocialNetwork, PersonSocialNetwork, ProgramType
+    AbstractSocialNetworkModel, CourseRunSocialNetwork, PersonSocialNetwork, ProgramType, SeatType,
 )
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
@@ -173,8 +173,8 @@ class ProgramFactory(factory.django.DjangoModelFactory):
     title = factory.Sequence(lambda n: 'test-program-{}'.format(n))  # pylint: disable=unnecessary-lambda
     uuid = factory.LazyFunction(uuid4)
     subtitle = 'test-subtitle'
-    category = 'xseries'
-    status = 'unpublished'
+    type = factory.SubFactory(ProgramTypeFactory)
+    status = Program.ProgramStatus.Unpublished
     marketing_slug = factory.Sequence(lambda n: 'test-slug-{}'.format(n))  # pylint: disable=unnecessary-lambda
     banner_image_url = FuzzyText(prefix='https://example.com/program/banner')
     card_image_url = FuzzyText(prefix='https://example.com/program/card')
@@ -229,3 +229,10 @@ class CourseRunSocialNetworkFactory(AbstractSocialNetworkModelFactory):
 
     class Meta:
         model = CourseRunSocialNetwork
+
+
+class SeatTypeFactory(factory.django.DjangoModelFactory):
+    class Meta(object):
+        model = SeatType
+
+    name = FuzzyText()

course_discovery/apps/course_metadata/tests/test_models.py

@@ -269,8 +269,8 @@ class ProgramTests(TestCase):
 
     def test_marketing_url(self):
         """ Verify the property creates a complete marketing URL. """
-        expected = '{root}/{category}/{slug}'.format(root=self.program.partner.marketing_site_url_root.strip('/'),
-                                                     category=self.program.category, slug=self.program.marketing_slug)
+        expected = '{root}/{type}/{slug}'.format(root=self.program.partner.marketing_site_url_root.strip('/'),
+                                                 type=self.program.type.name.lower(), slug=self.program.marketing_slug)
         self.assertEqual(self.program.marketing_url, expected)
 
     def test_marketing_url_without_slug(self):
@@ -316,7 +316,7 @@ class ProgramTests(TestCase):
         factories.SeatFactory(type='credit', currency=currency, course_run=course_run, price=600)
         factories.SeatFactory(type='verified', currency=currency, course_run=course_run, price=100)
         applicable_seat_types = SeatType.objects.filter(slug__in=['credit', 'verified'])
-        program_type = factories.ProgramTypeFactory(name='XSeries', applicable_seat_types=applicable_seat_types)
+        program_type = factories.ProgramTypeFactory(applicable_seat_types=applicable_seat_types)
         program = factories.ProgramFactory(type=program_type, courses=[course_run.course])
 
         expected_price_ranges = [{'currency': 'USD', 'min': Decimal(100), 'max': Decimal(600)}]
@@ -374,3 +374,17 @@ class CourseSocialNetworkTests(TestCase):
         factories.CourseRunSocialNetworkFactory(course_run=self.course_run, type='facebook')
         with self.assertRaises(IntegrityError):
             factories.CourseRunSocialNetworkFactory(course_run=self.course_run, type='facebook')
+
+
+class SeatTypeTests(TestCase):
+    """ Tests of the SeatType model. """
+    def test_str(self):
+        seat_type = factories.SeatTypeFactory()
+        self.assertEqual(str(seat_type), seat_type.name)
+
+
+class ProgramTypeTests(TestCase):
+    """ Tests of the ProgramType model. """
+    def test_str(self):
+        program_type = factories.ProgramTypeFactory()
+        self.assertEqual(str(program_type), program_type.name)

course_discovery/apps/edx_catalog_extensions/__init__.py

+""" edX extensions for the Catalog (Course Discovery) Service.
+
+This Django app is intended for usage by edX. Its usage is NOT supported for OpenEdX.
+"""

course_discovery/apps/edx_catalog_extensions/migrations/0001_create_program_types.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+PAID_SEAT_TYPES = ('credit', 'professional', 'verified',)
+PROGRAM_TYPES = ('XSeries', 'MicroMasters',)
+
+
+def add_program_types(apps, schema_editor):
+    SeatType = apps.get_model('course_metadata', 'SeatType')
+    ProgramType = apps.get_model('course_metadata', 'ProgramType')
+
+    seat_types = SeatType.objects.filter(slug__in=PAID_SEAT_TYPES)
+
+    for name in PROGRAM_TYPES:
+        program_type, __ = ProgramType.objects.update_or_create(name=name)
+        program_type.applicable_seat_types.clear()
+        program_type.applicable_seat_types.add(*seat_types)
+        program_type.save()
+
+
+def drop_program_types(apps, schema_editor):
+    ProgramType = apps.get_model('course_metadata', 'ProgramType')
+    ProgramType.objects.filter(name__in=PROGRAM_TYPES).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('course_metadata', '0012_create_seat_types'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_program_types, drop_program_types)
+    ]

course_discovery/settings/docs_settings.py

 # The bare minimum needed for Sphinx to import each file and generate documentation.
 
-from course_discovery.settings.base import INSTALLED_APPS
+# noinspection PyUnresolvedReferences
+from course_discovery.settings.base import *
 
 DATABASES = {
     'default': {

course_discovery/settings/test.py

@@ -3,6 +3,7 @@ from course_discovery.settings.base import *
 # TEST SETTINGS
 INSTALLED_APPS += (
     'django_nose',
+    'course_discovery.apps.edx_catalog_extensions',
 )
 
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

course_discovery/templates/search/indexes/course_metadata/program_text.txt

 {{ object.uuid }}
 {{ object.name }}
-{{ object.category }}
+{{ object.type.name }}
 {{ object.status }}
 {{ object.marketing_slug|default:'' }}
 

docs/edx_extensions.rst

+.. _edx-extensions:
+
+edX Extensions
+==============
+edX.org has its own app, ``edx_catalog_extensions``, to contain edX-specific customizations. These include data
+migrations, management commands, etc. specific to edX. Non-edX users should NOT use this app. This app
+is explicitly disabled by default in all non-test environments.
+
+At some point in the future this app will be moved to a separate repository to avoid confusion. It exists here now
+until we can determine what other edX-specific components need to be extracted from the general project.
+
+edX developers should add ``'course_discovery.apps.edx_catalog_extensions'`` to the ``INSTALLED_APPS`` setting in a
+``private.py`` settings file.

docs/getting_started.rst

@@ -30,6 +30,8 @@ When developing locally, it may be useful to have settings overrides that you do
 If you need such overrides, create a file :file:`course_discovery/settings/private.py`. This file's values are
 read by :file:`course_discovery/settings/local.py`, but ignored by Git.
 
+If you are an edX employee/developer, see :ref:`edx-extensions`.
+
 
 Configure edX OpenID Connect (OIDC)
 -----------------------------------

docs/index.rst

@@ -18,3 +18,4 @@ A service for serving course discovery and marketing information to partners, mo
    testing
    features
    internationalization
+   edx_extensions
\ No newline at end of file