Merge pull request #249 from edx/clintonb/course-model-update

Updated course and course run data models and data loaders

course_discovery/apps/api/fields.py

@@ -24,3 +24,17 @@ class StdImageSerializerField(serializers.Field):
     def to_internal_value(self, obj):
         """ We do not need to save/edit this banner image through serializer yet """
         pass
+
+
+class ImageField(serializers.Field):  # pylint:disable=abstract-method
+    """ This field mimics the format of `ImageSerializer`. It is intended to aid the transition away from the
+    `Image` model to simple URLs.
+    """
+
+    def to_representation(self, value):
+        return {
+            'src': value,
+            'description': None,
+            'height': None,
+            'width': None
+        }

course_discovery/apps/api/serializers.py

@@ -9,7 +9,7 @@ from rest_framework import serializers
 from rest_framework.fields import DictField
 from taggit_serializer.serializers import TagListSerializerField, TaggitSerializer
 
-from course_discovery.apps.api.fields import StdImageSerializerField
+from course_discovery.apps.api.fields import StdImageSerializerField, ImageField
 from course_discovery.apps.catalogs.models import Catalog
 from course_discovery.apps.course_metadata.models import (
     Course, CourseRun, Image, Organization, Person, Prerequisite, Seat, Subject, Video, Program, ProgramType,
@@ -210,10 +210,10 @@ class CourseRunSerializer(TimestampModelSerializer):
         help_text=_('Language in which the course is administered')
     )
     transcript_languages = serializers.SlugRelatedField(many=True, read_only=True, slug_field='code')
-    image = ImageSerializer()
+    image = ImageField(read_only=True, source='card_image_url')
     video = VideoSerializer()
     seats = SeatSerializer(many=True)
-    instructors = PersonSerializer(many=True)
+    instructors = serializers.SerializerMethodField(help_text='This field is deprecated. Use staff.')
     staff = PersonSerializer(many=True)
     marketing_url = serializers.SerializerMethodField()
     level_type = serializers.SlugRelatedField(read_only=True, slug_field='name')
@@ -230,6 +230,9 @@ class CourseRunSerializer(TimestampModelSerializer):
     def get_marketing_url(self, obj):
         return get_marketing_url_for_user(self.context['request'].user, obj.marketing_url)
 
+    def get_instructors(self, obj):  # pylint: disable=unused-argument
+        return []
+
 
 class CourseRunWithProgramsSerializer(CourseRunSerializer):
     """A ``CourseRunSerializer`` which includes programs derived from parent course."""
@@ -254,10 +257,10 @@ class CourseSerializer(TimestampModelSerializer):
     subjects = SubjectSerializer(many=True)
     prerequisites = PrerequisiteSerializer(many=True)
     expected_learning_items = serializers.SlugRelatedField(many=True, read_only=True, slug_field='value')
-    image = ImageSerializer()
+    image = ImageField(read_only=True, source='card_image_url')
     video = VideoSerializer()
-    owners = OrganizationSerializer(many=True)
-    sponsors = OrganizationSerializer(many=True)
+    owners = OrganizationSerializer(many=True, source='authoring_organizations')
+    sponsors = OrganizationSerializer(many=True, source='sponsoring_organizations')
     course_runs = CourseRunSerializer(many=True)
     marketing_url = serializers.SerializerMethodField()
 
@@ -344,7 +347,7 @@ class AffiliateWindowSerializer(serializers.ModelSerializer):
     name = serializers.CharField(source='course_run.title')
     desc = serializers.CharField(source='course_run.short_description')
     purl = serializers.CharField(source='course_run.marketing_url')
-    imgurl = serializers.CharField(source='course_run.image')
+    imgurl = serializers.CharField(source='course_run.card_image_url')
     category = serializers.SerializerMethodField()
     price = serializers.SerializerMethodField()
 
@@ -375,13 +378,14 @@ class FlattenedCourseRunWithCourseSerializer(CourseRunSerializer):
     level_type = serializers.SerializerMethodField()
     expected_learning_items = serializers.SerializerMethodField()
     course_key = serializers.SerializerMethodField()
+    image = ImageField(read_only=True, source='card_image_url')
 
     class Meta(object):
         model = CourseRun
         fields = (
             'key', 'title', 'short_description', 'full_description', 'level_type', 'subjects', 'prerequisites',
             'start', 'end', 'enrollment_start', 'enrollment_end', 'announcement', 'seats', 'content_language',
-            'transcript_languages', 'instructors', 'staff', 'pacing_type', 'min_effort', 'max_effort', 'course_key',
+            'transcript_languages', 'staff', 'pacing_type', 'min_effort', 'max_effort', 'course_key',
             'expected_learning_items', 'image', 'video', 'owners', 'sponsors', 'modified', 'marketing_url',
         )
 
@@ -428,10 +432,10 @@ class FlattenedCourseRunWithCourseSerializer(CourseRunSerializer):
         return seats
 
     def get_owners(self, obj):
-        return ','.join([owner.key for owner in obj.course.owners.all()])
+        return ','.join([owner.key for owner in obj.course.authoring_organizations.all()])
 
     def get_sponsors(self, obj):
-        return ','.join([sponsor.key for sponsor in obj.course.sponsors.all()])
+        return ','.join([sponsor.key for sponsor in obj.course.sponsoring_organizations.all()])
 
     def get_subjects(self, obj):
         return ','.join([subject.name for subject in obj.course.subjects.all()])

course_discovery/apps/api/tests/test_fields.py

+from django.test import TestCase
+
+from course_discovery.apps.api.fields import ImageField
+
+
+class ImageFieldTests(TestCase):
+    def test_to_representation(self):
+        value = 'https://example.com/image.jpg'
+        expected = {
+            'src': value,
+            'description': None,
+            'height': None,
+            'width': None
+        }
+        self.assertEqual(ImageField().to_representation(value), expected)

course_discovery/apps/api/tests/test_serializers.py

@@ -7,6 +7,7 @@ from haystack.query import SearchQuerySet
 from opaque_keys.edx.keys import CourseKey
 from rest_framework.test import APIRequestFactory
 
+from course_discovery.apps.api.fields import ImageField
 from course_discovery.apps.api.serializers import (
     CatalogSerializer, CourseSerializer, CourseRunSerializer, ContainedCoursesSerializer, ImageSerializer,
     SubjectSerializer, PrerequisiteSerializer, VideoSerializer, OrganizationSerializer, SeatSerializer,
@@ -71,7 +72,6 @@ class CatalogSerializerTests(TestCase):
 class CourseSerializerTests(TestCase):
     def test_data(self):
         course = CourseFactory()
-        image = course.image
         video = course.video
 
         request = make_request()
@@ -88,10 +88,10 @@ class CourseSerializerTests(TestCase):
             'subjects': [],
             'prerequisites': [],
             'expected_learning_items': [],
-            'image': ImageSerializer(image).data,
+            'image': ImageField().to_representation(course.card_image_url),
             'video': VideoSerializer(video).data,
-            'owners': [],
-            'sponsors': [],
+            'owners': OrganizationSerializer(course.authoring_organizations, many=True).data,
+            'sponsors': OrganizationSerializer(course.sponsoring_organizations, many=True).data,
             'modified': json_date_format(course.modified),  # pylint: disable=no-member
             'course_runs': CourseRunSerializer(course.course_runs, many=True, context={'request': request}).data,
             'marketing_url': '{url}?{params}'.format(
@@ -106,23 +106,12 @@ class CourseSerializerTests(TestCase):
 
         self.assertDictEqual(serializer.data, expected)
 
-    def test_data_url_none(self):
-        """
-        Verify that the course serializer does not attempt to add URL
-        parameters if the course has no marketing URL.
-        """
-        course = CourseFactory(marketing_url=None)
-        request = make_request()
-        serializer = CourseSerializer(course, context={'request': request})
-        self.assertEqual(serializer.data['marketing_url'], None)
-
 
 class CourseRunSerializerTests(TestCase):
     def test_data(self):
         request = make_request()
         course_run = CourseRunFactory()
         course = course_run.course
-        image = course_run.image
         video = course_run.video
         serializer = CourseRunWithProgramsSerializer(course_run, context={'request': request})
         ProgramFactory(courses=[course])
@@ -138,7 +127,7 @@ class CourseRunSerializerTests(TestCase):
             'enrollment_start': json_date_format(course_run.enrollment_start),
             'enrollment_end': json_date_format(course_run.enrollment_end),
             'announcement': json_date_format(course_run.announcement),
-            'image': ImageSerializer(image).data,
+            'image': ImageField().to_representation(course_run.card_image_url),
             'video': VideoSerializer(video).data,
             'pacing_type': course_run.pacing_type,
             'content_language': course_run.language.code,
@@ -163,16 +152,6 @@ class CourseRunSerializerTests(TestCase):
 
         self.assertDictEqual(serializer.data, expected)
 
-    def test_data_url_none(self):
-        """
-        Verify that the course run serializer does not attempt to add URL
-        parameters if the course has no marketing URL.
-        """
-        course_run = CourseRunFactory(marketing_url=None)
-        request = make_request()
-        serializer = CourseRunSerializer(course_run, context={'request': request})
-        self.assertEqual(serializer.data['marketing_url'], None)
-
 
 class ProgramCourseSerializerTests(TestCase):
     def setUp(self):
@@ -218,35 +197,12 @@ class ProgramCourseSerializerTests(TestCase):
         excluded_runs.append(course_runs[0])
         program = ProgramFactory(courses=[course], excluded_course_runs=excluded_runs)
 
-        serializer = ProgramCourseSerializer(
-            course,
-            context={'request': self.request, 'program': program}
-        )
-
-        expected = {
-            'key': course.key,
-            'title': course.title,
-            'short_description': course.short_description,
-            'full_description': course.full_description,
-            'level_type': course.level_type.name,
-            'subjects': [],
-            'prerequisites': [],
-            'expected_learning_items': [],
-            'image': ImageSerializer(course.image).data,
-            'video': VideoSerializer(course.video).data,
-            'owners': [],
-            'sponsors': [],
-            'modified': json_date_format(course.modified),  # pylint: disable=no-member
-            'course_runs': CourseRunSerializer([course_runs[1]], many=True, context={'request': self.request}).data,
-            'marketing_url': '{url}?{params}'.format(
-                url=course.marketing_url,
-                params=urlencode({
-                    'utm_source': self.request.user.username,
-                    'utm_medium': self.request.user.referral_tracking_id,
-                })
-            ),
-        }
+        serializer_context = {'request': self.request, 'program': program}
+        serializer = ProgramCourseSerializer(course, context=serializer_context)
 
+        expected = CourseSerializer(course, context=serializer_context).data
+        expected['course_runs'] = CourseRunSerializer([course_runs[1]], many=True,
+                                                      context={'request': self.request}).data
         self.assertDictEqual(serializer.data, expected)
 
 
@@ -496,7 +452,7 @@ class AffiliateWindowSerializerTests(TestCase):
                 'actualp': seat.price
             },
             'currency': seat.currency.code,
-            'imgurl': course_run.image.src,
+            'imgurl': course_run.card_image_url,
             'category': 'Other Experiences'
         }
         self.assertDictEqual(serializer.data, expected)
@@ -535,7 +491,7 @@ class CourseRunSearchSerializerTests(TestCase):
             'org': course_run_key.org,
             'number': course_run_key.course,
             'seat_types': course_run.seat_types,
-            'image_url': course_run.image_url,
+            'image_url': course_run.card_image_url,
             'type': course_run.type,
             'level_type': course_run.level_type.name,
             'availability': course_run.availability,

course_discovery/apps/api/v1/tests/test_views/test_affiliate_window.py

@@ -103,7 +103,7 @@ class AffiliateWindowViewSetTests(ElasticsearchTestMixin, SerializationMixin, AP
         self.assertEqual(content.find('name').text, self.course_run.title)
         self.assertEqual(content.find('desc').text, self.course_run.short_description)
         self.assertEqual(content.find('purl').text, self.course_run.marketing_url)
-        self.assertEqual(content.find('imgurl').text, self.course_run.image.src)
+        self.assertEqual(content.find('imgurl').text, self.course_run.card_image_url)
         self.assertEqual(content.find('price/actualp').text, str(seat.price))
         self.assertEqual(content.find('currency').text, seat.currency.code)
         self.assertEqual(content.find('category').text, AffiliateWindowSerializer.CATEGORY)

course_discovery/apps/api/v1/tests/test_views/test_catalogs.py

@@ -168,7 +168,7 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
         response = self.client.get(url)
 
         course_run = self.serialize_catalog_flat_course_run(self.course_run)
-        course_run_csv = ','.join([
+        expected = ','.join([
             course_run['key'],
             course_run['title'],
             course_run['pacing_type'],
@@ -181,9 +181,9 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
             course_run['short_description'],
             course_run['marketing_url'],
             course_run['image']['src'],
-            course_run['image']['description'],
-            str(course_run['image']['height']),
-            str(course_run['image']['width']),
+            '',
+            '',
+            '',
             course_run['video']['src'],
             course_run['video']['description'],
             course_run['video']['image']['src'],
@@ -219,7 +219,7 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
         ])
 
         self.assertEqual(response.status_code, 200)
-        self.assertIn(course_run_csv, response.content.decode('utf-8'))
+        self.assertIn(expected, response.content.decode('utf-8'))
 
     def test_get(self):
         """ Verify the endpoint returns the details for a single catalog. """

course_discovery/apps/core/models.py

@@ -84,3 +84,7 @@ class Partner(TimeStampedModel):
     class Meta:
         verbose_name = _('Partner')
         verbose_name_plural = _('Partners')
+
+    @property
+    def has_marketing_site(self):
+        return bool(self.marketing_site_url_root)

course_discovery/apps/core/tests/test_models.py

 """ Tests for core models. """
-
+import ddt
 from django.test import TestCase
 from social.apps.django_app.default.models import UserSocialAuth
 
@@ -56,6 +56,7 @@ class CurrencyTests(TestCase):
         self.assertEqual(str(instance), '{code} - {name}'.format(code=code, name=name))
 
 
+@ddt.ddt
 class PartnerTests(TestCase):
     """ Tests for the Partner class. """
 
@@ -64,3 +65,13 @@ class PartnerTests(TestCase):
 
         partner = PartnerFactory()
         self.assertEqual(str(partner), partner.name)
+
+    @ddt.unpack
+    @ddt.data(
+        ('', False),
+        (None, False),
+        ('https://example.com', True),
+    )
+    def test_has_marketing_site(self, marketing_site_url_root, expected):
+        partner = PartnerFactory(marketing_site_url_root=marketing_site_url_root)
+        self.assertEqual(partner.has_marketing_site, expected)  # pylint: disable=no-member

course_discovery/apps/course_metadata/admin.py

@@ -4,11 +4,6 @@ from simple_history.admin import SimpleHistoryAdmin
 from course_discovery.apps.course_metadata.models import *  # pylint: disable=wildcard-import
 
 
-class CourseOrganizationInline(admin.TabularInline):
-    model = CourseOrganization
-    extra = 1
-
-
 class SeatInline(admin.TabularInline):
     model = Seat
     extra = 1
@@ -21,19 +16,24 @@ class PositionInline(admin.TabularInline):
 
 @admin.register(Course)
 class CourseAdmin(admin.ModelAdmin):
-    inlines = (CourseOrganizationInline,)
-    list_display = ('key', 'title',)
+    list_display = ('uuid', 'key', 'title',)
     list_filter = ('partner',)
     ordering = ('key', 'title',)
-    search_fields = ('key', 'title',)
+    readonly_fields = ('uuid',)
+    search_fields = ('uuid', 'key', 'title',)
 
 
 @admin.register(CourseRun)
 class CourseRunAdmin(admin.ModelAdmin):
     inlines = (SeatInline,)
-    list_display = ('key', 'title',)
+    list_display = ('uuid', 'key', 'title',)
+    list_filter = (
+        'course__partner',
+        ('language', admin.RelatedOnlyFieldListFilter,)
+    )
     ordering = ('key',)
-    search_fields = ('key', 'title_override', 'course__title',)
+    readonly_fields = ('uuid',)
+    search_fields = ('uuid', 'key', 'title_override', 'course__title', 'slug',)
 
 
 @admin.register(Program)

course_discovery/apps/course_metadata/data_loaders/__init__.py

@@ -7,7 +7,7 @@ from edx_rest_api_client.client import EdxRestApiClient
 from opaque_keys.edx.keys import CourseKey
 
 from course_discovery.apps.core.utils import delete_orphans
-from course_discovery.apps.course_metadata.models import Image, Person, Video
+from course_discovery.apps.course_metadata.models import Image, Video
 
 
 class AbstractDataLoader(metaclass=abc.ABCMeta):
@@ -104,18 +104,17 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
         return None
 
     @classmethod
-    def convert_course_run_key(cls, course_run_key_str):
+    def get_course_key_from_course_run_key(cls, course_run_key):
         """
         Given a serialized course run key, return the corresponding
         serialized course key.
 
         Args:
-            course_run_key_str (str): The serialized course run key.
+            course_run_key (CourseKey): Course run key.
 
         Returns:
             str
         """
-        course_run_key = CourseKey.from_string(course_run_key_str)
         return '{org}+{course}'.format(org=course_run_key.org, course=course_run_key.course)
 
     @classmethod
@@ -125,10 +124,25 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
             delete_orphans(model)
 
     @classmethod
-    def get_or_create_video(cls, url):
-        video = None
+    def _get_or_create_media(cls, media_type, url):
+        media = None
 
         if url:
-            video, __ = Video.objects.get_or_create(src=url)
+            media, __ = media_type.objects.get_or_create(src=url)
+
+        return media
+
+    @classmethod
+    def get_or_create_video(cls, url, image_url=None):
+        video = cls._get_or_create_media(Video, url)
+
+        if video:
+            image = cls.get_or_create_image(image_url)
+            video.image = image
+            video.save()
 
         return video
+
+    @classmethod
+    def get_or_create_image(cls, url):
+        return cls._get_or_create_media(Image, url)

course_discovery/apps/course_metadata/data_loaders/api.py

 import logging
 from decimal import Decimal
 from io import BytesIO
-import requests
 
-from opaque_keys.edx.keys import CourseKey
+import requests
 from django.core.files import File
+from opaque_keys.edx.keys import CourseKey
 
 from course_discovery.apps.core.models import Currency
 from course_discovery.apps.course_metadata.data_loaders import AbstractDataLoader
 from course_discovery.apps.course_metadata.models import (
-    Image, Video, Organization, Seat, CourseRun, Program, Course, CourseOrganization, ProgramType,
+    Video, Organization, Seat, CourseRun, Program, Course, ProgramType,
 )
 
 logger = logging.getLogger(__name__)
@@ -44,14 +44,16 @@ class OrganizationsApiDataLoader(AbstractDataLoader):
         self.delete_orphans()
 
     def update_organization(self, body):
+        key = body['short_name']
         defaults = {
+            'key': key,
             'name': body['name'],
             'description': body['description'],
             'logo_image_url': body['logo'],
             'partner': self.partner,
         }
-        Organization.objects.update_or_create(key=body['short_name'], defaults=defaults)
-        logger.info('Processed organization "%s"', body['short_name'])
+        Organization.objects.update_or_create(key__iexact=key, defaults=defaults)
+        logger.info('Processed organization "%s"', key)
 
 
 class CoursesApiDataLoader(AbstractDataLoader):
@@ -94,52 +96,51 @@ class CoursesApiDataLoader(AbstractDataLoader):
         self.delete_orphans()
 
     def update_course(self, body):
-        # NOTE (CCB): Use the data from the CourseKey since the Course API exposes display names for org and number,
-        # which may not be unique for an organization.
-        course_run_key_str = body['id']
-        course_run_key = CourseKey.from_string(course_run_key_str)
-        organization, __ = Organization.objects.get_or_create(key=course_run_key.org,
-                                                              defaults={'partner': self.partner})
-        course_key = self.convert_course_run_key(course_run_key_str)
+        course_run_key = CourseKey.from_string(body['id'])
+        course_key = self.get_course_key_from_course_run_key(course_run_key)
+
         defaults = {
+            'key': course_key,
             'title': body['name'],
-            'partner': self.partner,
         }
-        course, __ = Course.objects.update_or_create(key=course_key, defaults=defaults)
-
-        course.organizations.clear()
-        CourseOrganization.objects.create(
-            course=course, organization=organization, relation_type=CourseOrganization.OWNER)
-
+        course, created = Course.objects.get_or_create(key__iexact=course_key, partner=self.partner, defaults=defaults)
+
+        if created:
+            # NOTE (CCB): Use the data from the CourseKey since the Course API exposes display names for org and number,
+            # which may not be unique for an organization.
+            key = course_run_key.org
+            defaults = {'key': key}
+            organization, __ = Organization.objects.get_or_create(key__iexact=key, partner=self.partner,
+                                                                  defaults=defaults)
+            course.authoring_organizations.add(organization)
+
+        logger.info('Processed course with key [%s].', course_key)
         return course
 
     def update_course_run(self, course, body):
+        key = body['id']
         defaults = {
-            'course': course,
+            'key': key,
             'start': self.parse_date(body['start']),
             'end': self.parse_date(body['end']),
             'enrollment_start': self.parse_date(body['enrollment_start']),
             'enrollment_end': self.parse_date(body['enrollment_end']),
-            'title': body['name'],
-            'short_description': body['short_description'],
-            'video': self.get_courserun_video(body),
             'pacing_type': self.get_pacing_type(body),
         }
-        # If there is no marketing site setup for this partner, use the image from the course API.
-        # If there is a marketing site defined, it takes prededence.
-        if not self.partner.marketing_site_url_root:
-            defaults.update({'image': self.get_courserun_image(body)})
-
-        CourseRun.objects.update_or_create(key=body['id'], defaults=defaults)
 
-    def get_courserun_image(self, body):
-        image = None
-        image_url = body['media'].get('image', {}).get('raw')
+        # When using a marketing site, only date and pacing information should come from the Course API
+        if not self.partner.has_marketing_site:
+            defaults.update({
+                'card_image_url': body['media'].get('image', {}).get('raw'),
+                'title_override': body['name'],
+                'short_description_override': body['short_description'],
+                'video': self.get_courserun_video(body),
+            })
 
-        if image_url:
-            image, __ = Image.objects.get_or_create(src=image_url)
+        course_run, __ = course.course_runs.update_or_create(key__iexact=key, defaults=defaults)
 
-        return image
+        logger.info('Processed course run with key [%s].', course_run.key)
+        return course_run
 
     def get_pacing_type(self, body):
         pacing = body.get('pacing')
@@ -196,7 +197,7 @@ class EcommerceApiDataLoader(AbstractDataLoader):
     def update_seats(self, body):
         course_run_key = body['id']
         try:
-            course_run = CourseRun.objects.get(key=course_run_key)
+            course_run = CourseRun.objects.get(key__iexact=course_run_key)
         except CourseRun.DoesNotExist:
             logger.warning('Could not find course run [%s]', course_run_key)
             return None

course_discovery/apps/course_metadata/data_loaders/marketing_site.py

 import abc
 import logging
-from urllib.parse import urljoin, urlencode
+from urllib.parse import urlencode
 from uuid import UUID
 
 import requests
 from django.db.models import Q
 from django.utils.functional import cached_property
+from opaque_keys.edx.keys import CourseKey
 
 from course_discovery.apps.course_metadata.data_loaders import AbstractDataLoader
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseOrganization, CourseRun, Image, LanguageTag, LevelType, Organization, Person, Subject, Program,
-    Position,
+    Course, Organization, Person, Subject, Program, Position, LevelType, CourseRun
 )
+from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
 logger = logging.getLogger(__name__)
 
 
-class DrupalApiDataLoader(AbstractDataLoader):
-    """Loads course runs from the Drupal API."""
-
-    def ingest(self):
-        api_url = self.partner.marketing_site_api_url
-        logger.info('Refreshing Courses and CourseRuns from %s...', api_url)
-        response = self.api_client.courses.get()
-
-        data = response['items']
-        logger.info('Retrieved %d course runs...', len(data))
-
-        for body in data:
-            # NOTE (CCB): Some of the entries are empty arrays. We will fix this on the Drupal side of things
-            # later (ECOM-4493). For now, ignore them.
-            if not body:
-                continue
-
-            course_run_id = body['course_id']
-            try:
-                cleaned_body = self.clean_strings(body)
-                course = self.update_course(cleaned_body)
-                self.update_course_run(course, cleaned_body)
-            except:  # pylint: disable=bare-except
-                msg = 'An error occurred while updating {course_run} from {api_url}'.format(
-                    course_run=course_run_id,
-                    api_url=api_url
-                )
-                logger.exception(msg)
-
-        # Clean Organizations separately from other orphaned instances to avoid removing all orgnaziations
-        # after an initial data load on an empty table.
-        Organization.objects.filter(courseorganization__isnull=True, authored_programs__isnull=True,
-                                    credit_backed_programs__isnull=True).delete()
-        self.delete_orphans()
-
-        logger.info('Retrieved %d course runs from %s.', len(data), api_url)
-
-    def update_course(self, body):
-        """Create or update a course from Drupal data given by `body`."""
-        course_key = self.convert_course_run_key(body['course_id'])
-        try:
-            course = Course.objects.get(key=course_key)
-        except Course.DoesNotExist:
-            logger.warning('Course not find course [%s]', course_key)
-            return None
-
-        course.full_description = self.clean_html(body['description'])
-        course.short_description = self.clean_html(body['subtitle'])
-        course.partner = self.partner
-        course.title = self.clean_html(body['title'])
-
-        level_type, __ = LevelType.objects.get_or_create(name=body['level']['title'])
-        course.level_type = level_type
-
-        self.set_subjects(course, body)
-        self.set_sponsors(course, body)
-
-        course.save()
-        return course
-
-    def set_subjects(self, course, body):
-        """Update `course` with subjects from `body`."""
-        course.subjects.clear()
-        subjects = (s['title'] for s in body['subjects'])
-        subjects = Subject.objects.filter(name__in=subjects, partner=self.partner)
-        course.subjects.add(*subjects)
-
-    def set_sponsors(self, course, body):
-        """Update `course` with sponsors from `body`."""
-        course.courseorganization_set.filter(relation_type=CourseOrganization.SPONSOR).delete()
-        for sponsor_body in body['sponsors']:
-            defaults = {
-                'name': sponsor_body['title'],
-                'logo_image_url': sponsor_body['image'],
-                'homepage_url': urljoin(self.partner.marketing_site_url_root, sponsor_body['uri']),
-            }
-            organization, __ = Organization.objects.update_or_create(key=sponsor_body['uuid'], defaults=defaults)
-            CourseOrganization.objects.create(
-                course=course,
-                organization=organization,
-                relation_type=CourseOrganization.SPONSOR
-            )
-
-    def update_course_run(self, course, body):
-        """
-        Create or update a run of `course` from Drupal data given by `body`.
-        """
-        course_run_key = body['course_id']
-        try:
-            course_run = CourseRun.objects.get(key=course_run_key)
-        except CourseRun.DoesNotExist:
-            logger.warning('Could not find course run [%s]', course_run_key)
-            return None
-
-        course_run.language = self.get_language_tag(body)
-        course_run.course = course
-        course_run.marketing_url = urljoin(self.partner.marketing_site_url_root, body['course_about_uri'])
-        course_run.start = self.parse_date(body['start'])
-        course_run.end = self.parse_date(body['end'])
-        course_run.image = self.get_courserun_image(body)
-
-        self.set_staff(course_run, body)
-
-        course_run.save()
-        return course_run
-
-    def set_staff(self, course_run, body):
-        """Update `course_run` with staff from `body`."""
-        course_run.staff.clear()
-        uuids = [staff['uuid'] for staff in body['staff']]
-        staff = Person.objects.filter(uuid_in=uuids)
-        course_run.staff.add(*staff)
-
-    def get_language_tag(self, body):
-        """Get a language tag from Drupal data given by `body`."""
-        iso_code = body['current_language']
-        if iso_code is None:
-            return None
-
-        # NOTE (CCB): Default to U.S. English for edx.org to avoid spewing
-        # unnecessary warnings.
-        if iso_code == 'en':
-            iso_code = 'en-us'
-
-        try:
-            return LanguageTag.objects.get(code=iso_code)
-        except LanguageTag.DoesNotExist:
-            logger.warning('Could not find language with ISO code [%s].', iso_code)
-            return None
-
-    def get_courserun_image(self, body):
-        image = None
-        image_url = body['image']
-
-        if image_url:
-            image, __ = Image.objects.get_or_create(src=image_url)
-
-        return image
-
-
 class AbstractMarketingSiteDataLoader(AbstractDataLoader):
     def __init__(self, partner, api_url, access_token=None, token_type=None):
         super(AbstractMarketingSiteDataLoader, self).__init__(partner, api_url, access_token, token_type)
@@ -187,7 +48,11 @@ class AbstractMarketingSiteDataLoader(AbstractDataLoader):
         return session
 
     def get_query_kwargs(self):
-        return {}
+        return {
+            'type': self.node_type,
+            'max-depth': 2,
+            'load-entity-refs': 'file',
+        }
 
     def ingest(self):
         """ Load data for all supported objects (e.g. courses, runs). """
@@ -196,9 +61,6 @@ class AbstractMarketingSiteDataLoader(AbstractDataLoader):
 
         while page is not None and page >= 0:  # pragma: no cover
             kwargs = {
-                'type': self.node_type,
-                'max-depth': 2,
-                'load-entity-refs': 'subject,file,taxonomy_term,taxonomy_vocabulary,node,field_collection_item',
                 'page': page,
             }
             kwargs.update(query_kwargs)
@@ -371,20 +233,29 @@ class PersonMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
     def node_type(self):
         return 'person'
 
+    def get_query_kwargs(self):
+        kwargs = super(PersonMarketingSiteDataLoader, self).get_query_kwargs()
+        # NOTE (CCB): We need to include the nested field_collection_item data since that is where
+        # the positions are stored.
+        kwargs['load-entity-refs'] = 'file,field_collection_item'
+        return kwargs
+
     def process_node(self, data):
         uuid = UUID(data['uuid'])
+        slug = data['url'].split('/')[-1]
         defaults = {
             'given_name': data['field_person_first_middle_name'],
             'family_name': data['field_person_last_name'],
             'bio': self.clean_html(data['field_person_resume']['value']),
             'profile_image_url': self._get_nested_url(data.get('field_person_image')),
+            'slug': slug,
         }
         person, created = Person.objects.update_or_create(uuid=uuid, partner=self.partner, defaults=defaults)
 
         # NOTE (CCB): The AutoSlug field kicks in at creation time. We need to apply overrides in a separate
         # operation.
         if created:
-            person.slug = data['url'].split('/')[-1]
+            person.slug = slug
             person.save()
 
         self.set_position(person, data)
@@ -411,13 +282,9 @@ class PersonMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
                     organization_name = (data.get('field_person_position_org_link', {}) or {}).get('title')
 
                     if organization_name:
-                        try:
-                            # TODO Consider using Elasticsearch as a method of finding better inexact matches.
-                            organization = Organization.objects.get(
-                                Q(name__iexact=organization_name) | Q(key__iexact=organization_name) & Q(
-                                    partner=self.partner))
-                        except Organization.DoesNotExist:
-                            pass
+                        organization = Organization.objects.filter(
+                            Q(name__iexact=organization_name) | Q(key__iexact=organization_name) & Q(
+                                partner=self.partner)).first()
 
                     defaults = {
                         'title': title,
@@ -433,3 +300,147 @@ class PersonMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
                     Position.objects.update_or_create(person=person, defaults=defaults)
         except:  # pylint: disable=bare-except
             logger.exception('Failed to set position for person with UUID [%s]!', uuid)
+
+
+class CourseMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
+    LANGUAGE_MAP = {
+        'English': 'en-us',
+        '日本語': 'ja',
+        '繁體中文': 'zh-Hant',
+        'Indonesian': 'id',
+        'Italian': 'it-it',
+        'Korean': 'ko',
+        'Simplified Chinese': 'zh-Hans',
+        'Deutsch': 'de-de',
+        'Español': 'es-es',
+        'Français': 'fr-fr',
+        'Nederlands': 'nl-nl',
+        'Português': 'pt-pt',
+        'Pусский': 'ru',
+        'Svenska': 'sv-se',
+        'Türkçe': 'tr',
+        'العربية': 'ar-sa',
+        'हिंदी': 'hi',
+        '中文': 'zh-cmn',
+    }
+
+    @property
+    def node_type(self):
+        return 'course'
+
+    @classmethod
+    def get_language_tags_from_names(cls, names):
+        language_codes = [cls.LANGUAGE_MAP.get(name) for name in names]
+        return LanguageTag.objects.filter(code__in=language_codes)
+
+    def get_query_kwargs(self):
+        kwargs = super(CourseMarketingSiteDataLoader, self).get_query_kwargs()
+        # NOTE (CCB): We need to include the nested taxonomy_term data since that is where the
+        # language information is stored.
+        kwargs['load-entity-refs'] = 'file,taxonomy_term'
+        return kwargs
+
+    def process_node(self, data):
+        course_run_key = CourseKey.from_string(data['field_course_id'])
+        key = self.get_course_key_from_course_run_key(course_run_key)
+
+        defaults = {
+            'key': key,
+            'title': data['field_course_course_title']['value'],
+            'number': data['field_course_code'],
+            'full_description': self.get_description(data),
+            'video': self.get_video(data),
+            'short_description': self.clean_html(data['field_course_sub_title_short']),
+            'level_type': self.get_level_type(data['field_course_level']),
+            'card_image_url': self._get_nested_url(data.get('field_course_image_promoted')),
+        }
+        course, __ = Course.objects.update_or_create(key__iexact=key, partner=self.partner, defaults=defaults)
+
+        self.set_subjects(course, data)
+        self.set_authoring_organizations(course, data)
+        self.create_course_run(course, data)
+
+        logger.info('Processed course with key [%s].', key)
+        return course
+
+    def get_description(self, data):
+        description = (data.get('field_course_body', {}) or {}).get('value')
+        description = description or (data.get('field_course_description', {}) or {}).get('value')
+        description = description or ''
+        description = self.clean_html(description)
+        return description
+
+    def get_level_type(self, name):
+        level_type = None
+
+        if name:
+            level_type, __ = LevelType.objects.get_or_create(name=name)
+
+        return level_type
+
+    def get_video(self, data):
+        video_url = self._get_nested_url(data.get('field_product_video'))
+        image_url = self._get_nested_url(data.get('field_course_image_featured_card'))
+        return self.get_or_create_video(video_url, image_url)
+
+    def create_course_run(self, course, data):
+        uuid = data['uuid']
+        key = data['field_course_id']
+        slug = data['url'].split('/')[-1]
+        language_tags = self._extract_language_tags(data['field_course_languages'])
+        language = language_tags[0] if language_tags else None
+
+        defaults = {
+            'key': key,
+            'course': course,
+            'uuid': uuid,
+            'language': language,
+            'slug': slug,
+        }
+
+        try:
+            course_run, created = CourseRun.objects.update_or_create(key__iexact=key, defaults=defaults)
+        except TypeError:
+            # TODO Fix the data in Drupal (ECOM-5304)
+            logger.error('Multiple course runs are identified by the key [%s] or UUID [%s].', key, uuid)
+            return None
+
+        # NOTE (CCB): The AutoSlug field kicks in at creation time. We need to apply overrides in a separate
+        # operation.
+        if created:
+            course_run.slug = slug
+            course_run.save()
+
+        self.set_course_run_staff(course_run, data)
+        self.set_course_run_transcript_languages(course_run, data)
+
+        logger.info('Processed course run with UUID [%s].', uuid)
+        return course_run
+
+    def _get_objects_by_uuid(self, object_type, raw_objects_data):
+        uuids = [_object.get('uuid') for _object in raw_objects_data]
+        return object_type.objects.filter(uuid__in=uuids)
+
+    def _extract_language_tags(self, raw_objects_data):
+        language_names = [_object['name'].strip() for _object in raw_objects_data]
+        return self.get_language_tags_from_names(language_names)
+
+    def set_authoring_organizations(self, course, data):
+        schools = self._get_objects_by_uuid(Organization, data['field_course_school_node'])
+        course.authoring_organizations.clear()
+        course.authoring_organizations.add(*schools)
+
+    def set_subjects(self, course, data):
+        subjects = self._get_objects_by_uuid(Subject, data['field_course_subject'])
+        course.subjects.clear()
+        course.subjects.add(*subjects)
+
+    def set_course_run_staff(self, course_run, data):
+        staff = self._get_objects_by_uuid(Person, data['field_course_staff'])
+        course_run.staff.clear()
+        course_run.staff.add(*staff)
+
+    def set_course_run_transcript_languages(self, course_run, data):
+        language_tags = self._extract_language_tags(data['field_course_video_locale_lang'])
+        course_run.transcript_languages.clear()
+        course_run.transcript_languages.add(*language_tags)

course_discovery/apps/course_metadata/data_loaders/tests/mock_data.py

@@ -852,15 +852,15 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
 MARKETING_SITE_API_SCHOOL_BODIES = [
     {
         'field_school_description': {
-            'value': '\u003Cp\u003EHarvard University is devoted to excellence in teaching, learning, and '
+            'value': '<p>Harvard University is devoted to excellence in teaching, learning, and '
                      'research, and to developing leaders in many disciplines who make a difference globally. '
                      'Harvard faculty are engaged with teaching and research to push the boundaries of human '
                      'knowledge. The University has twelve degree-granting Schools in addition to the Radcliffe '
-                     'Institute for Advanced Study.\u003C/p\u003E\n\n\u003Cp\u003EEstablished in 1636, Harvard '
+                     'Institute for Advanced Study.</p>\n\n<p>Established in 1636, Harvard '
                      'is the oldest institution of higher education in the United States. The University, which '
                      'is based in Cambridge and Boston, Massachusetts, has an enrollment of over 20,000 degree '
                      'candidates, including undergraduate, graduate, and professional students. Harvard has more '
-                     'than 360,000 alumni around the world.\u003C/p\u003E',
+                     'than 360,000 alumni around the world.</p>',
             'format': 'standard_html'
         },
         'field_school_name': 'Harvard University',
@@ -886,17 +886,17 @@ MARKETING_SITE_API_SCHOOL_BODIES = [
     },
     {
         'field_school_description': {
-            'value': '\u003Cp\u003EMassachusetts Institute of Technology \u2014 a coeducational, privately '
+            'value': '<p>Massachusetts Institute of Technology \u2014 a coeducational, privately '
                      'endowed research university founded in 1861 \u2014 is dedicated to advancing knowledge '
                      'and educating students in science, technology, and other areas of scholarship that will '
-                     'best serve the nation and the world in the 21st century. \u003Ca href=\u0022http://web.'
-                     'mit.edu/aboutmit/\u0022 target=\u0022_blank\u0022\u003ELearn more about MIT\u003C/a\u003E'
+                     'best serve the nation and the world in the 21st century. <a href=\u0022http://web.'
+                     'mit.edu/aboutmit/\u0022 target=\u0022_blank\u0022>Learn more about MIT</a>'
                      '. Through MITx, the Institute furthers its commitment to improving education worldwide.'
-                     '\u003C/p\u003E\n\n\u003Cp\u003E\u003Cstrong\u003EMITx Courses\u003C/strong\u003E\u003Cbr '
-                     '/\u003E\nMITx courses embody the inventiveness, openness, rigor and quality that are '
+                     '</p>\n\n<p><strong>MITx Courses</strong><br '
+                     '/>\nMITx courses embody the inventiveness, openness, rigor and quality that are '
                      'hallmarks of MIT, and many use materials developed for MIT residential courses in the '
                      'Institute\u0027s five schools and 33 academic disciplines. Browse MITx courses below.'
-                     '\u003C/p\u003E\n\n\u003Cp\u003E\u00a0\u003C/p\u003E',
+                     '</p>\n\n<p>\u00a0</p>',
         },
         'field_school_name': 'MIT',
         'field_school_image_banner': {
@@ -966,14 +966,14 @@ MARKETING_SITE_API_PERSON_BODIES = [
         'field_person_position': None,
         'field_person_role': '1',
         'field_person_resume': {
-            'value': '\u003Cp\u003EProf. Cima has been a faculty member at MIT for 29 years. He earned a B.S. in '
+            'value': '<p>Prof. Cima has been a faculty member at MIT for 29 years. He earned a B.S. in '
                      'chemistry and a Ph.D. in chemical engineering, both from the University of California at '
                      'Berkeley. He was elected a Fellow of the American Ceramics Society in 1997 and was elected to '
                      'the National Academy of Engineering in 2011. Prof. Cima\u0027s research concerns advanced '
                      'technology for medical devices that are used for drug delivery and diagnostics, high-throughput '
                      'development methods for formulations of materials and pharmaceutical formulations. Prof. Cima '
                      'is an author of over 250 publications and fifty US patents, a co-inventor of MIT\u2019s '
-                     'three-dimensional printing process, and a co-founder of four companies.\u003C/p\u003E',
+                     'three-dimensional printing process, and a co-founder of four companies.</p>',
             'format': 'standard_html'
         },
         'field_person_image': {
@@ -1026,12 +1026,12 @@ MARKETING_SITE_API_PERSON_BODIES = [
         'field_person_position': None,
         'field_person_role': '1',
         'field_person_resume': {
-            'value': '\u003Cp\u003ECEO of edX and Professor of Electrical Engineering and Computer Science at MIT. '
+            'value': '<p>CEO of edX and Professor of Electrical Engineering and Computer Science at MIT. '
                      'His research focus is in parallel computer architectures and cloud software systems, and he is '
                      'a founder of several successful startups, including Tilera, a company that produces scalable '
                      'multicore processors. Prof. Agarwal won MIT\u2019s Smullin and Jamieson prizes for teaching and '
                      'co-authored the course textbook \u201cFoundations of Analog and Digital Electronic Circuits.'
-                     '\u201d\u003C/p\u003E',
+                     '\u201d</p>',
             'format': 'standard_html'
         },
         'field_person_image': {
@@ -1167,3 +1167,879 @@ MARKETING_SITE_API_PERSON_BODIES = [
         'uuid': 'abcea90b-7b9a-49a2-ba4f-165cbf6a3636',
     }
 ]
+
+MARKETING_SITE_API_COURSE_BODIES = [
+    {
+        'field_course_code': 'CS50x',
+        'field_course_course_title': {
+            'value': 'Introduction to Computer Science',
+            'format': None
+        },
+        'field_course_description': {
+            'value': '<p>CS50x is Harvard College\u0027s introduction to the intellectual enterprises of c'
+                     'omputer science and the art of programming for majors and non-majors alike, with or without '
+                     'prior programming experience. An entry-level course taught by David J. Malan, CS50x teaches '
+                     'students how to think algorithmically and solve problems efficiently. Topics include '
+                     'abstraction, algorithms, data structures, encapsulation, resource management, security, software '
+                     'engineering, and web development. Languages include C, PHP, and JavaScript plus SQL, CSS, and '
+                     'HTML. Problem sets inspired by real-world domains of biology, cryptography, finance, forensics, '
+                     'and gaming. As of Fall 2012, the on-campus version of CS50x is Harvard\u0027s second-largest '
+                     'course.</p>\n<p>This course will run again starting January 2014. <a '
+                     'href=\u0022https://www.edx.org/course/harvard-university/cs50x/introduction-computer-science/1022'
+                     '\u0022>Click here for the registration page</a> of the new version.</p>',
+            'format': 'standard_html'
+        },
+        'field_course_start_date': '1350273600',
+        'field_course_effort': '8 problem sets (15 - 20 hours each), 2 quizzes, 1 final project',
+        'field_course_faq': [
+            {
+                'question': 'Will certificates be awarded?',
+                'answer': '<p>Yes. Online learners who achieve a passing grade in CS50x will earn a '
+                          'certificate that indicates successful completion of the course, but will not include a '
+                          'specific grade. Certificates will be issued by edX under the name of HarvardX.</p>\r\n'
+            }
+        ],
+        'field_course_school_node': [
+            {
+                'uri': 'https://www.edx.org/node/242',
+                'id': '242',
+                'resource': 'node',
+                'uuid': '44022f13-20df-4666-9111-cede3e5dc5b6'
+            }
+        ],
+        'field_course_end_date': None,
+        'field_course_video': {
+            'fid': '32570',
+            'name': 'cs50 teaser final HD',
+            'mime': 'video/youtube',
+            'size': '0',
+            'url': 'http://www.youtube.com/watch?v=ZAldYMFUIac',
+            'timestamp': '1384349212',
+            'owner': {
+                'uri': 'https://www.edx.org/user/143',
+                'id': '143',
+                'resource': 'user',
+                'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
+            },
+            'uuid': '51642ba0-ff0f-4fad-b109-e55376f35b29'
+        },
+        'field_course_resources': [],
+        'field_course_sub_title_long': {
+            'value': '<p>An introduction to the intellectual enterprises of computer science and the art of '
+                     'programming.</p>\n',
+            'format': 'plain_text'
+        },
+        'field_course_subject': [
+            {
+                'uri': 'https://www.edx.org/node/375',
+                'id': '375',
+                'resource': 'node',
+                'uuid': 'e52e2134-a4e4-4fcb-805f-cbef40812580'
+            },
+            {
+                'uri': 'https://www.edx.org/node/577',
+                'id': '577',
+                'resource': 'node',
+                'uuid': '0d7bb9ed-4492-419a-bb44-415adafd9406'
+            }
+        ],
+        'field_course_statement_title': None,
+        'field_course_statement_body': [],
+        'field_course_status': 'past',
+        'field_course_start_override': None,
+        'field_course_email': None,
+        'field_course_syllabus': [],
+        'field_course_prerequisites': {
+            'value': '<p>None. CS50x is designed for students with or without prior programming experience.</p>',
+            'format': 'standard_html'
+        },
+        'field_course_staff': [
+            {
+                'uri': 'https://www.edx.org/node/349',
+                'id': '349',
+                'resource': 'node',
+                'uuid': '1752b28e-8ac9-40a0-b468-326e03cafdd4'
+            },
+            {
+                'uri': 'https://www.edx.org/node/350',
+                'id': '350',
+                'resource': 'node',
+                'uuid': 'c5ba296e-bc91-4e5e-8d59-77f425f0863f'
+            },
+            {
+                'uri': 'https://www.edx.org/node/351',
+                'id': '351',
+                'resource': 'node',
+                'uuid': '6fec9136-5f1d-4205-8da2-a354c678c653'
+            },
+            {
+                'uri': 'https://www.edx.org/node/352',
+                'id': '352',
+                'resource': 'node',
+                'uuid': 'e1080080-98b4-4427-9004-3c331c8e6d05'
+            },
+            {
+                'uri': 'https://www.edx.org/node/353',
+                'id': '353',
+                'resource': 'node',
+                'uuid': 'cb6cde02-5bb3-45ab-9616-57c33d622ccc'
+            }
+        ],
+        'field_course_staff_override': 'D. Malan, N. Hardison, R. Bowden',
+        'field_course_image_promoted': {
+            'fid': '32379',
+            'name': 'cs50_home_tombstone.jpg',
+            'mime': 'image/jpeg',
+            'size': '19895',
+            'url': 'https://www.edx.org/sites/default/files/course/image/promoted/cs50_home_tombstone.jpg',
+            'timestamp': '1384348699',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': 'c531e644-4ca6-40ab-bddb-d41da56662a8'
+        },
+        'field_course_image_banner': {
+            'fid': '32283',
+            'name': 'cs50x-course-detail-banner.jpg',
+            'mime': 'image/jpeg',
+            'size': '17873',
+            'url': 'https://www.edx.org/sites/default/files/course/image/banner/cs50x-course-detail-banner.jpg',
+            'timestamp': '1384348498',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '3edd5c03-853c-455c-bdcd-e4d1859ce102'
+        },
+        'field_course_image_tile': {
+            'fid': '32473',
+            'name': 'cs50x-course-listing-banner.jpg',
+            'mime': 'image/jpeg',
+            'size': '34535',
+            'url': 'https://www.edx.org/sites/default/files/course/image/tile/cs50x-course-listing-banner.jpg',
+            'timestamp': '1384348906',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': 'c2998b1a-6c82-4d89-a85d-3786cdceaa6f'
+        },
+        'field_course_image_video': {
+            'fid': '32569',
+            'name': 'cs50x-video-thumbnail.jpg',
+            'mime': 'image/jpeg',
+            'size': '23035',
+            'url': 'https://www.edx.org/sites/default/files/course/image/video/cs50x-video-thumbnail.jpg',
+            'timestamp': '1384349121',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '14e9c85d-8836-4237-a497-0059d7379bce'
+        },
+        'field_course_id': 'HarvardX/CS50x/2012',
+        'field_course_image_sample_cert': [],
+        'field_course_image_sample_thumb': [],
+        'field_course_enrollment_audit': True,
+        'field_course_enrollment_honor': False,
+        'field_course_enrollment_verified': False,
+        'field_course_xseries_enable': False,
+        'field_course_statement_image': [],
+        'field_course_image_card': [],
+        'field_course_image_featured_card': [],
+        'field_course_code_override': None,
+        'field_course_video_link_mp4': [],
+        'field_course_video_duration': None,
+        'field_course_self_paced': False,
+        'field_course_new': None,
+        'field_course_registration_dates': {
+            'value': '1384348442',
+            'value2': None,
+            'duration': None
+        },
+        'field_course_enrollment_prof_ed': None,
+        'field_course_enrollment_ap_found': None,
+        'field_cource_price': None,
+        'field_course_additional_keywords': 'Free,',
+        'field_course_enrollment_mobile': None,
+        'field_course_part_of_products': [],
+        'field_course_level': None,
+        'field_course_what_u_will_learn': [],
+        'field_course_video_locale_lang': [],
+        'field_course_languages': [],
+        'field_couse_is_hidden': None,
+        'field_xseries_display_override': [],
+        'field_course_extra_description': [],
+        'field_course_extra_desc_title': None,
+        'field_course_body': [],
+        'field_course_enrollment_no_id': None,
+        'field_course_has_prerequisites': True,
+        'field_course_enrollment_credit': None,
+        'field_course_is_disabled': None,
+        'field_course_tags': [],
+        'field_course_sub_title_short': 'An introduction to the intellectual enterprises of computer science and the '
+                                        'art of programming.',
+        'field_course_length_weeks': None,
+        'field_course_start_date_style': None,
+        'field_course_head_prom_bkg_color': None,
+        'field_course_head_promo_image': [],
+        'field_course_head_promo_text': [],
+        'field_course_outcome': None,
+        'field_course_required_weeks': None,
+        'field_course_required_days': None,
+        'field_course_required_hours': None,
+        'nid': '254',
+        'vid': '8078',
+        'is_new': False,
+        'type': 'course',
+        'title': 'HarvardX: CS50x: Introduction to Computer Science',
+        'language': 'und',
+        'url': 'https://www.edx.org/course/introduction-computer-science-harvardx-cs50x-1',
+        'edit_url': 'https://www.edx.org/node/254/edit',
+        'status': '0',
+        'promote': '0',
+        'sticky': '0',
+        'created': '1384348442',
+        'changed': '1443028629',
+        'author': {
+            'uri': 'https://www.edx.org/user/143',
+            'id': '143',
+            'resource': 'user',
+            'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
+        },
+        'log': 'Updated by FeedsNodeProcessor',
+        'revision': None,
+        'body': [],
+        'uuid': '98da7bb8-dd9f-4747-aeb8-a068a863b9f8',
+        'vuuid': 'd3363b80-b402-4d66-8637-f6540e23ad0d'
+    },
+    {
+        'field_course_code': 'PH207x',
+        'field_course_course_title': {
+            'value': 'Health in Numbers: Quantitative Methods in Clinical \u0026amp; Public Health Research',
+            'format': 'basic_html'
+        },
+        'field_course_description': {
+            'value': '<h4>*Note - This is an Archived course*</h4>\n\n<p>This is a past/archived course. At this time, '
+                     'you can only explore this course in a self-paced fashion. Certain features of this course may '
+                     'not be active, but many people enjoy watching the videos and working with the materials. Make '
+                     'sure to check for reruns of this course.</p>\n\n<hr /><p>Quantitative Methods in Clinical and '
+                     'Public Health Research is the online adaptation of material from the Harvard School of Public '
+                     'Health\u0027s classes in epidemiology and biostatistics. Principled investigations to monitor '
+                     'and thus improve the health of individuals are firmly based on a sound understanding of modern '
+                     'quantitative methods.',
+            'format': 'standard_html'
+        },
+        'field_course_start_date': '1350273600',
+        'field_course_effort': '10 hours/week',
+        'field_course_school_node': [
+            {
+                'uri': 'https://www.edx.org/node/242',
+                'id': '242',
+                'resource': 'node',
+                'uuid': '44022f13-20df-4666-9111-cede3e5dc5b6'
+            }
+        ],
+        'field_course_end_date': '1358053200',
+        'field_course_video': {
+            'fid': '32572',
+            'name': 'PH207x Intro Video - Fall 2012',
+            'mime': 'video/youtube',
+            'size': '0',
+            'url': 'http://www.youtube.com/watch?v=j9CqWffkVNw',
+            'timestamp': '1384349121',
+            'owner': {
+                'uri': 'https://www.edx.org/user/143',
+                'id': '143',
+                'resource': 'user',
+                'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
+            },
+            'uuid': '2869f990-324e-41f5-8787-343e72d6134d'
+        },
+        'field_course_resources': [],
+        'field_course_sub_title_long': {
+            'value': '<p>PH207x is the online adaptation of material from the Harvard School of Public Health'
+                     '\u0026#039;s classes in epidemiology and biostatistics.</p>\n',
+            'format': 'plain_text'
+        },
+        'field_course_subject': [
+            {
+                'uri': 'https://www.edx.org/node/651',
+                'id': '651',
+                'resource': 'node',
+                'uuid': '51a13a1c-7fc8-42a6-9e96-6636d10056e2'
+            },
+            {
+                'uri': 'https://www.edx.org/node/376',
+                'id': '376',
+                'resource': 'node',
+                'uuid': 'a669e004-cbc0-4b68-8882-234c12e1cce4'
+            },
+            {
+                'uri': 'https://www.edx.org/node/657',
+                'id': '657',
+                'resource': 'node',
+                'uuid': 'a5db73b2-05b4-4284-beef-c7876ec1499b'
+            },
+            {
+                'uri': 'https://www.edx.org/node/658',
+                'id': '658',
+                'resource': 'node',
+                'uuid': 'a168a80a-4b6c-4d92-9f1d-4c235206feaf'
+            }
+        ],
+        'field_course_statement_title': None,
+        'field_course_statement_body': [],
+        'field_course_status': 'past',
+        'field_course_start_override': None,
+        'field_course_email': None,
+        'field_course_syllabus': [],
+        'field_course_prerequisites': {
+            'value': '<p>Students should have a sound grasp of algebra.</p>',
+            'format': 'standard_html'
+        },
+        'field_course_staff': [
+            {
+                'uri': 'https://www.edx.org/node/355',
+                'id': '355',
+                'resource': 'node',
+                'uuid': 'f4fe549c-6290-44ad-9be2-4b48692bd233'
+            },
+            {
+                'uri': 'https://www.edx.org/node/356',
+                'id': '356',
+                'resource': 'node',
+                'uuid': 'fa26fc74-28ce-4b21-97b6-0799e947ce3a'
+            }
+        ],
+        'field_course_staff_override': 'E. F. Cook, M. Pagano',
+        'field_course_image_promoted': {
+            'fid': '32380',
+            'name': 'ph207x-home-page-promotion.jpg',
+            'mime': 'image/jpeg',
+            'size': '99225',
+            'url': 'https://www.edx.org/sites/default/files/course/image/promoted/ph207x-home-page-promotion.jpg',
+            'timestamp': '1384348699',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '24da5041-ada5-4bb6-b0b0-099c8f3b4dc5'
+        },
+        'field_course_image_banner': {
+            'fid': '32284',
+            'name': 'ph207x-detail-banner.jpg',
+            'mime': 'image/jpeg',
+            'size': '21145',
+            'url': 'https://www.edx.org/sites/default/files/course/image/banner/ph207x-detail-banner.jpg',
+            'timestamp': '1384348498',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '4f1f88eb-9f24-44f2-8f40-f5893c41566f'
+        },
+        'field_course_image_tile': {
+            'fid': '32474',
+            'name': 'ph207x-listing-banner.jpg',
+            'mime': 'image/jpeg',
+            'size': '30833',
+            'url': 'https://www.edx.org/sites/default/files/course/image/tile/ph207x-listing-banner.jpg',
+            'timestamp': '1384348906',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': 'eeed52c1-79c8-422a-acd1-11ba9d985bc3'
+        },
+        'field_course_image_video': {
+            'fid': '32571',
+            'name': 'ph207x-video-thumbnail.jpg',
+            'mime': 'image/jpeg',
+            'size': '15015',
+            'url': 'https://www.edx.org/sites/default/files/course/image/video/ph207x-video-thumbnail.jpg',
+            'timestamp': '1384349121',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '2fbd2e9b-4f19-4c1a-aa03-e25d26bf53c1'
+        },
+        'field_course_id': 'HarvardX/PH207x/2012_Fall',
+        'field_course_image_sample_cert': [],
+        'field_course_image_sample_thumb': [],
+        'field_course_enrollment_audit': False,
+        'field_course_enrollment_honor': True,
+        'field_course_enrollment_verified': False,
+        'field_course_xseries_enable': False,
+        'field_course_statement_image': [],
+        'field_course_image_card': [],
+        'field_course_image_featured_card': {
+            'fid': '54386',
+            'name': 'ph207x_378x225.jpg',
+            'mime': 'image/jpeg',
+            'size': '12250',
+            'url': 'https://www.edx.org/sites/default/files/course/image/featured-card/ph207x_378x225.jpg',
+            'timestamp': '1427916395',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1781',
+                'id': '1781',
+                'resource': 'user',
+                'uuid': '22d74975-3826-4549-99e0-91cf86801c54'
+            },
+            'uuid': 'e7a1b891-d680-41cb-aa0b-7e9eb4f52b3a'
+        },
+        'field_course_code_override': None,
+        'field_course_video_link_mp4': [],
+        'field_course_video_duration': None,
+        'field_course_self_paced': False,
+        'field_course_new': False,
+        'field_course_registration_dates': {
+            'value': '1384318800',
+            'value2': '1384318800',
+            'duration': 0
+        },
+        'field_course_enrollment_prof_ed': False,
+        'field_course_enrollment_ap_found': False,
+        'field_cource_price': None,
+        'field_course_additional_keywords': 'Free,',
+        'field_course_enrollment_mobile': False,
+        'field_course_part_of_products': [],
+        'field_course_level': 'Intermediate',
+        'field_course_video_locale_lang': [
+            {
+                'tid': '281',
+                'name': 'English',
+                'description': '',
+                'weight': '0',
+                'node_count': 10,
+                'url': 'https://www.edx.org/video-languages/english',
+                'vocabulary': {
+                    'uri': 'https://www.edx.org/taxonomy_vocabulary/21',
+                    'id': '21',
+                    'resource': 'taxonomy_vocabulary'
+                },
+                'parent': [],
+                'parents_all': [
+                    {
+                        'tid': '281',
+                        'name': 'English',
+                        'description': '',
+                        'weight': '0',
+                        'node_count': 10,
+                        'url': 'https://www.edx.org/video-languages/english',
+                        'vocabulary': {
+                            'uri': 'https://www.edx.org/taxonomy_vocabulary/21',
+                            'id': '21',
+                            'resource': 'taxonomy_vocabulary'
+                        },
+                        'parent': [],
+                        'parents_all': [
+                            {
+                                'uri': 'https://www.edx.org/taxonomy_term/281',
+                                'id': '281',
+                                'resource': 'taxonomy_term',
+                                'uuid': 'b8155d9c-126f-4661-9518-c4d798b0a21f'
+                            }
+                        ],
+                        'uuid': 'b8155d9c-126f-4661-9518-c4d798b0a21f'
+                    }
+                ],
+                'uuid': 'b8155d9c-126f-4661-9518-c4d798b0a21f'
+            }
+        ],
+        'field_course_languages': [
+            {
+                'field_language_tag': 'en',
+                'tid': '321',
+                'name': 'English',
+                'description': '',
+                'weight': '0',
+                'node_count': 10,
+                'url': 'https://www.edx.org/course-languages/english',
+                'vocabulary': {
+                    'uri': 'https://www.edx.org/taxonomy_vocabulary/26',
+                    'id': '26',
+                    'resource': 'taxonomy_vocabulary'
+                },
+                'parent': [],
+                'parents_all': [
+                    {
+                        'field_language_tag': 'en',
+                        'tid': '321',
+                        'name': 'English',
+                        'description': '',
+                        'weight': '0',
+                        'node_count': 10,
+                        'url': 'https://www.edx.org/course-languages/english',
+                        'vocabulary': {
+                            'uri': 'https://www.edx.org/taxonomy_vocabulary/26',
+                            'id': '26',
+                            'resource': 'taxonomy_vocabulary'
+                        },
+                        'parent': [],
+                        'parents_all': [
+                            {
+                                'uri': 'https://www.edx.org/taxonomy_term/321',
+                                'id': '321',
+                                'resource': 'taxonomy_term',
+                                'uuid': '55a95f47-6ebd-475b-853a-3aff18024c1c'
+                            }
+                        ],
+                        'uuid': '55a95f47-6ebd-475b-853a-3aff18024c1c'
+                    }
+                ],
+                'uuid': '55a95f47-6ebd-475b-853a-3aff18024c1c'
+            }
+        ],
+        'field_couse_is_hidden': False,
+        'field_xseries_display_override': [],
+        'field_course_extra_description': [],
+        'field_course_extra_desc_title': None,
+        'field_course_body': {
+            'value': '<p>Quantitative Methods in Clinical and Public Health Research is the online adaptation of '
+                     'material from the Harvard T.H. Chan School of Public Health\u0027s classes in epidemiology and '
+                     'biostatistics. Principled investigations to monitor and thus improve the health of individuals '
+                     'are firmly based on a sound understanding of modern quantitative methods. This involves the '
+                     'ability to discover patterns and extract knowledge from health data on a sample of individuals '
+                     'and then to infer, with measured uncertainty, the unobserved population characteristics. This '
+                     'course will address this need by covering the principles of biostatistics and epidemiology used '
+                     'for public health and clinical research. These include outcomes measurement, measures of '
+                     'associations between outcomes and their determinants, study design options, bias and '
+                     'confounding, probability and diagnostic tests, confidence intervals and hypothesis testing, '
+                     'power and sample size determinations, life tables and survival methods, regression methods '
+                     '(both, linear and logistic), and sample survey techniques. Students will analyze sample data '
+                     'sets to acquire knowledge of appropriate computer software. By the end of the course the '
+                     'successful student should have attained a sound understanding of these methods and a solid '
+                     'foundation for further study.<br />\n\u00a0</p>',
+            'summary': '',
+            'format': 'standard_html'
+        },
+        'field_course_enrollment_no_id': False,
+        'field_course_has_prerequisites': True,
+        'field_course_enrollment_credit': False,
+        'field_course_is_disabled': None,
+        'field_course_tags': [],
+        'field_course_sub_title_short': 'PH207x is the online adaptation of material from the Harvard School of Public '
+                                        'Health\u0027s classes in epidemiology and biostatistics.',
+        'field_course_length_weeks': '13 weeks',
+        'field_course_start_date_style': None,
+        'field_course_head_prom_bkg_color': None,
+        'field_course_head_promo_image': [],
+        'field_course_head_promo_text': [],
+        'field_course_outcome': None,
+        'field_course_required_weeks': '4',
+        'field_course_required_days': '0',
+        'field_course_required_hours': '0',
+        'nid': '354',
+        'vid': '112156',
+        'is_new': False,
+        'type': 'course',
+        'title': 'HarvardX: PH207x: Health in Numbers: Quantitative Methods in Clinical \u0026 Public Health Research',
+        'language': 'und',
+        'url': 'https://www.edx.org/course/health-numbers-quantitative-methods-harvardx-ph207x',
+        'edit_url': 'https://www.edx.org/node/354/edit',
+        'status': '1',
+        'promote': '0',
+        'sticky': '0',
+        'created': '1384348442',
+        'changed': '1464108885',
+        'author': {
+            'uri': 'https://www.edx.org/user/143',
+            'id': '143',
+            'resource': 'user',
+            'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
+        },
+        'log': '',
+        'revision': None,
+        'body': [],
+        'uuid': 'aebbadcc-4e3a-4be3-a351-edaabd025ce7',
+        'vuuid': '28da5064-b570-4883-8c53-330d1893ab49'
+    },
+    {
+        'field_course_code': 'CB22x',
+        'field_course_course_title': {
+            'value': 'The Ancient Greek Hero',
+            'format': 'basic_html'
+        },
+        'field_course_description': {
+            'value': '<p><strong>NOTE ABOUT OUR START DATE:</strong> Although the course was launched on March 13th, '
+                     'it\u0027s not too late to start participating! New participants will be joining the course until '
+                     '<strong>registration closes on July 11</strong>. We offer everyone a flexible schedule and '
+                     'multiple paths for participation. You can work through the course videos and readings at your '
+                     'own pace to complete the associated exercises <strong>by August 26</strong>, the official course '
+                     'end date. Or, you may choose to \u0022audit\u0022 the course by exploring just the particular '
+                     'videos and readings that seem most suited to your interests. You are free to do as much or as '
+                     'little as you would like!</p>\n<h3>\n\tOverview</h3>\n<p>What is it to be human, and how can '
+                     'ancient concepts of the heroic and anti-heroic inform our understanding of the human condition? '
+                     'That question is at the core of The Ancient Greek Hero, which introduces (or reintroduces) '
+                     'students to the great texts of classical Greek culture by focusing on concepts of the Hero in an '
+                     'engaging, highly comparative way.</p>\n<p>The classical Greeks\u0027 concepts of Heroes and the '
+                     '\u0022heroic\u0022 were very different from the way we understand the term today. In this '
+                     'course, students analyze Greek heroes and anti-heroes in their own historical contexts, in order '
+                     'to gain an understanding of these concepts as they were originally understood while also '
+                     'learning how they can inform our understanding of the human condition in general.</p>\n<p>In '
+                     'Greek tradition, a hero was a human, male or female, of the remote past, who was endowed with '
+                     'superhuman abilities by virtue of being descended from an immortal god. Rather than being '
+                     'paragons of virtue, as heroes are viewed in many modern cultures, ancient Greek heroes had all '
+                     'of the qualities and faults of their fellow humans, but on a much larger scale. Further, despite '
+                     'their mortality, heroes, like the gods, were objects of cult worship \u2013 a dimension which is '
+                     'also explored in depth in the course.</p>\n<p>The original sources studied in this course include'
+                     ' the Homeric Iliad and Odyssey; tragedies of Aeschylus, Sophocles, and Euripides; songs of Sappho'
+                     ' and Pindar; dialogues of Plato; historical texts of Herodotus; and more, including the '
+                     'intriguing but rarely studied dialogue \u0022On Heroes\u0022 by Philostratus. All works are '
+                     'presented in English translation, with attention to the subtleties of the original Greek. These '
+                     'original sources are frequently supplemented both by ancient art and by modern comparanda, '
+                     'including opera and cinema (from Jacques Offenbach\u0027s opera Tales of Hoffman to Ridley '
+                     'Scott\u0027s science fiction classic Blade Runner).</p>',
+            'format': 'standard_html'
+        },
+        'field_course_start_date': '1363147200',
+        'field_course_effort': '4-6 hours / week',
+        'field_course_school_node': [
+            {
+                'uri': 'https://www.edx.org/node/242',
+                'id': '242',
+                'resource': 'node',
+                'uuid': '44022f13-20df-4666-9111-cede3e5dc5b6'
+            }
+        ],
+        'field_course_end_date': '1376971200',
+        'field_course_video': [],
+        'field_course_resources': [],
+        'field_course_sub_title_long': {
+            'value': '<p>A survey of ancient Greek literature focusing on classical concepts of the hero and how they '
+                     'can inform our understanding of the human condition.</p>\n',
+            'format': 'plain_text'
+        },
+        'field_course_subject': [
+            {
+                'uri': 'https://www.edx.org/node/652',
+                'id': '652',
+                'resource': 'node',
+                'uuid': 'c8579e1c-99f2-4a95-988c-3542909f055e'
+            },
+            {
+                'uri': 'https://www.edx.org/node/653',
+                'id': '653',
+                'resource': 'node',
+                'uuid': '00e5d5e0-ce45-4114-84a1-50a5be706da5'
+            },
+            {
+                'uri': 'https://www.edx.org/node/655',
+                'id': '655',
+                'resource': 'node',
+                'uuid': '74b6ed2a-3ba0-49be-adc9-53f7256a12e1'
+            }
+        ],
+        'field_course_statement_title': None,
+        'field_course_statement_body': [],
+        'field_course_status': 'past',
+        'field_course_start_override': None,
+        'field_course_email': None,
+        'field_course_syllabus': [],
+        'field_course_staff': [
+            {
+                'uri': 'https://www.edx.org/node/564',
+                'id': '564',
+                'resource': 'node',
+                'uuid': 'ae56688a-f2b6-4981-9aa7-5c66b68cb13e'
+            },
+            {
+                'uri': 'https://www.edx.org/node/565',
+                'id': '565',
+                'resource': 'node',
+                'uuid': '56d13e72-353f-48fd-9be7-6f20ef467bb7'
+            },
+            {
+                'uri': 'https://www.edx.org/node/566',
+                'id': '566',
+                'resource': 'node',
+                'uuid': '69a415db-3db7-436a-8d02-e571c4c4c75a'
+            },
+            {
+                'uri': 'https://www.edx.org/node/567',
+                'id': '567',
+                'resource': 'node',
+                'uuid': '1639460f-598c-45b7-90c2-bbdbf87cdd54'
+            },
+            {
+                'uri': 'https://www.edx.org/node/568',
+                'id': '568',
+                'resource': 'node',
+                'uuid': '09154d2c-7f31-477c-9d3c-d8cba9af846e'
+            },
+            {
+                'uri': 'https://www.edx.org/node/820',
+                'id': '820',
+                'resource': 'node',
+                'uuid': '05b7ab45-de9a-49d6-8010-04c68fc9fd55'
+            },
+            {
+                'uri': 'https://www.edx.org/node/821',
+                'id': '821',
+                'resource': 'node',
+                'uuid': '8a8d68c4-ab5b-40c5-b897-2d44aed2194d'
+            },
+            {
+                'uri': 'https://www.edx.org/node/822',
+                'id': '822',
+                'resource': 'node',
+                'uuid': 'c3e16519-a23f-4f21-908b-463375b492df'
+            }
+        ],
+        'field_course_staff_override': 'G. Nagy, L. Muellner...',
+        'field_course_image_promoted': {
+            'fid': '32381',
+            'name': 'tombstone_courses.jpg',
+            'mime': 'image/jpeg',
+            'size': '34861',
+            'url': 'https://www.edx.org/sites/default/files/course/image/promoted/tombstone_courses.jpg',
+            'timestamp': '1384348699',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '1471888c-a451-4f97-9bb2-ad20c9a43c2d'
+        },
+        'field_course_image_banner': {
+            'fid': '32285',
+            'name': 'cb22x_608x211.jpg',
+            'mime': 'image/jpeg',
+            'size': '25909',
+            'url': 'https://www.edx.org/sites/default/files/course/image/banner/cb22x_608x211.jpg',
+            'timestamp': '1384348498',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '15022bf7-e367-4a5c-b115-3755016de286'
+        },
+        'field_course_image_tile': {
+            'fid': '32475',
+            'name': 'cb22x-listing-banner.jpg',
+            'mime': 'image/jpeg',
+            'size': '47678',
+            'url': 'https://www.edx.org/sites/default/files/course/image/tile/cb22x-listing-banner.jpg',
+            'timestamp': '1384348906',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '71735cc4-7ac3-4065-ad92-6f18f979eb0e'
+        },
+        'field_course_image_video': {
+            'fid': '32573',
+            'name': 'h_no_video_320x211_1_0.jpg',
+            'mime': 'image/jpeg',
+            'size': '2829',
+            'url': 'https://www.edx.org/sites/default/files/course/image/video/h_no_video_320x211_1_0.jpg',
+            'timestamp': '1384349121',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': '4d18789f-0909-4289-9d58-2292e5d03aee'
+        },
+        'field_course_id': 'HarvardX/CB22x/2013_Spring',
+        'field_course_image_sample_cert': [],
+        'field_course_image_sample_thumb': [],
+        'field_course_enrollment_audit': True,
+        'field_course_enrollment_honor': False,
+        'field_course_enrollment_verified': False,
+        'field_course_xseries_enable': False,
+        'field_course_statement_image': [],
+        'field_course_image_card': [],
+        'field_course_image_featured_card': [],
+        'field_course_code_override': None,
+        'field_course_video_link_mp4': [],
+        'field_course_video_duration': None,
+        'field_course_self_paced': False,
+        'field_course_new': None,
+        'field_course_registration_dates': {
+            'value': '1384348442',
+            'value2': None,
+            'duration': None
+        },
+        'field_course_enrollment_prof_ed': None,
+        'field_course_enrollment_ap_found': None,
+        'field_cource_price': None,
+        'field_course_additional_keywords': 'Free,',
+        'field_course_enrollment_mobile': None,
+        'field_course_part_of_products': [],
+        'field_course_level': None,
+        'field_course_what_u_will_learn': [],
+        'field_course_video_locale_lang': [],
+        'field_course_languages': [],
+        'field_couse_is_hidden': None,
+        'field_xseries_display_override': [],
+        'field_course_extra_description': [],
+        'field_course_extra_desc_title': None,
+        'field_course_body': [],
+        'field_course_enrollment_no_id': None,
+        'field_course_has_prerequisites': True,
+        'field_course_enrollment_credit': None,
+        'field_course_is_disabled': None,
+        'field_course_tags': [],
+        'field_course_sub_title_short': 'A survey of ancient Greek literature focusing on classical concepts of the '
+                                        'hero and how they can inform our understanding of the human condition.',
+        'field_course_length_weeks': '23 weeks',
+        'field_course_start_date_style': None,
+        'field_course_head_prom_bkg_color': None,
+        'field_course_head_promo_image': [],
+        'field_course_head_promo_text': [],
+        'field_course_outcome': None,
+        'field_course_required_weeks': None,
+        'field_course_required_days': None,
+        'field_course_required_hours': None,
+        'nid': '563',
+        'vid': '8080',
+        'is_new': False,
+        'type': 'course',
+        'title': 'HarvardX: CB22x: The Ancient Greek Hero',
+        'language': 'und',
+        'url': 'https://www.edx.org/course/ancient-greek-hero-harvardx-cb22x',
+        'edit_url': 'https://www.edx.org/node/563/edit',
+        'status': '0',
+        'promote': '0',
+        'sticky': '0',
+        'created': '1384348442',
+        'changed': '1443028625',
+        'author': {
+            'uri': 'https://www.edx.org/user/143',
+            'id': '143',
+            'resource': 'user',
+            'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
+        },
+        'log': 'Updated by FeedsNodeProcessor',
+        'revision': None,
+        'body': [],
+        'uuid': '6b8b779f-f567-4e98-aa41-a265d6fa073c',
+        'vuuid': 'e0f8c80a-b377-4546-b247-1c94ab3a218b'
+    }
+]

course_discovery/apps/course_metadata/data_loaders/tests/test_api.py

@@ -125,7 +125,7 @@ class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCas
         )
         return bodies
 
-    def assert_course_run_loaded(self, body, use_marketing_url=True):
+    def assert_course_run_loaded(self, body, partner_has_marketing_site=True):
         """ Assert a CourseRun corresponding to the specified data body was properly loaded into the database. """
 
         # Validate the Course
@@ -134,31 +134,43 @@ class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCas
         course = Course.objects.get(key=course_key)
 
         self.assertEqual(course.title, body['name'])
-        self.assertListEqual(list(course.organizations.all()), [organization])
+        self.assertListEqual(list(course.authoring_organizations.all()), [organization])
 
         # Validate the course run
-        course_run = CourseRun.objects.get(key=body['id'])
-        self.assertEqual(course_run.course, course)
-        self.assertEqual(course_run.title, AbstractDataLoader.clean_string(body['name']))
-        self.assertEqual(course_run.short_description, AbstractDataLoader.clean_string(body['short_description']))
-        self.assertEqual(course_run.start, AbstractDataLoader.parse_date(body['start']))
-        self.assertEqual(course_run.end, AbstractDataLoader.parse_date(body['end']))
-        self.assertEqual(course_run.enrollment_start, AbstractDataLoader.parse_date(body['enrollment_start']))
-        self.assertEqual(course_run.enrollment_end, AbstractDataLoader.parse_date(body['enrollment_end']))
-        self.assertEqual(course_run.pacing_type, self.loader.get_pacing_type(body))
-        self.assertEqual(course_run.video, self.loader.get_courserun_video(body))
-        if use_marketing_url:
-            self.assertEqual(course_run.image, None)
-        else:
-            self.assertEqual(course_run.image, self.loader.get_courserun_image(body))
+        course_run = course.course_runs.get(key=body['id'])
+        expected_values = {
+            'title': self.loader.clean_string(body['name']),
+            'short_description': self.loader.clean_string(body['short_description']),
+            'start': self.loader.parse_date(body['start']),
+            'end': self.loader.parse_date(body['end']),
+            'enrollment_start': self.loader.parse_date(body['enrollment_start']),
+            'enrollment_end': self.loader.parse_date(body['enrollment_end']),
+            'pacing_type': self.loader.get_pacing_type(body),
+            'card_image_url': None,
+            'title_override': None,
+            'short_description_override': None,
+            'video': None,
+        }
+
+        if not partner_has_marketing_site:
+            expected_values.update({
+                'card_image_url': body['media'].get('image', {}).get('raw'),
+                'title_override': body['name'],
+                'short_description_override': self.loader.clean_string(body['short_description']),
+                'video': self.loader.get_courserun_video(body),
+            })
+
+        for field, value in expected_values.items():
+            self.assertEqual(getattr(course_run, field), value, 'Field {} is invalid.'.format(field))
 
     @responses.activate
     @ddt.data(True, False)
-    def test_ingest(self, use_marketing_url):
+    def test_ingest(self, partner_has_marketing_site):
         """ Verify the method ingests data from the Courses API. """
         api_data = self.mock_api()
-        if not use_marketing_url:
+        if not partner_has_marketing_site:
             self.partner.marketing_site_url_root = None
+            self.partner.save()  # pylint: disable=no-member
 
         self.assertEqual(Course.objects.count(), 0)
         self.assertEqual(CourseRun.objects.count(), 0)
@@ -173,7 +185,7 @@ class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCas
         self.assertEqual(CourseRun.objects.count(), expected_num_course_runs)
 
         for datum in api_data:
-            self.assert_course_run_loaded(datum, use_marketing_url)
+            self.assert_course_run_loaded(datum, partner_has_marketing_site)
 
         # Verify multiple calls to ingest data do NOT result in data integrity errors.
         self.loader.ingest()

course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py

@@ -9,14 +9,12 @@ from django.test import TestCase
 from opaque_keys.edx.keys import CourseKey
 
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
-    DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
-    SponsorMarketingSiteDataLoader, PersonMarketingSiteDataLoader,
+    XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
+    SponsorMarketingSiteDataLoader, PersonMarketingSiteDataLoader, CourseMarketingSiteDataLoader
 )
 from course_discovery.apps.course_metadata.data_loaders.tests import JSON, mock_data
-from course_discovery.apps.course_metadata.data_loaders.tests.mixins import ApiClientTestMixin, DataLoaderTestMixin
-from course_discovery.apps.course_metadata.models import (
-    Course, CourseOrganization, CourseRun, Organization, Subject, Program, Video, Person,
-)
+from course_discovery.apps.course_metadata.data_loaders.tests.mixins import DataLoaderTestMixin
+from course_discovery.apps.course_metadata.models import Organization, Subject, Program, Video, Person, Course
 from course_discovery.apps.course_metadata.tests import factories
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
@@ -24,192 +22,6 @@ ENGLISH_LANGUAGE_TAG = LanguageTag(code='en-us', name='English - United States')
 LOGGER_PATH = 'course_discovery.apps.course_metadata.data_loaders.marketing_site.logger'
 
 
-@ddt.ddt
-class DrupalApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase):
-    loader_class = DrupalApiDataLoader
-
-    @property
-    def api_url(self):
-        return self.partner.marketing_site_api_url
-
-    def setUp(self):
-        super(DrupalApiDataLoaderTests, self).setUp()
-        for course_dict in mock_data.EXISTING_COURSE_AND_RUN_DATA:
-            course = Course.objects.create(key=course_dict['course_key'], title=course_dict['title'])
-            CourseRun.objects.create(
-                key=course_dict['course_run_key'],
-                language=self.loader.get_language_tag(course_dict),
-                course=course
-            )
-
-            # Add some data that doesn't exist in Drupal already
-            organization = Organization.objects.create(key='orphan_org_' + course.key)
-            CourseOrganization.objects.create(
-                organization=organization,
-                course=course,
-                relation_type=CourseOrganization.SPONSOR
-            )
-
-        Course.objects.create(key=mock_data.EXISTING_COURSE['course_key'], title=mock_data.EXISTING_COURSE['title'])
-        Organization.objects.create(key=mock_data.ORPHAN_ORGANIZATION_KEY)
-
-    def create_mock_subjects(self, course_runs):
-        course_runs = course_runs['items']
-
-        for course_run in course_runs:
-            if course_run:
-                for subject in course_run['subjects']:
-                    Subject.objects.get_or_create(name=subject['title'], partner=self.partner)
-
-    def mock_api(self):
-        """Mock out the Drupal API. Returns a list of mocked-out course runs."""
-        body = mock_data.MARKETING_API_BODY
-        self.create_mock_subjects(body)
-
-        responses.add(
-            responses.GET,
-            self.api_url + 'courses/',
-            body=json.dumps(body),
-            status=200,
-            content_type='application/json'
-        )
-        return body['items']
-
-    def assert_course_run_loaded(self, body):
-        """
-        Verify that the course run corresponding to `body` has been saved
-        correctly.
-        """
-        course_run_key_str = body['course_id']
-        course_run_key = CourseKey.from_string(course_run_key_str)
-        course_key = '{org}+{course}'.format(org=course_run_key.org, course=course_run_key.course)
-        course = Course.objects.get(key=course_key)
-        course_run = CourseRun.objects.get(key=course_run_key_str)
-
-        self.assertEqual(course_run.course, course)
-
-        self.assert_course_loaded(course, body)
-
-        if course_run.language:
-            self.assertEqual(course_run.language.code, body['current_language'])
-        else:
-            self.assertEqual(body['current_language'], '')
-
-    def assert_course_loaded(self, course, body):
-        """Verify that the course has been loaded correctly."""
-        self.assertEqual(course.title, body['title'])
-        self.assertEqual(course.full_description, self.loader.clean_html(body['description']))
-        self.assertEqual(course.short_description, self.loader.clean_html(body['subtitle']))
-        self.assertEqual(course.level_type.name, body['level']['title'])
-
-        self.assert_subjects_loaded(course, body)
-        self.assert_sponsors_loaded(course, body)
-
-    def assert_subjects_loaded(self, course, body):
-        """Verify that subjects have been loaded correctly."""
-        course_subjects = course.subjects.all()
-        expected_subjects = body['subjects']
-        expected_subjects = [subject['title'] for subject in expected_subjects]
-        actual_subjects = list(course_subjects.values_list('name', flat=True))
-        self.assertEqual(actual_subjects, expected_subjects)
-
-    def assert_sponsors_loaded(self, course, body):
-        """Verify that sponsors have been loaded correctly."""
-        course_sponsors = course.sponsors.all()
-        api_sponsors = body['sponsors']
-        self.assertEqual(len(course_sponsors), len(api_sponsors))
-        for api_sponsor in api_sponsors:
-            loaded_sponsor = Organization.objects.get(key=api_sponsor['uuid'])
-            self.assertIn(loaded_sponsor, course_sponsors)
-
-    @responses.activate
-    def test_ingest(self):
-        """Verify the data loader ingests data from Drupal."""
-        api_data = self.mock_api()
-        # Neither the faked course, nor the empty array, should not be loaded from Drupal.
-        # Change this back to -2 as part of ECOM-4493.
-        loaded_data = api_data[:-3]
-
-        self.loader.ingest()
-
-        # Drupal does not paginate its response or check authorization
-        self.assert_api_called(1, check_auth=False)
-
-        # Assert that the fake course was not created
-        self.assertEqual(CourseRun.objects.count(), len(loaded_data))
-
-        for datum in loaded_data:
-            self.assert_course_run_loaded(datum)
-
-        Course.objects.get(key=mock_data.EXISTING_COURSE['course_key'], title=mock_data.EXISTING_COURSE['title'])
-
-        # Verify multiple calls to ingest data do NOT result in data integrity errors.
-        self.loader.ingest()
-
-        # Verify that orphan data is deleted
-        self.assertFalse(Organization.objects.filter(key=mock_data.ORPHAN_ORGANIZATION_KEY).exists())
-        self.assertFalse(Organization.objects.filter(key__startswith='orphan_org_').exists())
-
-    @responses.activate
-    def test_ingest_exception_handling(self):
-        """ Verify the data loader properly handles exceptions during processing of the data from the API. """
-        api_data = self.mock_api()
-        # Include all data, except the empty array.
-        # TODO: Remove the -1 after ECOM-4493 is in production.
-        expected_call_count = len(api_data) - 1
-
-        with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
-            with mock.patch(LOGGER_PATH) as mock_logger:
-                self.loader.ingest()
-                self.assertEqual(mock_logger.exception.call_count, expected_call_count)
-
-                # TODO: Change the -2 to -1 after ECOM-4493 is in production.
-                msg = 'An error occurred while updating {0} from {1}'.format(
-                    api_data[-2]['course_id'],
-                    self.partner.marketing_site_api_url
-                )
-                mock_logger.exception.assert_called_with(msg)
-
-    @ddt.unpack
-    @ddt.data(
-        ({'image': {}}, None),
-        ({'image': 'http://example.com/image.jpg'}, 'http://example.com/image.jpg'),
-    )
-    def test_get_courserun_image(self, media_body, expected_image_url):
-        """ Verify the method returns an Image object with the correct URL. """
-        actual = self.loader.get_courserun_image(media_body)
-
-        if expected_image_url:
-            self.assertEqual(actual.src, expected_image_url)
-        else:
-            self.assertIsNone(actual)
-
-    @ddt.data(
-        ('', ''),
-        ('<h1>foo</h1>', '# foo'),
-        ('<a href="http://example.com">link</a>', '[link](http://example.com)'),
-        ('<strong>foo</strong>', '**foo**'),
-        ('<em>foo</em>', '_foo_'),
-        ('\nfoo\n', 'foo'),
-        ('<span>foo</span>', 'foo'),
-        ('<div>foo</div>', 'foo'),
-    )
-    @ddt.unpack
-    def test_clean_html(self, to_clean, expected):
-        self.assertEqual(self.loader.clean_html(to_clean), expected)
-
-    @ddt.data(
-        ({'current_language': ''}, None),
-        ({'current_language': 'not-real'}, None),
-        ({'current_language': 'en-us'}, ENGLISH_LANGUAGE_TAG),
-        ({'current_language': 'en'}, ENGLISH_LANGUAGE_TAG),
-        ({'current_language': None}, None),
-    )
-    @ddt.unpack
-    def test_get_language_tag(self, body, expected):
-        self.assertEqual(self.loader.get_language_tag(body), expected)
-
-
 class AbstractMarketingSiteDataLoaderTestMixin(DataLoaderTestMixin):
     mocked_data = []
 
@@ -501,3 +313,133 @@ class PersonMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixi
 
         for person in people:
             self.assert_person_loaded(person)
+
+
+@ddt.ddt
+class CourseMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
+    loader_class = CourseMarketingSiteDataLoader
+    mocked_data = mock_data.MARKETING_SITE_API_COURSE_BODIES
+
+    def _get_uuids(self, items):
+        return [item['uuid'] for item in items]
+
+    def mock_api(self):
+        bodies = super().mock_api()
+
+        data_map = {
+            factories.SubjectFactory: 'field_course_subject',
+            factories.OrganizationFactory: 'field_course_school_node',
+            factories.PersonFactory: 'field_course_staff',
+        }
+
+        for factory, field in data_map.items():
+            uuids = set()
+
+            for body in bodies:
+                uuids.update(self._get_uuids(body.get(field, [])))
+
+            for uuid in uuids:
+                factory(uuid=uuid, partner=self.partner)
+
+        return bodies
+
+    def test_get_language_tags_from_names(self):
+        names = ('English', '中文', None)
+        expected = list(LanguageTag.objects.filter(code__in=('en-us', 'zh-cmn')))
+        self.assertEqual(list(self.loader.get_language_tags_from_names(names)), expected)
+
+    def test_get_level_type(self):
+        self.assertIsNone(self.loader.get_level_type(None))
+
+        name = 'Advanced'
+        self.assertEqual(self.loader.get_level_type(name).name, name)
+
+    @ddt.data(
+        {'field_course_body': {'value': 'Test'}},
+        {'field_course_description': {'value': 'Test'}},
+        {'field_course_description': {'value': 'Test2'}, 'field_course_body': {'value': 'Test'}},
+    )
+    def test_get_description(self, data):
+        self.assertEqual(self.loader.get_description(data), 'Test')
+
+    def test_get_video(self):
+        image_url = 'https://example.com/image.jpg'
+        video_url = 'https://example.com/video.mp4'
+        data = {
+            'field_product_video': {'url': video_url},
+            'field_course_image_featured_card': {'url': image_url}
+        }
+        video = self.loader.get_video(data)
+        self.assertEqual(video.src, video_url)
+        self.assertEqual(video.image.src, image_url)
+
+        self.assertIsNone(self.loader.get_video({}))
+
+    def assert_course_loaded(self, data):
+        course = self._get_course(data)
+
+        expected_values = {
+            'title': data['field_course_course_title']['value'],
+            'number': data['field_course_code'],
+            'full_description': self.loader.get_description(data),
+            'video': self.loader.get_video(data),
+            'short_description': self.loader.clean_html(data['field_course_sub_title_short']),
+            'level_type': self.loader.get_level_type(data['field_course_level']),
+            'card_image_url': (data.get('field_course_image_promoted') or {}).get('url'),
+        }
+
+        for field, value in expected_values.items():
+            self.assertEqual(getattr(course, field), value)
+
+        # Verify the subject and authoring organization relationships
+        data_map = {
+            course.subjects: 'field_course_subject',
+            course.authoring_organizations: 'field_course_school_node',
+        }
+
+        self.validate_relationships(data, data_map)
+
+    def validate_relationships(self, data, data_map):
+        for relationship, field in data_map.items():
+            expected = sorted(self._get_uuids(data.get(field, [])))
+            actual = list(relationship.order_by('uuid').values_list('uuid', flat=True))
+            actual = [str(item) for item in actual]
+            self.assertListEqual(actual, expected, 'Data not properly pulled from {}'.format(field))
+
+    def assert_course_run_loaded(self, data):
+        course = self._get_course(data)
+        course_run = course.course_runs.get(uuid=data['uuid'])
+        language_names = [language['name'] for language in data['field_course_languages']]
+        language = self.loader.get_language_tags_from_names(language_names).first()
+
+        expected_values = {
+            'key': data['field_course_id'],
+            'language': language,
+            'slug': data['url'].split('/')[-1],
+        }
+
+        for field, value in expected_values.items():
+            self.assertEqual(getattr(course_run, field), value)
+
+        # Verify the staff relationship
+        self.validate_relationships(data, {course_run.staff: 'field_course_staff'})
+
+        language_names = [language['name'] for language in data['field_course_video_locale_lang']]
+        expected_transcript_languages = self.loader.get_language_tags_from_names(language_names)
+        self.assertEqual(list(course_run.transcript_languages.all()), list(expected_transcript_languages))
+
+    def _get_course(self, data):
+        course_run_key = CourseKey.from_string(data['field_course_id'])
+        return Course.objects.get(key=self.loader.get_course_key_from_course_run_key(course_run_key),
+                                  partner=self.partner)
+
+    @responses.activate
+    def test_ingest(self):
+        self.mock_login_response()
+        data = self.mock_api()
+
+        self.loader.ingest()
+
+        for datum in data:
+            self.assert_course_run_loaded(datum)
+            self.assert_course_loaded(datum)

course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py

@@ -5,11 +5,11 @@ from edx_rest_api_client.client import EdxRestApiClient
 
 from course_discovery.apps.core.models import Partner
 from course_discovery.apps.course_metadata.data_loaders.api import (
-    CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
+    OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader, CoursesApiDataLoader,
 )
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
-    DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
-    SponsorMarketingSiteDataLoader, PersonMarketingSiteDataLoader,
+    XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
+    SponsorMarketingSiteDataLoader, PersonMarketingSiteDataLoader, CourseMarketingSiteDataLoader,
 )
 
 logger = logging.getLogger(__name__)
@@ -83,11 +83,11 @@ class Command(BaseCommand):
                 (partner.marketing_site_url_root, SchoolMarketingSiteDataLoader,),
                 (partner.marketing_site_url_root, SponsorMarketingSiteDataLoader,),
                 (partner.marketing_site_url_root, PersonMarketingSiteDataLoader,),
+                (partner.marketing_site_api_url, CourseMarketingSiteDataLoader,),
                 (partner.organizations_api_url, OrganizationsApiDataLoader,),
                 (partner.courses_api_url, CoursesApiDataLoader,),
                 (partner.ecommerce_api_url, EcommerceApiDataLoader,),
                 (partner.programs_api_url, ProgramsApiDataLoader,),
-                (partner.marketing_site_api_url, DrupalApiDataLoader,),
                 (partner.marketing_site_url_root, XSeriesMarketingSiteDataLoader,),
             )
 
@@ -97,3 +97,5 @@ class Command(BaseCommand):
                         loader_class(partner, api_url, access_token, token_type).ingest()
                     except Exception:  # pylint: disable=broad-except
                         logger.exception('%s failed!', loader_class.__name__)
+
+            # TODO Cleanup CourseRun overrides equivalent to the Course values.

course_discovery/apps/course_metadata/management/commands/tests/test_refresh_course_metadata.py

@@ -8,13 +8,13 @@ from django.test import TestCase
 from course_discovery.apps.core.tests.factories import PartnerFactory
 from course_discovery.apps.core.tests.utils import mock_api_callback
 from course_discovery.apps.course_metadata.data_loaders.api import (
-    CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
+    OrganizationsApiDataLoader, CoursesApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
 )
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
-    DrupalApiDataLoader, XSeriesMarketingSiteDataLoader,
+    XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
+    SponsorMarketingSiteDataLoader, PersonMarketingSiteDataLoader, CourseMarketingSiteDataLoader
 )
 from course_discovery.apps.course_metadata.data_loaders.tests import mock_data
-from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Program
 
 ACCESS_TOKEN = 'secret'
 JSON = 'application/json'
@@ -74,7 +74,6 @@ class RefreshCourseMetadataCommandTests(TestCase):
         return bodies
 
     def mock_ecommerce_courses_api(self):
-
         bodies = mock_data.ECOMMERCE_API_BODIES
         url = self.partner.ecommerce_api_url + 'courses/'
         responses.add_callback(
@@ -108,49 +107,14 @@ class RefreshCourseMetadataCommandTests(TestCase):
         )
         return bodies
 
-    @responses.activate
-    def test_refresh_course_metadata(self):
-        """ Verify the refresh_course_metadata management command creates new objects. """
-        self.mock_apis()
-        call_command('refresh_course_metadata')
-
-        organizations = Organization.objects.all()
-        self.assertEqual(organizations.count(), 3)
-
-        for organization in organizations:
-            self.assertEqual(organization.partner.short_code, self.partner.short_code)
-
-        courses = Course.objects.all()
-        self.assertEqual(courses.count(), 2)
-        for course in courses:
-            self.assertEqual(course.partner.short_code, self.partner.short_code)
-
-        course_runs = CourseRun.objects.all()
-        self.assertEqual(course_runs.count(), 3)
-        for course_run in course_runs:
-            self.assertEqual(course_run.course.partner.short_code, self.partner.short_code)
-
-        programs = Program.objects.all()
-        self.assertEqual(programs.count(), 2)
-        for program in programs:
-            self.assertEqual(program.partner.short_code, self.partner.short_code)
-
-        # Refresh only a specific partner
-        command_args = ['--partner_code={0}'.format(self.partner.short_code)]
-        call_command('refresh_course_metadata', *command_args)
-
-    @responses.activate
     def test_refresh_course_metadata_with_invalid_partner_code(self):
         """ Verify an error is raised if an invalid partner code is passed on the command line. """
-        self.mock_apis()
         with self.assertRaises(CommandError):
             command_args = ['--partner_code=invalid']
             call_command('refresh_course_metadata', *command_args)
 
-    @responses.activate
     def test_refresh_course_metadata_with_no_token_type(self):
         """ Verify an error is raised if an access token is passed in without a token type. """
-        self.mock_apis()
         with self.assertRaises(CommandError):
             command_args = ['--access_token=test-access-token']
             call_command('refresh_course_metadata', *command_args)
@@ -164,7 +128,17 @@ class RefreshCourseMetadataCommandTests(TestCase):
             with mock.patch(logger_target) as mock_logger:
                 call_command('refresh_course_metadata')
 
-                loader_classes = (OrganizationsApiDataLoader, CoursesApiDataLoader, EcommerceApiDataLoader,
-                                  ProgramsApiDataLoader, DrupalApiDataLoader, XSeriesMarketingSiteDataLoader)
+                loader_classes = (
+                    SubjectMarketingSiteDataLoader,
+                    SchoolMarketingSiteDataLoader,
+                    SponsorMarketingSiteDataLoader,
+                    PersonMarketingSiteDataLoader,
+                    CourseMarketingSiteDataLoader,
+                    OrganizationsApiDataLoader,
+                    CoursesApiDataLoader,
+                    EcommerceApiDataLoader,
+                    ProgramsApiDataLoader,
+                    XSeriesMarketingSiteDataLoader,
+                )
                 expected_calls = [mock.call('%s failed!', loader_class.__name__) for loader_class in loader_classes]
                 mock_logger.exception.assert_has_calls(expected_calls)

course_discovery/apps/course_metadata/migrations/0021_auto_20160819_2005.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import uuid
+
+import django_extensions.db.fields
+import sortedm2m.fields
+from django.db import migrations, models
+
+
+def delete_partnerless_courses(apps, schema_editor):
+    Course = apps.get_model('course_metadata', 'Course')
+    Course.objects.filter(partner__isnull=True).delete()
+
+
+def add_uuid_to_courses_and_course_runs(apps, schema_editor):
+    Course = apps.get_model('course_metadata', 'Course')
+    CourseRun = apps.get_model('course_metadata', 'CourseRun')
+
+    for objects in (Course.objects.filter(uuid__isnull=True), CourseRun.objects.filter(uuid__isnull=True)):
+        for obj in objects:
+            obj.uuid = uuid.uuid4()
+            obj.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('course_metadata', '0020_auto_20160819_1942'),
+    ]
+
+    operations = [
+        migrations.RunPython(delete_partnerless_courses, reverse_code=migrations.RunPython.noop),
+        migrations.AlterUniqueTogether(
+            name='courseorganization',
+            unique_together=set([]),
+        ),
+        migrations.AlterIndexTogether(
+            name='courseorganization',
+            index_together=set([]),
+        ),
+        migrations.RemoveField(
+            model_name='courseorganization',
+            name='course',
+        ),
+        migrations.RemoveField(
+            model_name='courseorganization',
+            name='organization',
+        ),
+        migrations.AlterModelOptions(
+            name='course',
+            options={},
+        ),
+        migrations.RemoveField(
+            model_name='courserun',
+            name='image',
+        ),
+        migrations.RemoveField(
+            model_name='courserun',
+            name='instructors',
+        ),
+        migrations.RemoveField(
+            model_name='courserun',
+            name='marketing_url',
+        ),
+        migrations.RemoveField(
+            model_name='historicalcourse',
+            name='image',
+        ),
+        migrations.RemoveField(
+            model_name='historicalcourse',
+            name='learner_testimonial',
+        ),
+        migrations.RemoveField(
+            model_name='historicalcourse',
+            name='marketing_url',
+        ),
+        migrations.RemoveField(
+            model_name='historicalcourserun',
+            name='image',
+        ),
+        migrations.RemoveField(
+            model_name='historicalcourserun',
+            name='marketing_url',
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='authoring_organizations',
+            field=sortedm2m.fields.SortedManyToManyField(help_text=None, blank=True, to='course_metadata.Organization',
+                                                         related_name='authored_courses'),
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='card_image_url',
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='slug',
+            field=django_extensions.db.fields.AutoSlugField(blank=True, populate_from='key', editable=False),
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='sponsoring_organizations',
+            field=sortedm2m.fields.SortedManyToManyField(help_text=None, blank=True, to='course_metadata.Organization',
+                                                         related_name='sponsored_courses'),
+        ),
+        migrations.AddField(
+            model_name='course',
+            name='uuid',
+            field=models.UUIDField(editable=False, verbose_name='UUID', null=True),
+        ),
+        migrations.AddField(
+            model_name='courserun',
+            name='card_image_url',
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='courserun',
+            name='slug',
+            field=django_extensions.db.fields.AutoSlugField(blank=True, populate_from='key', editable=False),
+        ),
+        migrations.AddField(
+            model_name='courserun',
+            name='uuid',
+            field=models.UUIDField(editable=False, verbose_name='UUID', null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='card_image_url',
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='slug',
+            field=django_extensions.db.fields.AutoSlugField(blank=True, populate_from='key', editable=False),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='uuid',
+            field=models.UUIDField(editable=False, verbose_name='UUID', null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalcourserun',
+            name='card_image_url',
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalcourserun',
+            name='slug',
+            field=django_extensions.db.fields.AutoSlugField(blank=True, populate_from='key', editable=False),
+        ),
+        migrations.AddField(
+            model_name='historicalcourserun',
+            name='uuid',
+            field=models.UUIDField(editable=False, verbose_name='UUID', null=True),
+        ),
+        migrations.AlterField(
+            model_name='course',
+            name='key',
+            field=models.CharField(max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='course',
+            name='partner',
+            field=models.ForeignKey(to='core.Partner'),
+        ),
+        migrations.AlterField(
+            model_name='historicalcourse',
+            name='key',
+            field=models.CharField(max_length=255),
+        ),
+        migrations.RunPython(add_uuid_to_courses_and_course_runs, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name='course',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, verbose_name='UUID', editable=False),
+        ),
+        migrations.AlterField(
+            model_name='courserun',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, verbose_name='UUID', editable=False),
+        ),
+        migrations.AlterField(
+            model_name='historicalcourse',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, verbose_name='UUID', editable=False),
+        ),
+        migrations.AlterField(
+            model_name='historicalcourserun',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, verbose_name='UUID', editable=False),
+        ),
+        migrations.AlterUniqueTogether(
+            name='course',
+            unique_together=set([('partner', 'key'), ('partner', 'uuid')]),
+        ),
+        migrations.RemoveField(
+            model_name='course',
+            name='image',
+        ),
+        migrations.RemoveField(
+            model_name='course',
+            name='learner_testimonial',
+        ),
+        migrations.RemoveField(
+            model_name='course',
+            name='marketing_url',
+        ),
+        migrations.RemoveField(
+            model_name='course',
+            name='organizations',
+        ),
+        migrations.DeleteModel(
+            name='CourseOrganization',
+        ),
+
+    ]

course_discovery/apps/course_metadata/models.py

@@ -19,9 +19,9 @@ from taggit.managers import TaggableManager
 
 from course_discovery.apps.core.models import Currency, Partner
 from course_discovery.apps.course_metadata.query import CourseQuerySet
+from course_discovery.apps.course_metadata.utils import UploadToFieldNamePath
 from course_discovery.apps.course_metadata.utils import clean_query
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
-from course_discovery.apps.course_metadata.utils import UploadToFieldNamePath
 
 logger = logging.getLogger(__name__)
 
@@ -219,41 +219,47 @@ class Position(TimeStampedModel):
 
 class Course(TimeStampedModel):
     """ Course model. """
-    key = models.CharField(max_length=255, db_index=True, unique=True)
+    partner = models.ForeignKey(Partner)
+    uuid = models.UUIDField(default=uuid4, editable=False, verbose_name=_('UUID'))
+    key = models.CharField(max_length=255)
     title = models.CharField(max_length=255, default=None, null=True, blank=True)
     short_description = models.CharField(max_length=255, default=None, null=True, blank=True)
     full_description = models.TextField(default=None, null=True, blank=True)
-    organizations = models.ManyToManyField('Organization', through='CourseOrganization', blank=True)
+    authoring_organizations = SortedManyToManyField(Organization, blank=True, related_name='authored_courses')
+    sponsoring_organizations = SortedManyToManyField(Organization, blank=True, related_name='sponsored_courses')
     subjects = models.ManyToManyField(Subject, blank=True)
     prerequisites = models.ManyToManyField(Prerequisite, blank=True)
     level_type = models.ForeignKey(LevelType, default=None, null=True, blank=True)
     expected_learning_items = SortedManyToManyField(ExpectedLearningItem, blank=True)
-    image = models.ForeignKey(Image, default=None, null=True, blank=True)
+    card_image_url = models.URLField(null=True, blank=True)
+    slug = AutoSlugField(populate_from='key', editable=True)
     video = models.ForeignKey(Video, default=None, null=True, blank=True)
-    marketing_url = models.URLField(max_length=255, null=True, blank=True)
-    learner_testimonial = models.CharField(
-        max_length=50, null=True, blank=True, help_text=_(
-            "A quote from a learner in the course, demonstrating the value of taking the course"
-        )
-    )
-
     number = models.CharField(
         max_length=50, null=True, blank=True, help_text=_(
-            "Course number format e.g CS002x, BIO1.1x, BIO1.2x"
+            'Course number format e.g CS002x, BIO1.1x, BIO1.2x'
         )
     )
-    partner = models.ForeignKey(Partner, null=True, blank=False)
 
     history = HistoricalRecords()
     objects = CourseQuerySet.as_manager()
 
-    @property
-    def owners(self):
-        return self.organizations.filter(courseorganization__relation_type=CourseOrganization.OWNER)
+    class Meta:
+        unique_together = (
+            ('partner', 'uuid'),
+            ('partner', 'key'),
+        )
+
+    def __str__(self):
+        return '{key}: {title}'.format(key=self.key, title=self.title)
 
     @property
-    def sponsors(self):
-        return self.organizations.filter(courseorganization__relation_type=CourseOrganization.SPONSOR)
+    def marketing_url(self):
+        url = None
+        if self.partner.marketing_site_url_root:
+            path = 'course/{slug}'.format(slug=self.slug)
+            url = urljoin(self.partner.marketing_site_url_root, path)
+
+        return url
 
     @property
     def active_course_runs(self):
@@ -289,9 +295,6 @@ class Course(TimeStampedModel):
         ids = [result.pk for result in results]
         return cls.objects.filter(pk__in=ids)
 
-    def __str__(self):
-        return '{key}: {title}'.format(key=self.key, title=self.title)
-
 
 class CourseRun(TimeStampedModel):
     """ CourseRun model. """
@@ -307,6 +310,7 @@ class CourseRun(TimeStampedModel):
         (INSTRUCTOR_PACED, _('Instructor-paced')),
     )
 
+    uuid = models.UUIDField(default=uuid4, editable=False, verbose_name=_('UUID'))
     course = models.ForeignKey(Course, related_name='course_runs')
     key = models.CharField(max_length=255, unique=True)
     title_override = models.CharField(
@@ -328,7 +332,6 @@ class CourseRun(TimeStampedModel):
         help_text=_(
             "Full description specific for this run of a course. Leave this value blank to default to "
             "the parent course's full_description attribute."))
-    instructors = SortedManyToManyField(Person, blank=True, related_name='courses_instructed')
     staff = SortedManyToManyField(Person, blank=True, related_name='courses_staffed')
     min_effort = models.PositiveSmallIntegerField(
         null=True, blank=True,
@@ -340,13 +343,18 @@ class CourseRun(TimeStampedModel):
     transcript_languages = models.ManyToManyField(LanguageTag, blank=True, related_name='transcript_courses')
     pacing_type = models.CharField(max_length=255, choices=PACING_CHOICES, db_index=True, null=True, blank=True)
     syllabus = models.ForeignKey(SyllabusItem, default=None, null=True, blank=True)
-    image = models.ForeignKey(Image, default=None, null=True, blank=True)
+    card_image_url = models.URLField(null=True, blank=True)
     video = models.ForeignKey(Video, default=None, null=True, blank=True)
-    marketing_url = models.URLField(max_length=255, null=True, blank=True)
+    slug = AutoSlugField(populate_from='key', editable=True)
 
     history = HistoricalRecords()
 
     @property
+    def marketing_url(self):
+        path = 'course/{slug}'.format(slug=self.slug)
+        return urljoin(self.course.partner.marketing_site_url_root, path)
+
+    @property
     def title(self):
         return self.title_override or self.course.title
 
@@ -381,8 +389,12 @@ class CourseRun(TimeStampedModel):
         return self.course.subjects
 
     @property
-    def organizations(self):
-        return self.course.organizations
+    def authoring_organizations(self):
+        return self.course.authoring_organizations
+
+    @property
+    def sponsoring_organizations(self):
+        return self.course.sponsoring_organizations
 
     @property
     def prerequisites(self):
@@ -390,7 +402,7 @@ class CourseRun(TimeStampedModel):
 
     @property
     def programs(self):
-        return self.course.programs
+        return self.course.programs  # pylint: disable=no-member
 
     @property
     def seat_types(self):
@@ -415,13 +427,6 @@ class CourseRun(TimeStampedModel):
         return None
 
     @property
-    def image_url(self):
-        if self.image:
-            return self.image.src
-
-        return None
-
-    @property
     def level_type(self):
         return self.course.level_type
 
@@ -503,29 +508,6 @@ class Seat(TimeStampedModel):
         )
 
 
-class CourseOrganization(TimeStampedModel):
-    """ CourseOrganization model. """
-    OWNER = 'owner'
-    SPONSOR = 'sponsor'
-
-    RELATION_TYPE_CHOICES = (
-        (OWNER, _('Owner')),
-        (SPONSOR, _('Sponsor')),
-    )
-
-    course = models.ForeignKey(Course)
-    organization = models.ForeignKey(Organization)
-    relation_type = models.CharField(max_length=63, choices=RELATION_TYPE_CHOICES)
-
-    class Meta(object):
-        index_together = (
-            ('course', 'relation_type'),
-        )
-        unique_together = (
-            ('course', 'organization', 'relation_type'),
-        )
-
-
 class Endorsement(TimeStampedModel):
     endorser = models.ForeignKey(Person, blank=False, null=False)
     quote = models.TextField(blank=False, null=False)
@@ -671,10 +653,10 @@ class Program(TimeStampedModel):
         return min([course_run.start for course_run in self.course_runs])
 
     @property
-    def instructors(self):
-        instructors = [list(course_run.instructors.all()) for course_run in self.course_runs]
-        instructors = itertools.chain.from_iterable(instructors)
-        return set(instructors)
+    def staff(self):
+        staff = [list(course_run.staff.all()) for course_run in self.course_runs]
+        staff = itertools.chain.from_iterable(staff)
+        return set(staff)
 
 
 class PersonSocialNetwork(AbstractSocialNetworkModel):

course_discovery/apps/course_metadata/search_indexes.py

@@ -17,8 +17,11 @@ class OrganizationsMixin:
 
         return json.dumps(OrganizationSerializer(organization).data)
 
-    def prepare_organizations(self, obj):
-        return [self.format_organization(organization) for organization in obj.organizations.all()]
+    def _prepare_organizations(self, organizations):
+        return [self.format_organization(organization) for organization in organizations]
+
+    def prepare_authoring_organizations(self, obj):
+        return self._prepare_organizations(obj.authoring_organizations.all())
 
 
 class BaseIndex(indexes.SearchIndex):
@@ -47,12 +50,23 @@ class BaseCourseIndex(OrganizationsMixin, BaseIndex):
     full_description = indexes.CharField(model_attr='full_description', null=True)
     subjects = indexes.MultiValueField(faceted=True)
     organizations = indexes.MultiValueField(faceted=True)
+    authoring_organizations = indexes.MultiValueField(faceted=True)
+    sponsoring_organizations = indexes.MultiValueField(faceted=True)
     level_type = indexes.CharField(model_attr='level_type__name', null=True, faceted=True)
     partner = indexes.CharField(model_attr='partner__short_code', null=True, faceted=True)
 
     def prepare_subjects(self, obj):
         return [subject.name for subject in obj.subjects.all()]
 
+    def prepare_organizations(self, obj):
+        return self.prepare_authoring_organizations(obj) + self.prepare_sponsoring_organizations(obj)
+
+    def prepare_authoring_organizations(self, obj):
+        return self._prepare_organizations(obj.authoring_organizations.all())
+
+    def prepare_sponsoring_organizations(self, obj):
+        return self._prepare_organizations(obj.sponsoring_organizations.all())
+
 
 class CourseIndex(BaseCourseIndex, indexes.Indexable):
     model = Course
@@ -88,10 +102,11 @@ class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
     language = indexes.CharField(null=True, faceted=True)
     transcript_languages = indexes.MultiValueField(faceted=True)
     pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True)
-    marketing_url = indexes.CharField(model_attr='marketing_url', null=True)
+    marketing_url = indexes.CharField(null=True)
+    slug = indexes.CharField(model_attr='slug', null=True)
     seat_types = indexes.MultiValueField(model_attr='seat_types', null=True, faceted=True)
     type = indexes.CharField(model_attr='type', null=True, faceted=True)
-    image_url = indexes.CharField(model_attr='image_url', null=True)
+    image_url = indexes.CharField(model_attr='card_image_url', null=True)
     partner = indexes.CharField(model_attr='course__partner__short_code', null=True, faceted=True)
 
     def _prepare_language(self, language):
@@ -113,6 +128,9 @@ class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
     def prepare_transcript_languages(self, obj):
         return [self._prepare_language(language) for language in obj.transcript_languages.all()]
 
+    def prepare_marketing_url(self, obj):
+        return obj.marketing_url
+
 
 class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
     model = Program
@@ -133,14 +151,11 @@ class ProgramIndex(BaseIndex, indexes.Indexable, OrganizationsMixin):
     def prepare_organizations(self, obj):
         return self.prepare_authoring_organizations(obj) + self.prepare_credit_backing_organizations(obj)
 
-    def prepare_authoring_organizations(self, obj):
-        return [self.format_organization(organization) for organization in obj.authoring_organizations.all()]
-
     def prepare_authoring_organization_bodies(self, obj):
         return [self.format_organization_body(organization) for organization in obj.authoring_organizations.all()]
 
     def prepare_credit_backing_organizations(self, obj):
-        return [self.format_organization(organization) for organization in obj.credit_backing_organizations.all()]
+        return self._prepare_organizations(obj.credit_backing_organizations.all())
 
     def prepare_marketing_url(self, obj):
         return obj.marketing_url

course_discovery/apps/course_metadata/templates/search/indexes/course_metadata/basecourse_text.txt

+{{ object.uuid }}
 {{ object.key }}
 {{ object.title }}
 {{ object.short_description|default:'' }}
@@ -11,7 +12,11 @@
     {{ expected_learning_item.value }}
 {% endfor %}
 
-{% for organization in object.organizations.all %}
+{% for organization in object.authoring_organizations.all %}
+    {% include 'search/indexes/course_metadata/partials/organization.txt' %}
+{% endfor %}
+
+{% for organization in object.sponsoring_organizations.all %}
     {% include 'search/indexes/course_metadata/partials/organization.txt' %}
 {% endfor %}
 
@@ -22,3 +27,7 @@
 {% for subject in object.subjects.all %}
     {{ subject.name }}
 {% endfor %}
+
+{% for program in object.programs.all %}
+    {{ program.title }}
+{% endfor %}

course_discovery/apps/course_metadata/templates/search/indexes/course_metadata/courserun_text.txt

 {% include 'search/indexes/course_metadata/basecourse_text.txt' %}
 
 {{ object.pacing_type|default:'' }}
+{{ object.language|default:'' }}
 
 {% for language in object.transcript_languages.all %}
     {{ language }}
 {% endfor %}
+
+{% for person in object.staff.all %}
+    {{ person.full_name }}
+{% endfor %}

course_discovery/apps/course_metadata/tests/factories.py

@@ -75,12 +75,13 @@ class SeatFactory(factory.DjangoModelFactory):
 
 
 class CourseFactory(factory.DjangoModelFactory):
+    uuid = factory.LazyFunction(uuid4)
     key = FuzzyText(prefix='course-id/')
     title = FuzzyText(prefix="Test çօմɾʂҽ ")
     short_description = FuzzyText(prefix="Test çօմɾʂҽ short description")
     full_description = FuzzyText(prefix="Test çօմɾʂҽ FULL description")
     level_type = factory.SubFactory(LevelTypeFactory)
-    image = factory.SubFactory(ImageFactory)
+    card_image_url = FuzzyURL()
     video = factory.SubFactory(VideoFactory)
     marketing_url = FuzzyText(prefix='https://example.com/test-course-url')
     partner = factory.SubFactory(PartnerFactory)
@@ -93,8 +94,19 @@ class CourseFactory(factory.DjangoModelFactory):
         if create:  # pragma: no cover
             add_m2m_data(self.subjects, extracted)
 
+    @factory.post_generation
+    def authoring_organizations(self, create, extracted, **kwargs):
+        if create:
+            add_m2m_data(self.authoring_organizations, extracted)
+
+    @factory.post_generation
+    def sponsoring_organizations(self, create, extracted, **kwargs):
+        if create:
+            add_m2m_data(self.sponsoring_organizations, extracted)
+
 
 class CourseRunFactory(factory.DjangoModelFactory):
+    uuid = factory.LazyFunction(uuid4)
     key = FuzzyText(prefix='course-run-id/', suffix='/fake')
     course = factory.SubFactory(CourseFactory)
     title_override = None
@@ -106,13 +118,18 @@ class CourseRunFactory(factory.DjangoModelFactory):
     enrollment_start = FuzzyDateTime(datetime.datetime(2014, 1, 1, tzinfo=UTC))
     enrollment_end = FuzzyDateTime(datetime.datetime(2014, 1, 1, tzinfo=UTC)).end_dt
     announcement = FuzzyDateTime(datetime.datetime(2014, 1, 1, tzinfo=UTC))
-    image = factory.SubFactory(ImageFactory)
+    card_image_url = FuzzyURL()
     video = factory.SubFactory(VideoFactory)
     min_effort = FuzzyInteger(1, 10)
     max_effort = FuzzyInteger(10, 20)
     pacing_type = FuzzyChoice([name for name, __ in CourseRun.PACING_CHOICES])
     marketing_url = FuzzyText(prefix='https://example.com/test-course-url')
 
+    @factory.post_generation
+    def staff(self, create, extracted, **kwargs):
+        if create:
+            add_m2m_data(self.staff, extracted)
+
     class Meta:
         model = CourseRun
 

course_discovery/apps/course_metadata/tests/test_models.py

@@ -13,8 +13,8 @@ from freezegun import freeze_time
 from course_discovery.apps.core.models import Currency
 from course_discovery.apps.core.utils import SearchQuerySetWrapper
 from course_discovery.apps.course_metadata.models import (
-    AbstractNamedModel, AbstractMediaModel, AbstractValueModel, CourseOrganization, Course, CourseRun,
-    SeatType)
+    AbstractNamedModel, AbstractMediaModel, AbstractValueModel, Course, CourseRun, SeatType,
+)
 from course_discovery.apps.course_metadata.tests import factories
 from course_discovery.apps.core.tests.helpers import make_image_file
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
@@ -29,35 +29,11 @@ class CourseTests(TestCase):
     def setUp(self):
         super(CourseTests, self).setUp()
         self.course = factories.CourseFactory()
-        self.owner = factories.OrganizationFactory()
-        self.sponsor = factories.OrganizationFactory()
-        CourseOrganization.objects.create(
-            course=self.course,
-            organization=self.owner,
-            relation_type=CourseOrganization.OWNER
-        )
-        CourseOrganization.objects.create(
-            course=self.course,
-            organization=self.sponsor,
-            relation_type=CourseOrganization.SPONSOR
-        )
 
     def test_str(self):
         """ Verify casting an instance to a string returns a string containing the key and title. """
         self.assertEqual(str(self.course), '{key}: {title}'.format(key=self.course.key, title=self.course.title))
 
-    def test_owners(self):
-        """ Verify that the owners property returns only owner related organizations. """
-        owners = self.course.owners
-        self.assertEqual(len(owners), 1)
-        self.assertEqual(owners[0], self.owner)
-
-    def test_sponsors(self):
-        """ Verify that the sponsors property returns only sponsor related organizations. """
-        sponsors = self.course.sponsors
-        self.assertEqual(len(sponsors), 1)
-        self.assertEqual(sponsors[0], self.sponsor)
-
     def test_active_course_runs(self):
         """ Verify the property returns only course runs currently open for enrollment or opening in the future. """
         self.assertListEqual(list(self.course.active_course_runs), [])
@@ -143,14 +119,6 @@ class CourseRunTests(TestCase):
         expected = sorted([seat.type for seat in seats])
         self.assertEqual(sorted(self.course_run.seat_types), expected)
 
-    def test_image_url(self):
-        """ Verify the property returns the associated image's URL. """
-        self.assertEqual(self.course_run.image_url, self.course_run.image.src)
-
-        self.course_run.image = None
-        self.assertIsNone(self.course_run.image)
-        self.assertIsNone(self.course_run.image_url)
-
     @ddt.data(
         ('obviously-wrong', None,),
         (('audit',), 'audit',),
@@ -358,12 +326,12 @@ class ProgramTests(TestCase):
         expected_price_ranges = [{'currency': 'USD', 'min': Decimal(100), 'max': Decimal(600)}]
         self.assertEqual(program.price_ranges, expected_price_ranges)
 
-    def test_instructors(self):
-        instructors = factories.PersonFactory.create_batch(2)
-        self.course_runs[0].instructors.add(instructors[0])
-        self.course_runs[1].instructors.add(instructors[1])
+    def test_staff(self):
+        staff = factories.PersonFactory.create_batch(2)
+        self.course_runs[0].staff.add(staff[0])
+        self.course_runs[1].staff.add(staff[1])
 
-        self.assertEqual(self.program.instructors, set(instructors))
+        self.assertEqual(self.program.staff, set(staff))
 
     def test_banner_image(self):
         self.program.banner_image = make_image_file('test_banner.jpg')

course_discovery/apps/ietf_language_tags/models.py

@@ -9,7 +9,7 @@ class LanguageTag(models.Model):
     name = models.CharField(max_length=255)
 
     def __str__(self):
-        return '{code} - {name}'.format(code=self.code, name=self.name)
+        return self.name
 
     @property
     def macrolanguage(self):

course_discovery/apps/ietf_language_tags/tests/test_models.py

@@ -14,7 +14,7 @@ class LanguageTagTests(TestCase):
         code = 'te-st',
         name = 'Test LanguageTag'
         tag = LanguageTag(code=code, name=name)
-        self.assertEqual(str(tag), '{code} - {name}'.format(code=code, name=name))
+        self.assertEqual(str(tag), tag.name)
 
     def test_macrolanguage(self):
         """ Verify the property returns the macrolanguage for a given LanguageTag. """