Updated programs-related data loaders (#217)

- Creating programs only based on data from the Programs API
- Pulling additional info from the marketing site
- Associating programs with courses

ECOM-5099

course_discovery/apps/course_metadata/data_loaders/__init__.py

 import abc
 
+import html2text
 from dateutil.parser import parse
 from django.utils.functional import cached_property
 from edx_rest_api_client.client import EdxRestApiClient
@@ -78,6 +79,15 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
         return {k: cls.clean_string(v) for k, v in data.items()}
 
     @classmethod
+    def clean_html(cls, content):
+        """Cleans HTML from a string and returns a Markdown version."""
+        stripped = content.replace(' ', '')
+        html_converter = html2text.HTML2Text()
+        html_converter.wrap_links = False
+        html_converter.body_width = None
+        return html_converter.handle(stripped).strip()
+
+    @classmethod
     def parse_date(cls, date_string):
         """
         Returns a parsed date.
@@ -113,3 +123,12 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
         """ Remove orphaned objects from the database. """
         for model in (Image, Person, Video):
             delete_orphans(model)
+
+    @classmethod
+    def get_or_create_video(cls, url):
+        video = None
+
+        if url:
+            video, __ = Video.objects.get_or_create(src=url)
+
+        return video

course_discovery/apps/course_metadata/data_loaders/api.py

@@ -238,8 +238,8 @@ class EcommerceApiDataLoader(AbstractDataLoader):
 
 class ProgramsApiDataLoader(AbstractDataLoader):
     """ Loads programs from the Programs API. """
-    image_width = 435
-    image_height = 145
+    image_width = 1440
+    image_height = 480
 
     def ingest(self):
         api_url = self.partner.programs_api_url
@@ -281,15 +281,31 @@ class ProgramsApiDataLoader(AbstractDataLoader):
 
             program, __ = Program.objects.update_or_create(uuid=uuid, defaults=defaults)
 
-            organizations = []
-            for org in body['organizations']:
-                organization, __ = Organization.objects.get_or_create(
-                    key=org['key'], defaults={'name': org['display_name'], 'partner': self.partner}
-                )
-                organizations.append(organization)
+            org_keys = [org['key'] for org in body['organizations']]
+            organizations = Organization.objects.filter(key__in=org_keys, partner=self.partner)
+
+            if len(org_keys) != organizations.count():
+                logger.error('Organizations for program [%s] are invalid!', uuid)
 
             program.authoring_organizations.clear()
             program.authoring_organizations.add(*organizations)
+
+            course_run_keys = set()
+            for course_code in body.get('course_codes', []):
+                course_run_keys.update([course_run['course_key'] for course_run in course_code['run_modes']])
+
+            # The course_code key field is technically useless, so we must build the course list from the
+            # associated course runs.
+            courses = Course.objects.filter(course_runs__key__in=course_run_keys).distinct()
+            program.courses.clear()
+            program.courses.add(*courses)
+
+            excluded_course_runs = CourseRun.objects.filter(course__in=courses). \
+                exclude(key__in=course_run_keys)
+            program.excluded_course_runs.clear()
+            program.excluded_course_runs.add(*excluded_course_runs)
+
+            program.save()
         except Exception:  # pylint: disable=broad-except
             logger.exception('Failed to load program %s', uuid)
 

course_discovery/apps/course_metadata/data_loaders/marketing_site.py

+import abc
 import logging
 from urllib.parse import urljoin, urlencode
 
-import html2text
 import requests
 from django.utils.functional import cached_property
 
@@ -152,14 +152,6 @@ class DrupalApiDataLoader(AbstractDataLoader):
             logger.warning('Could not find language with ISO code [%s].', iso_code)
             return None
 
-    def clean_html(self, content):
-        """Cleans HTML from a string and returns a Markdown version."""
-        stripped = content.replace(' ', '')
-        html_converter = html2text.HTML2Text()
-        html_converter.wrap_links = False
-        html_converter.body_width = None
-        return html_converter.handle(stripped).strip()
-
     def get_courserun_image(self, body):
         image = None
         image_url = body['image']
@@ -170,9 +162,9 @@ class DrupalApiDataLoader(AbstractDataLoader):
         return image
 
 
-class MarketingSiteDataLoader(AbstractDataLoader):
+class AbstractMarketingSiteDataLoader(AbstractDataLoader):
     def __init__(self, partner, api_url, access_token=None, token_type=None):
-        super(MarketingSiteDataLoader, self).__init__(partner, api_url, access_token, token_type)
+        super(AbstractMarketingSiteDataLoader, self).__init__(partner, api_url, access_token, token_type)
 
         if not (self.partner.marketing_site_api_username and self.partner.marketing_site_api_password):
             msg = 'Marketing Site API credentials are not properly configured for Partner [{partner}]!'.format(
@@ -200,30 +192,23 @@ class MarketingSiteDataLoader(AbstractDataLoader):
 
         return session
 
-    def ingest(self):  # pragma: no cover
-        """ Load data for all supported objects (e.g. courses, runs). """
-        # TODO Ingest schools
-        # TODO Ingest instructors
-        # TODO Ingest course runs (courses)
-        self.retrieve_and_ingest_node_type('xseries', self.update_xseries)
+    def get_query_kwargs(self):
+        return {}
 
-    def retrieve_and_ingest_node_type(self, node_type, update_method):
-        """
-        Retrieves all nodes of the specified type, and calls `update_method` for each node.
-
-        Args:
-            node_type (str): Type of node to retrieve (e.g. course, xseries, school, instructor)
-            update_method: Method to which the retrieved data should be passed.
-        """
+    def ingest(self):
+        """ Load data for all supported objects (e.g. courses, runs). """
         page = 0
+        query_kwargs = self.get_query_kwargs()
 
-        while page is not None and page >= 0:
+        while page is not None and page >= 0:  # pragma: no cover
             kwargs = {
-                'type': node_type,
+                'type': self.node_type,
                 'max-depth': 2,
                 'load-entity-refs': 'subject,file,taxonomy_term,taxonomy_vocabulary,node,field_collection_item',
                 'page': page,
             }
+            kwargs.update(query_kwargs)
+
             qs = urlencode(kwargs)
             url = '{root}/node.json?{qs}'.format(root=self.api_url, qs=qs)
             response = self.api_client.get(url)
@@ -241,7 +226,7 @@ class MarketingSiteDataLoader(AbstractDataLoader):
                 try:
                     url = datum['url']
                     datum = self.clean_strings(datum)
-                    update_method(datum)
+                    self.process_node(datum)
                 except:  # pylint: disable=bare-except
                     logger.exception('Failed to load %s.', url)
 
@@ -250,16 +235,54 @@ class MarketingSiteDataLoader(AbstractDataLoader):
             else:
                 break
 
-    def update_xseries(self, data):
+    def _get_nested_url(self, field):
+        """ Helper method that retrieves the nested `url` field in the specified field, if it exists.
+        This works around the fact that Drupal represents empty objects as arrays instead of objects."""
+        field = field or {}
+        return field.get('url')
+
+    @abc.abstractmethod
+    def process_node(self, data):  # pragma: no cover
+        pass
+
+    @abc.abstractproperty
+    def node_type(self):  # pragma: no cover
+        pass
+
+
+class XSeriesMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
+    @property
+    def node_type(self):
+        return 'xseries'
+
+    def process_node(self, data):
         marketing_slug = data['url'].split('/')[-1]
-        card_image_url = data.get('field_card_image', {}).get('url')
 
-        defaults = {
-            'title': data['title'],
+        try:
+            program = Program.objects.get(marketing_slug=marketing_slug, partner=self.partner)
+        except Program.DoesNotExist:
+            logger.error('Program [%s] exists on the marketing site, but not in the Programs Service!', marketing_slug)
+            return None
+
+        card_image_url = self._get_nested_url(data.get('field_card_image'))
+        video_url = self._get_nested_url(data.get('field_product_video'))
+
+        # NOTE (CCB): Remove the heading at the beginning of the overview. Why this isn't part of the template
+        # is beyond me. It's just silly.
+        overview = self.clean_html(data['body']['value'])
+        overview = overview.lstrip('### XSeries Program Overview').strip()
+
+        data = {
             'subtitle': data.get('field_xseries_subtitle_short'),
             'category': 'XSeries',
-            'partner': self.partner,
             'card_image_url': card_image_url,
+            'overview': overview,
+            'video': self.get_or_create_video(video_url)
         }
 
-        Program.objects.update_or_create(marketing_slug=marketing_slug, defaults=defaults)
+        for field, value in data.items():
+            setattr(program, field, value)
+
+        program.save()
+        logger.info('Processed XSeries with marketing_slug [%s].', marketing_slug)
+        return program

course_discovery/apps/course_metadata/data_loaders/tests/test_api.py

@@ -18,8 +18,8 @@ from course_discovery.apps.course_metadata.models import (
 )
 from course_discovery.apps.course_metadata.tests import mock_data
 from course_discovery.apps.course_metadata.tests.factories import (
-    CourseRunFactory, SeatFactory, ImageFactory, PersonFactory, VideoFactory
-)
+    CourseRunFactory, SeatFactory, ImageFactory, PersonFactory, VideoFactory,
+    OrganizationFactory, CourseFactory)
 
 LOGGER_PATH = 'course_discovery.apps.course_metadata.data_loaders.api.logger'
 
@@ -354,8 +354,28 @@ class ProgramsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCa
     def api_url(self):
         return self.partner.programs_api_url
 
+    def create_mock_organizations(self, programs):
+        for program in programs:
+            for organization in program.get('organizations', []):
+                OrganizationFactory(key=organization['key'], partner=self.partner)
+
+    def create_mock_courses_and_runs(self, programs):
+        for program in programs:
+            for course_code in program.get('course_codes', []):
+                key = '{org}+{course}'.format(org=course_code['organization']['key'], course=course_code['key'])
+                course = CourseFactory(key=key, partner=self.partner)
+
+                for course_run in course_code['run_modes']:
+                    CourseRunFactory(course=course, key=course_run['course_key'])
+
+                # Add an additional course run that should be excluded
+                CourseRunFactory(course=course)
+
     def mock_api(self):
         bodies = mock_data.PROGRAMS_API_BODIES
+        self.create_mock_organizations(bodies)
+        self.create_mock_courses_and_runs(bodies)
+
         url = self.api_url + 'programs/'
         responses.add_callback(
             responses.GET,
@@ -369,7 +389,7 @@ class ProgramsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCa
 
     def assert_program_loaded(self, body):
         """ Assert a Program corresponding to the specified data body was properly loaded into the database. """
-        program = Program.objects.get(uuid=AbstractDataLoader.clean_string(body['uuid']))
+        program = Program.objects.get(uuid=AbstractDataLoader.clean_string(body['uuid']), partner=self.partner)
 
         self.assertEqual(program.title, body['name'])
         for attr in ('subtitle', 'category', 'status', 'marketing_slug',):
@@ -380,9 +400,20 @@ class ProgramsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCa
         self.assertEqual(keys, [org.key for org in expected_organizations])
         self.assertListEqual(list(program.authoring_organizations.all()), expected_organizations)
 
-        banner_image_url = body.get('banner_image_urls', {}).get('w435h145')
+        banner_image_url = body.get('banner_image_urls', {}).get('w1440h480')
         self.assertEqual(program.banner_image_url, banner_image_url)
 
+        course_run_keys = set()
+        course_codes = body.get('course_codes', [])
+        for course_code in course_codes:
+            course_run_keys.update([course_run['course_key'] for course_run in course_code['run_modes']])
+
+        courses = list(Course.objects.filter(course_runs__key__in=course_run_keys).distinct().order_by('key'))
+        self.assertEqual(list(program.courses.order_by('key')), courses)
+
+        # Verify the additional course runs added in create_mock_courses_and_runs are excluded.
+        self.assertEqual(program.excluded_course_runs.count(), len(course_codes))
+
     @responses.activate
     def test_ingest(self):
         """ Verify the method ingests data from the Organizations API. """
@@ -401,3 +432,19 @@ class ProgramsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCa
             self.assert_program_loaded(datum)
 
         self.loader.ingest()
+
+    @responses.activate
+    def test_ingest_with_missing_organizations(self):
+        api_data = self.mock_api()
+        Organization.objects.all().delete()
+
+        self.assertEqual(Program.objects.count(), 0)
+        self.assertEqual(Organization.objects.count(), 0)
+
+        with mock.patch(LOGGER_PATH) as mock_logger:
+            self.loader.ingest()
+            calls = [mock.call('Organizations for program [%s] are invalid!', datum['uuid']) for datum in api_data]
+            mock_logger.error.assert_has_calls(calls)
+
+        self.assertEqual(Program.objects.count(), len(api_data))
+        self.assertEqual(Organization.objects.count(), 0)

course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py

@@ -8,14 +8,15 @@ from django.test import TestCase
 from opaque_keys.edx.keys import CourseKey
 
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
-    DrupalApiDataLoader, MarketingSiteDataLoader
+    DrupalApiDataLoader, XSeriesMarketingSiteDataLoader,
 )
 from course_discovery.apps.course_metadata.data_loaders.tests import JSON
 from course_discovery.apps.course_metadata.data_loaders.tests.mixins import ApiClientTestMixin, DataLoaderTestMixin
 from course_discovery.apps.course_metadata.models import (
-    Course, CourseOrganization, CourseRun, Organization, Person, Subject, Program
+    Course, CourseOrganization, CourseRun, Organization, Person, Subject, Program, Video,
 )
 from course_discovery.apps.course_metadata.tests import mock_data
+from course_discovery.apps.course_metadata.tests.factories import ProgramFactory
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
 ENGLISH_LANGUAGE_TAG = LanguageTag(code='en-us', name='English - United States')
@@ -215,26 +216,11 @@ class DrupalApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase
         self.assertEqual(self.loader.get_language_tag(body), expected)
 
 
-class MarketingSiteDataLoaderTests(DataLoaderTestMixin, TestCase):
-    loader_class = MarketingSiteDataLoader
-    LOGIN_COOKIE = ('session_id', 'abc123')
-
+class AbstractMarketingSiteDataLoaderTestMixin(DataLoaderTestMixin):
     @property
     def api_url(self):
         return self.partner.marketing_site_url_root
 
-    def mock_login_response(self, failure=False):
-        url = self.api_url + 'user'
-        landing_url = '{base}users/{username}'.format(base=self.api_url,
-                                                      username=self.partner.marketing_site_api_username)
-        status = 500 if failure else 302
-        adding_headers = {}
-
-        if not failure:
-            adding_headers['Location'] = landing_url
-        responses.add(responses.POST, url, status=status, adding_headers=adding_headers)
-        responses.add(responses.GET, landing_url)
-
     def mock_api_callback(self, url, data):
         """ Paginate the data, one item per page. """
 
@@ -260,8 +246,68 @@ class MarketingSiteDataLoaderTests(DataLoaderTestMixin, TestCase):
 
         return request_callback
 
+    def mock_login_response(self, failure=False):
+        url = self.api_url + 'user'
+        landing_url = '{base}users/{username}'.format(base=self.api_url,
+                                                      username=self.partner.marketing_site_api_username)
+        status = 500 if failure else 302
+        adding_headers = {}
+
+        if not failure:
+            adding_headers['Location'] = landing_url
+        responses.add(responses.POST, url, status=status, adding_headers=adding_headers)
+        responses.add(responses.GET, landing_url)
+
+    def mock_api_failure(self):
+        url = self.api_url + 'node.json'
+        responses.add(responses.GET, url, status=500)
+
+    @responses.activate
+    def test_ingest_with_api_failure(self):
+        self.mock_login_response()
+        self.mock_api_failure()
+
+        with self.assertRaises(Exception):
+            self.loader.ingest()
+
+    @responses.activate
+    def test_ingest_exception_handling(self):
+        """ Verify the data loader properly handles exceptions during processing of the data from the API. """
+        self.mock_login_response()
+        api_data = self.mock_api()
+
+        with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
+            with mock.patch(LOGGER_PATH) as mock_logger:
+                self.loader.ingest()
+                self.assertEqual(mock_logger.exception.call_count, len(api_data))
+                calls = [mock.call('Failed to load %s.', datum['url']) for datum in api_data]
+                mock_logger.exception.assert_has_calls(calls)
+
+    @responses.activate
+    def test_api_client_login_failure(self):
+        self.mock_login_response(failure=True)
+        with self.assertRaises(Exception):
+            self.loader.api_client  # pylint: disable=pointless-statement
+
+    def test_constructor_without_credentials(self):
+        """ Verify the constructor raises an exception if the Partner has no marketing site credentials set. """
+        self.partner.marketing_site_api_username = None
+        with self.assertRaises(Exception):
+            self.loader_class(self.partner, self.api_url)  # pylint: disable=not-callable
+
+
+class XSeriesMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
+    loader_class = XSeriesMarketingSiteDataLoader
+    LOGIN_COOKIE = ('session_id', 'abc123')
+
+    def create_mock_programs(self, programs):
+        for program in programs:
+            marketing_slug = program['url'].split('/')[-1]
+            ProgramFactory(marketing_slug=marketing_slug, partner=self.partner)
+
     def mock_api(self):
         bodies = mock_data.MARKETING_SITE_API_XSERIES_BODIES
+        self.create_mock_programs(bodies)
         url = self.api_url + 'node.json'
 
         responses.add_callback(
@@ -273,63 +319,49 @@ class MarketingSiteDataLoaderTests(DataLoaderTestMixin, TestCase):
 
         return bodies
 
-    def mock_api_failure(self):
-        url = self.api_url + 'node.json'
-        responses.add(responses.GET, url, status=500)
-
     def assert_program_loaded(self, data):
         marketing_slug = data['url'].split('/')[-1]
-        program = Program.objects.get(marketing_slug=marketing_slug)
+        program = Program.objects.get(marketing_slug=marketing_slug, partner=self.partner)
+
+        overview = self.loader.clean_html(data['body']['value'])
+        overview = overview.lstrip('### XSeries Program Overview').strip()
+        self.assertEqual(program.overview, overview)
 
-        self.assertEqual(program.title, data['title'])
         self.assertEqual(program.subtitle, data.get('field_xseries_subtitle_short'))
         self.assertEqual(program.category, 'XSeries')
-        self.assertEqual(program.partner, self.partner)
 
         card_image_url = data.get('field_card_image', {}).get('url')
         self.assertEqual(program.card_image_url, card_image_url)
 
-    def test_constructor_without_credentials(self):
-        """ Verify the constructor raises an exception if the Partner has no marketing site credentials set. """
-        self.partner.marketing_site_api_username = None
-        with self.assertRaises(Exception):
-            self.loader_class(self.partner, self.api_url)
-
-    @responses.activate
-    def test_api_client_login_failure(self):
-        self.mock_login_response(failure=True)
-        with self.assertRaises(Exception):
-            self.loader.api_client  # pylint: disable=pointless-statement
+        video_url = data.get('field_product_video', {}).get('url')
+        if video_url:
+            video = Video.objects.get(src=video_url)
+            self.assertEqual(program.video, video)
 
     @responses.activate
     def test_ingest(self):
         self.mock_login_response()
         api_data = self.mock_api()
 
-        self.assertEqual(Program.objects.count(), 0)
-
         self.loader.ingest()
 
         for datum in api_data:
             self.assert_program_loaded(datum)
 
     @responses.activate
-    def test_ingest_with_api_failure(self):
+    def test_ingest_with_missing_programs(self):
+        """ Verify ingestion properly logs issues when programs exist on the marketing site,
+        but not the Programs API. """
         self.mock_login_response()
-        self.mock_api_failure()
+        api_data = self.mock_api()
 
-        with self.assertRaises(Exception):
-            self.loader.ingest()
+        Program.objects.all().delete()
+        self.assertEqual(Program.objects.count(), 0)
 
-    @responses.activate
-    def test_ingest_exception_handling(self):
-        """ Verify the data loader properly handles exceptions during processing of the data from the API. """
-        self.mock_login_response()
-        api_data = self.mock_api()
+        with mock.patch(LOGGER_PATH) as mock_logger:
+            self.loader.ingest()
+            self.assertEqual(Program.objects.count(), 0)
 
-        with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
-            with mock.patch(LOGGER_PATH) as mock_logger:
-                self.loader.ingest()
-                self.assertEqual(mock_logger.exception.call_count, len(api_data))
-                calls = [mock.call('Failed to load %s.', datum['url']) for datum in api_data]
-                mock_logger.exception.assert_has_calls(calls)
+            calls = [mock.call('Program [%s] exists on the marketing site, but not in the Programs Service!',
+                               datum['url'].split('/')[-1]) for datum in api_data]
+            mock_logger.error.assert_has_calls(calls)

course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py

@@ -8,7 +8,7 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
     CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
 )
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
-    DrupalApiDataLoader, MarketingSiteDataLoader,
+    DrupalApiDataLoader, XSeriesMarketingSiteDataLoader,
 )
 
 logger = logging.getLogger(__name__)
@@ -83,7 +83,7 @@ class Command(BaseCommand):
                 (partner.ecommerce_api_url, EcommerceApiDataLoader,),
                 (partner.programs_api_url, ProgramsApiDataLoader,),
                 (partner.marketing_site_api_url, DrupalApiDataLoader,),
-                (partner.marketing_site_url_root, MarketingSiteDataLoader,),
+                (partner.marketing_site_url_root, XSeriesMarketingSiteDataLoader,),
             )
 
             for api_url, loader_class in data_loaders:

course_discovery/apps/course_metadata/management/commands/tests/test_refresh_course_metadata.py

@@ -11,7 +11,7 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
     CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
 )
 from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
-    DrupalApiDataLoader, MarketingSiteDataLoader,
+    DrupalApiDataLoader, XSeriesMarketingSiteDataLoader,
 )
 from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Program
 from course_discovery.apps.course_metadata.tests import mock_data
@@ -165,6 +165,6 @@ class RefreshCourseMetadataCommandTests(TestCase):
                 call_command('refresh_course_metadata')
 
                 loader_classes = (OrganizationsApiDataLoader, CoursesApiDataLoader, EcommerceApiDataLoader,
-                                  ProgramsApiDataLoader, DrupalApiDataLoader, MarketingSiteDataLoader)
+                                  ProgramsApiDataLoader, DrupalApiDataLoader, XSeriesMarketingSiteDataLoader)
                 expected_calls = [mock.call('%s failed!', loader_class.__name__) for loader_class in loader_classes]
                 mock_logger.exception.assert_has_calls(expected_calls)

course_discovery/apps/course_metadata/tests/mock_data.py

@@ -434,7 +434,13 @@ ORGANIZATIONS_API_BODIES = [
         'short_name': 'MITx',
         'description': ' ',
         'logo': '',
-    }
+    },
+    {
+        'name': 'Delft University of Technology',
+        'short_name': 'DelftX',
+        'description': ' ',
+        'logo': '',
+    },
 ]
 
 PROGRAMS_API_BODIES = [
@@ -457,7 +463,81 @@ PROGRAMS_API_BODIES = [
             'w348h116': 'https://example.com/delft-water__348x116.jpg',
             'w726h242': 'https://example.com/delft-water__726x242.jpg',
             'w435h145': 'https://example.com/delft-water__435x145.jpg'
-        }
+        },
+        'course_codes': [
+            {
+                'display_name': 'Introduction to Water and Climate',
+                'key': 'CTB3300WCx',
+                'organization': {
+                    'display_name': 'Delft University of Technology',
+                    'key': 'DelftX'
+                },
+                'run_modes': [
+                    {
+                        'course_key': 'course-v1:Delftx+CTB3300WCx+2015_T3',
+                        'mode_slug': 'verified',
+                        'sku': 'EFF47EC',
+                        'start_date': '2015-11-05T07:39:02.791741Z',
+                        'run_key': '2015_T3'
+                    },
+                    {
+                        'course_key': 'DelftX/CTB3300WCx/2T2014',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2014-08-26T10:00:00Z',
+                        'run_key': '2T2014'
+                    }
+                ]
+            },
+            {
+                'display_name': 'Introduction to the Treatment of Urban Sewage',
+                'key': 'CTB3365STx',
+                'organization': {
+                    'display_name': 'Delft University of Technology',
+                    'key': 'DelftX'
+                },
+                'run_modes': [
+                    {
+                        'course_key': 'course-v1:DelftX+CTB3365STx+1T2016',
+                        'mode_slug': 'verified',
+                        'sku': 'F773612',
+                        'start_date': '2015-11-05T07:39:02.791741Z',
+                        'run_key': '1T2016'
+                    },
+                    {
+                        'course_key': 'DelftX/CTB3365STx/2T2015',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2015-01-27T12:00:00Z',
+                        'run_key': '2T2015'
+                    }
+                ]
+            },
+            {
+                'display_name': 'Introduction to Drinking Water Treatment',
+                'key': 'CTB3365DWx',
+                'organization': {
+                    'display_name': 'Delft University of Technology',
+                    'key': 'DelftX'
+                },
+                'run_modes': [
+                    {
+                        'course_key': 'course-v1:DelftX+CTB3365DWx+1T2016',
+                        'mode_slug': 'verified',
+                        'sku': '61B1920',
+                        'start_date': '2015-11-05T07:39:02.791741Z',
+                        'run_key': '1T2016'
+                    },
+                    {
+                        'course_key': 'DelftX/CTB3365DWx/3T2014',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2014-10-28T12:00:00Z',
+                        'run_key': '3T2014'
+                    }
+                ]
+            }
+        ],
     },
     {
         'uuid': 'b043f467-5e80-4225-93d2-248a93a8556a',
@@ -474,6 +554,66 @@ PROGRAMS_API_BODIES = [
             }
         ],
         'banner_image_urls': {},
+        'course_codes': [
+            {
+                'display_name': 'Supply Chain and Logistics Fundamentals',
+                'key': 'CTL.SC1x_1',
+                'organization': {
+                    'display_name': 'the Massachusetts Institute of Technology',
+                    'key': 'MITx'
+                },
+                'run_modes': [
+                    {
+                        'course_key': 'course-v1:MITx+CTL.SC1x_1+2T2015',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2015-05-27T00:00:00Z',
+                        'run_key': '2T2015'
+                    },
+                    {
+                        'course_key': 'MITx/ESD.SCM1x/3T2014',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2014-09-24T00:30:00Z',
+                        'run_key': '3T2014'
+                    }
+                ]
+            },
+            {
+                'display_name': 'Supply Chain Design',
+                'key': 'CTL.SC2x',
+                'organization': {
+                    'display_name': 'the Massachusetts Institute of Technology',
+                    'key': 'MITx'
+                },
+                'run_modes': [
+                    {
+                        'course_key': 'course-v1:MITx+CTL.SC2x+3T2015',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2015-09-30T00:00:00Z',
+                        'run_key': '3T2015'
+                    }
+                ]
+            },
+            {
+                'display_name': 'Supply Chain Dynamics',
+                'key': 'CTL.SC3x',
+                'organization': {
+                    'display_name': 'the Massachusetts Institute of Technology',
+                    'key': 'MITx'
+                },
+                'run_modes': [
+                    {
+                        'course_key': 'course-v1:MITx+CTL.SC3x+2T2016',
+                        'mode_slug': 'verified',
+                        'sku': '',
+                        'start_date': '2016-05-18T00:00:00Z',
+                        'run_key': '2T2016'
+                    }
+                ]
+            }
+        ],
     },
 
     # This item is invalid (due to a null marketing_slug) and will not be loaded.
@@ -498,207 +638,160 @@ PROGRAMS_API_BODIES = [
 
 MARKETING_SITE_API_XSERIES_BODIES = [
     {
-        'field_course_effort': 'self-paced: 3 hours per week',
         'body': {
-            'value': '<p>The Astrophysics XSeries Program consists of four foundational courses in astrophysics taught '
-                     'by prestigious leaders in the field, including Nobel Prize winners.  You will be taught by Brian '
-                     'Schmidt, who led the team that discovered dark energy – work which won him the 2011 Nobel Prize '
-                     'for Physics, and by prize-winning educator, science communicator and astrophysics researcher '
-                     'Paul Francis, who will take you through an incredible journey where you learn about the unsolved '
-                     'mysteries of the universe, exoplanets, black holes and supernovae, and general cosmology. '
-                     'Astronomy and astrophysics is the study of everything beyond Earth. Astronomers work in '
-                     'universities, at observatories, for various space agencies like NASA, and more. The study of '
-                     'astronomy provides you with a wide range of skills in math, engineering, and computation which '
-                     'are sought after skills across many occupations. This XSeries Program is great for anyone to '
-                     'start their studies in astronomy and astrophysics or individuals simply interested in what lies '
-                     'beyond Earth.</p>',
+            'value': '<h3><span>XSeries Program Overview</span></h3> <p>Safe water supply and hygienic water '
+                     'treatment are prerequisites for the well-being of communities all over the world. This '
+                     'Water XSeries, offered by the water management experts of TU Delft, will give you a unique '
+                     'opportunity to gain access to world-class knowledge and expertise in this field.</p> <p>'
+                     'This 3-course series will cover questions such as: How does climate change affect water '
+                     'cycle and public safety? How to use existing technologies to treat groundwater and surface '
+                     'water so we have safe drinking water? How do we take care of sewage produced in the cities '
+                     'on a daily basis? You will learn what are the physical, chemical and biological processes '
+                     'involved; carry out simple experiments at home; and have the chance to make a basic design '
+                     'of a drinking water treatment plant</p>',
             'summary': '',
             'format': 'standard_html'
         },
         'field_xseries_banner_image': {
-            'fid': '65336',
-            'name': 'aat075a_72.jpg',
+            'fid': '66321',
+            'name': 'waterxseries_course_image.jpg',
             'mime': 'image/jpeg',
-            'size': '146765',
-            'url': 'https://stage.edx.org/sites/default/files/xseries/image/banner/aat075a_72.jpg',
-            'timestamp': '1438027131',
+            'size': '399725',
+            'url': 'https://www.edx.org/sites/default/files/xseries/image/banner/waterxseries_course_image.jpg',
+            'timestamp': '1439307542',
             'owner': {
-                'uri': 'https://stage.edx.org/user/9761',
-                'id': '9761',
+                'uri': 'https://www.edx.org/user/10296',
+                'id': '10296',
                 'resource': 'user',
-                'uuid': '4af80bce-a315-4ea2-8eb2-a65d03014673'
+                'uuid': '45b915f3-5307-4fe0-b2ea-55ae92a2b078'
             },
-            'uuid': 'd2a87930-2d6a-4f2b-867b-8711d981404a'
+            'uuid': '79c103b4-98a1-4133-8b5d-665542997684'
         },
-        'field_course_level': 'Intermediate',
         'field_xseries_institutions': [
             {
-                'field_school_description': {
-                    'value': '<p>The Australian National University (ANU) is a celebrated place of intensive '
-                             'research, education and policy engagement. Our research has always been central to '
-                             'everything we do, shaping a holistic learning experience that goes beyond the classroom, '
-                             'giving students access to researchers who are among the best in their fields and to '
-                             'opportunities for development around Australia and the world.</p>',
-                    'format': 'standard_html'
-                },
-                'field_school_name': 'Australian National University',
-                'field_school_image_banner': {
-                    'fid': '31524',
-                    'name': 'anu-home-banner.jpg',
-                    'mime': 'image/jpeg',
-                    'size': '30181',
-                    'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu-home-banner_0.jpg',
-                    'timestamp': '1384283150',
-                    'owner': {
-                        'uri': 'https://stage.edx.org/user/1',
-                        'id': '1',
-                        'resource': 'user',
-                        'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
-                    },
-                    'uuid': 'f7fca9c1-078b-45bd-b4c9-ae5a927ba632'
-                },
-                'field_school_image_logo': {
-                    'fid': '31526',
-                    'name': 'anu_logo_200x101.png',
-                    'mime': 'image/png',
-                    'size': '13977',
-                    'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu_logo_200x101_0.png',
-                    'timestamp': '1384283150',
-                    'owner': {
-                        'uri': 'https://stage.edx.org/user/1',
-                        'id': '1',
-                        'resource': 'user',
-                        'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
-                    },
-                    'uuid': '74a40d7e-e81f-4de0-9733-04ca12d25605'
-                },
-                'field_school_image_logo_thumb': {
-                    'fid': '31525',
-                    'name': 'anu_logo_185x48.png',
-                    'mime': 'image/png',
-                    'size': '2732',
-                    'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu_logo_185x48_0.png',
-                    'timestamp': '1384283150',
-                    'owner': {
-                        'uri': 'https://stage.edx.org/user/1',
-                        'id': '1',
-                        'resource': 'user',
-                        'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
-                    },
-                    'uuid': '14fbc10e-c6a8-499f-a53c-032f92c9da32'
-                },
-                'field_school_image_logo_sub': {
-                    'fid': '31527',
-                    'name': 'anu-on-edx-logo.png',
-                    'mime': 'image/png',
-                    'size': '4517',
-                    'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu-on-edx-logo_0.png',
-                    'timestamp': '1384283150',
-                    'owner': {
-                        'uri': 'https://stage.edx.org/user/1',
-                        'id': '1',
-                        'resource': 'user',
-                        'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
-                    },
-                    'uuid': 'ea74abe3-66ce-48ba-bf6d-34b2e109fbeb'
-                },
-                'field_school_description_private': [],
-                'field_school_subdomain_prefix': None,
-                'field_school_url_slug': 'anux',
-                'field_school_is_school': True,
-                'field_school_is_partner': False,
-                'field_school_is_contributor': True,
-                'field_school_is_charter': True,
-                'field_school_is_founder': False,
-                'field_school_is_display': True,
-                'field_school_freeform': [],
-                'field_school_is_affiliate': False,
-                'field_school_display_name': None,
-                'field_school_catalog_heading': None,
-                'field_school_catalog_subheading': None,
-                'field_school_subtitle': None,
-                'nid': '635',
-                'vid': '7917',
-                'is_new': False,
+
+                'nid': '637',
                 'type': 'school',
-                'title': 'ANUx',
+                'title': 'DelftX',
                 'language': 'und',
-                'url': 'https://stage.edx.org/school/anux',
-                'edit_url': 'https://stage.edx.org/node/635/edit',
-                'status': '1',
-                'promote': '0',
-                'sticky': '0',
-                'created': '1384283059',
-                'changed': '1426706369',
-                'author': {
-                    'uri': 'https://stage.edx.org/user/143',
-                    'id': '143',
-                    'resource': 'user',
-                    'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
-                },
-                'log': 'Updated by FeedsNodeProcessor',
-                'revision': None,
+                'url': 'https://www.edx.org/school/delftx',
                 'body': [],
-                'uuid': '1e6df8ed-a3fe-4307-99b9-775af509fcba',
-                'vuuid': '98f08316-2d87-4412-8e03-838fa94a7f03'
+                'uuid': 'c484a523-d396-4aff-90f4-bb7e82e16bf6',
+                'vuuid': '7a5d8dba-9876-4d13-a4f8-75abbe1efa0b'
             }
         ],
+        'field_course_level': 'Introductory',
         'field_card_image': {
-            'fid': '65346',
-            'name': 'anu_astrophys_xseries_card.jpg',
-            'mime': 'image/jpeg',
-            'size': '53246',
-            'url': 'https://stage.edx.org/sites/default/files/card/images/anu_astrophys_xseries_card.jpg',
-            'timestamp': '1438043010',
+            'fid': '66771',
+            'name': 'waterxseries_course0.png',
+            'mime': 'image/png',
+            'size': '193569',
+            'url': 'https://www.edx.org/sites/default/files/card/images/waterxseries_course0.png',
+            'timestamp': '1439410202',
             'owner': {
-                'uri': 'https://stage.edx.org/user/9761',
-                'id': '9761',
+                'uri': 'https://www.edx.org/user/10296',
+                'id': '10296',
                 'resource': 'user',
-                'uuid': '4af80bce-a315-4ea2-8eb2-a65d03014673'
+                'uuid': '45b915f3-5307-4fe0-b2ea-55ae92a2b078'
             },
-            'uuid': '820b05ad-1283-47ab-a123-6a7a17868a37'
+            'uuid': '84e07f7f-0f42-44b3-b9f6-d24cde1d7618'
         },
-        'field_xseries_length': 'self-paced: ~9 weeks per course',
         'field_xseries_overview': {
-            'value': '<h3>What You\'ll Learn</h3> <ul><li>An understanding of the biggest unsolved mysteries in '
-                     'astrophysics and how researchers are attempting to answer them</li> <li>Methods used to find '
-                     'and study exoplanets</li> <li>How scientists tackle challenging problems</li> <li>About white '
-                     'dwarfs, novae, supernovae, neutro stars and black holes and how quantum mechanics and relativity '
-                     'help explain these objects</li> <li>How astrophysicists investigate the origin, nature and fate '
-                     'of our universe</li> </ul>',
-            'format': 'expanded_html'
+            'value': '<h3>What You\'ll Learn</h3> <ul><li>An understanding of the global water cycle and its '
+                     'various processes</li> <li>The mechanisms of climate change and their effects on water '
+                     'systems</li> <li>Drinking treatment and quality of groundwater and surfacewater</li> <li>'
+                     'The major pollutants that are present in the sewage</li> <li>The Physical, chemical, and '
+                     'biological processes involved in water treatment and distribution</li> <li>How urban water '
+                     'services function and the technologies they use</li> </ul>',
+            'format': 'standard_html'
         },
-        'field_xseries_price': '$50/Course',
-        'field_xseries_subtitle': 'Learn contemporary astrophysics from the leaders in the field.',
-        'field_xseries_subtitle_short': 'Learn contemporary astrophysics from the leaders in the field.',
-        'field_xseries_outcome': None,
-        'field_xseries_required_weeks': None,
-        'field_xseries_required_hours': None,
-        'nid': '7046',
-        'vid': '130386',
+        'field_xseries_subtitle': 'Explore water management concepts and technologies.',
+        'field_xseries_subtitle_short': 'Explore water management concepts and technologies.',
         'type': 'xseries',
-        'title': 'Astrophysics',
-        'language': 'und',
-        'url': 'https://stage.edx.org/xseries/astrophysics'
+        'title': 'Water Management',
+        'url': 'https://www.edx.org/xseries/water-management'
     },
     {
         'body': {
-            'value': '<p>In this XSeries, you will find all of the content required to be successful on the AP '
-                     'Biology exam including genetics, the cell, ecology, diversity and evolution. You will also '
-                     'find practice AP-style multiple choice and free response questions, tutorials on how to '
-                     'formulate great responses and lab experiences that will be crucial to your success on the AP '
-                     'exam.<br />  </p> <p><span>This XSeries consists of 5 courses.</span> The cost is $25 per '
-                     'course. The total cost of this XSeries is $125. The component courses for this XSeries may be '
-                     'taken individually.</p>',
+            'value': '<h3>XSeries Program Overview</h3> <p>This XSeries consists of three courses that enable '
+                     'students to learn and practice the art and science of supply chain management. The '
+                     'component courses build from fundamental concepts to advanced design and finally to '
+                     'strategic decision making. It is ideal preparation for anyone interested in succeeding in '
+                     'a career in logistics, operations, or supply chain management within any large global firm '
+                     'or organization.</p>',
             'summary': '',
             'format': 'standard_html'
         },
         'field_xseries_banner_image': {
-            'url': 'https://stage.edx.org/sites/default/files/xseries/image/banner/ap-biology-exam.jpg'
+            'fid': '76876',
+            'name': 'scm2x-gray-1440x260.jpg',
+            'mime': 'image/jpeg',
+            'size': '51626',
+            'url': 'https://www.edx.org/sites/default/files/xseries/image/banner/scm2x-gray-1440x260_0.jpg',
+            'timestamp': '1453500891',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': 'b0385fc9-9344-40f8-a094-b61f0ff66e54'
+        },
+        'field_product_video': {
+            'fid': '67536',
+            'name': 'EDXABVID2014-V064600',
+            'mime': 'video/youtube',
+            'size': '0',
+            'url': 'http://www.youtube.com/watch?v=C9DG0Nlszco',
+            'timestamp': '1457539040',
+            'owner': {
+                'uri': 'https://www.edx.org/user/10296',
+                'id': '10296',
+                'resource': 'user',
+                'uuid': '45b915f3-5307-4fe0-b2ea-55ae92a2b078'
+            },
+            'uuid': '18595aea-9c45-4df1-a3e3-cf68edbbe04b'
+        },
+        'field_xseries_institutions': [
+            {
+                'title': 'MITx',
+                'language': 'und',
+                'url': 'https://www.edx.org/school/mitx',
+                'body': [],
+                'uuid': '2a73d2ce-c34a-4e08-8223-83bca9d2f01d',
+                'vuuid': '2bf3a55e-cbde-4759-9199-fcb6c43a1d7a'
+            }
+        ],
+        'field_course_level': 'Advanced',
+        'field_card_image': {
+            'fid': '76886',
+            'name': 'banner-380x168_0.png',
+            'mime': 'image/png',
+            'size': '22072',
+            'url': 'https://www.edx.org/sites/default/files/card/images/banner-380x168_0.png',
+            'timestamp': '1453501343',
+            'owner': {
+                'uri': 'https://www.edx.org/user/1',
+                'id': '1',
+                'resource': 'user',
+                'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
+            },
+            'uuid': 'a8fbba26-4fe0-4dc2-9619-448730ff171c'
         },
-        'field_xseries_subtitle_short': 'Learn Biology!',
+        'field_xseries_overview': {
+            'value': '<h3>What You\'ll Learn</h3> <ul><li>How to make trade-offs between cost and service for '
+                     'both the design and operation of supply chains using total cost equations</li> '
+                     '<li>Fundamentals of demand planning from forecasting to Sales &amp; Operations Planning'
+                     '</li> <li>How supply chain strategies align to overall organizational strategy</li> <li>How '
+                     'supply chain activities translate into financial terms that the C-level suite understands'
+                     '</li> </ul>',
+            'format': 'standard_html'
+        },
+        'field_xseries_subtitle': 'Learn how to design and optimize the physical, financial, and information '
+                                  'flows of a supply chain to enhance business performance.',
+        'field_xseries_subtitle_short': 'Design and optimize the flow of a supply chain',
         'type': 'xseries',
-        'title': 'Biology',
-        'url': 'https://stage.edx.org/xseries/biology'
-    },
+        'title': 'Supply Chain Management',
+        'url': 'https://www.edx.org/xseries/supply-chain-management-0'
+    }
 ]