mattdrayer/SOL-1929.2: Add Partner metadata model

course_discovery/apps/core/admin.py

@@ -5,7 +5,7 @@ from django.contrib.auth.admin import UserAdmin
 from django.utils.translation import ugettext_lazy as _
 
 from course_discovery.apps.core.forms import UserThrottleRateForm
-from course_discovery.apps.core.models import User, UserThrottleRate, Currency
+from course_discovery.apps.core.models import User, UserThrottleRate, Currency, Partner
 
 
 @admin.register(User)
@@ -34,3 +34,10 @@ class CurrencyAdmin(admin.ModelAdmin):
     list_display = ('code', 'name',)
     ordering = ('code', 'name',)
     search_fields = ('code', 'name',)
+
+
+@admin.register(Partner)
+class PartnerAdmin(admin.ModelAdmin):
+    list_display = ('name', 'short_code',)
+    ordering = ('name', 'short_code',)
+    search_fields = ('name', 'short_code',)

course_discovery/apps/core/migrations/0008_partner.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0007_auto_20160510_2017'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Partner',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
+                ('created', django_extensions.db.fields.CreationDateTimeField(verbose_name='created', auto_now_add=True)),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(verbose_name='modified', auto_now=True)),
+                ('name', models.CharField(max_length=128, unique=True)),
+                ('short_code', models.CharField(max_length=8, unique=True)),
+                ('courses_api_url', models.URLField(max_length=255, null=True)),
+                ('ecommerce_api_url', models.URLField(max_length=255, null=True)),
+                ('organizations_api_url', models.URLField(max_length=255, null=True)),
+                ('programs_api_url', models.URLField(max_length=255, null=True)),
+                ('marketing_api_url', models.URLField(max_length=255, null=True)),
+                ('marketing_url_root', models.URLField(max_length=255, null=True)),
+                ('social_auth_edx_oidc_url_root', models.CharField(max_length=255, null=True)),
+                ('social_auth_edx_oidc_key', models.CharField(max_length=255, null=True)),
+                ('social_auth_edx_oidc_secret', models.CharField(max_length=255, null=True)),
+            ],
+            options={
+                'verbose_name': 'Partner',
+                'verbose_name_plural': 'Partners',
+            },
+        ),
+    ]

course_discovery/apps/core/models.py

@@ -3,6 +3,7 @@
 from django.contrib.auth.models import AbstractUser
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
+from django_extensions.db.models import TimeStampedModel
 from guardian.mixins import GuardianUserMixin
 
 
@@ -53,3 +54,24 @@ class Currency(models.Model):
 
     class Meta(object):
         verbose_name_plural = 'Currencies'
+
+
+class Partner(TimeStampedModel):
+    name = models.CharField(max_length=128, unique=True, null=False, blank=False)
+    short_code = models.CharField(max_length=8, unique=True, null=False, blank=False)
+    courses_api_url = models.URLField(max_length=255, null=True)
+    ecommerce_api_url = models.URLField(max_length=255, null=True)
+    organizations_api_url = models.URLField(max_length=255, null=True)
+    programs_api_url = models.URLField(max_length=255, null=True)
+    marketing_api_url = models.URLField(max_length=255, null=True)
+    marketing_url_root = models.URLField(max_length=255, null=True)
+    social_auth_edx_oidc_url_root = models.CharField(max_length=255, null=True)
+    social_auth_edx_oidc_key = models.CharField(max_length=255, null=True)
+    social_auth_edx_oidc_secret = models.CharField(max_length=255, null=True)
+
+    def __str__(self):
+        return '{name} ({code})'.format(name=self.name, code=self.short_code)
+
+    class Meta:
+        verbose_name = _('Partner')
+        verbose_name_plural = _('Partners')

course_discovery/apps/core/tests/factories.py

 import factory
+from factory.fuzzy import FuzzyText
 
-from course_discovery.apps.core.models import User
+from course_discovery.apps.core.models import User, Partner
+from course_discovery.apps.core.tests.utils import FuzzyUrlRoot
 
 USER_PASSWORD = 'password'
 
@@ -14,3 +16,20 @@ class UserFactory(factory.DjangoModelFactory):
 
     class Meta:
         model = User
+
+
+class PartnerFactory(factory.DjangoModelFactory):
+    name = factory.Sequence(lambda n: 'test-partner-{}'.format(n))  # pylint: disable=unnecessary-lambda
+    short_code = factory.Sequence(lambda n: 'test{}'.format(n))  # pylint: disable=unnecessary-lambda
+    courses_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
+    ecommerce_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
+    organizations_api_url = '{root}/api/organizations/v1/'.format(root=FuzzyUrlRoot().fuzz())
+    programs_api_url = '{root}/api/programs/v1/'.format(root=FuzzyUrlRoot().fuzz())
+    marketing_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
+    marketing_url_root = '{root}/'.format(root=FuzzyUrlRoot().fuzz())
+    social_auth_edx_oidc_url_root = '{root}'.format(root=FuzzyUrlRoot().fuzz())
+    social_auth_edx_oidc_key = FuzzyText().fuzz()
+    social_auth_edx_oidc_secret = FuzzyText().fuzz()
+
+    class Meta(object):
+        model = Partner

course_discovery/apps/core/tests/test_models.py

@@ -3,7 +3,7 @@
 from django.test import TestCase
 from social.apps.django_app.default.models import UserSocialAuth
 
-from course_discovery.apps.core.models import Currency
+from course_discovery.apps.core.models import Currency, Partner
 from course_discovery.apps.core.tests.factories import UserFactory
 
 
@@ -54,3 +54,17 @@ class CurrencyTests(TestCase):
         name = 'U.S. Dollar'
         instance = Currency(code=code, name=name)
         self.assertEqual(str(instance), '{code} - {name}'.format(code=code, name=name))
+
+
+class PartnerTests(TestCase):
+    """ Tests for the Partner class. """
+
+    def test_str(self):
+        """
+        Verify casting an instance to a string returns a string containing the name and short code of the partner.
+        """
+
+        code = 'test'
+        name = 'Test Partner'
+        instance = Partner(name=name, short_code=code)
+        self.assertEqual(str(instance), '{name} ({code})'.format(name=name, code=code))

course_discovery/apps/core/tests/utils.py

+import json
+
+from urllib.parse import parse_qs, urlparse
+from factory.fuzzy import (
+    BaseFuzzyAttribute, FuzzyText, FuzzyChoice
+)
+
+
+class FuzzyDomain(BaseFuzzyAttribute):
+    def fuzz(self):
+        subdomain = FuzzyText()
+        domain = FuzzyText()
+        tld = FuzzyChoice(('com', 'net', 'org', 'biz', 'pizza', 'coffee', 'diamonds', 'fail', 'win', 'wtf',))
+
+        return "{subdomain}.{domain}.{tld}".format(
+            subdomain=subdomain.fuzz(),
+            domain=domain.fuzz(),
+            tld=tld.fuzz()
+        )
+
+
+class FuzzyUrlRoot(BaseFuzzyAttribute):
+    def fuzz(self):
+        protocol = FuzzyChoice(('http', 'https',))
+        domain = FuzzyDomain()
+
+        return "{protocol}://{domain}".format(
+            protocol=protocol.fuzz(),
+            domain=domain.fuzz()
+        )
+
+
+class FuzzyURL(BaseFuzzyAttribute):
+    def fuzz(self):
+        root = FuzzyUrlRoot()
+        resource = FuzzyText()
+
+        return "{root}/{resource}".format(
+            root=root.fuzz(),
+            resource=resource.fuzz()
+        )
+
+
+def mock_api_callback(url, data, results_key=True, pagination=False):
+    def request_callback(request):
+        # pylint: disable=redefined-builtin
+
+        count = len(data)
+        next_url = None
+        previous_url = None
+
+        # Use the querystring to determine which page should be returned. Default to page 1.
+        # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
+        qs = parse_qs(urlparse(request.path_url).query)
+        page = int(qs.get('page', [1])[0])
+        page_size = int(qs.get('page_size', [1])[0])
+
+        if (page * page_size) < count:
+            next_page = page + 1
+            next_url = '{}?page={}'.format(url, next_page)
+
+        if page > 1:
+            previous_page = page - 1
+            previous_url = '{}?page={}'.format(url, previous_page)
+
+        body = {
+            'count': count,
+            'next': next_url,
+            'previous': previous_url,
+        }
+
+        if pagination:
+            body = {
+                'pagination': body
+            }
+        if results_key:
+            body['results'] = data
+        else:
+            body.update(data)
+
+        return 200, {}, json.dumps(body)
+
+    return request_callback

course_discovery/apps/course_metadata/data_loaders.py

@@ -6,8 +6,6 @@ from urllib.parse import urljoin
 
 import html2text
 from dateutil.parser import parse
-from django.conf import settings
-from django.utils.functional import cached_property
 from edx_rest_api_client.client import EdxRestApiClient
 from opaque_keys.edx.keys import CourseKey
 
@@ -25,7 +23,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
     """ Base class for all data loaders.
 
     Attributes:
-        api_url (str): URL of the API from which data is loaded
+        partner (Partner): Partner which owns the data for this data loader
         access_token (str): OAuth2 access token
         PAGE_SIZE (int): Number of items to load per API call
     """
@@ -33,12 +31,12 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
     PAGE_SIZE = 50
     SUPPORTED_TOKEN_TYPES = ('bearer', 'jwt',)
 
-    def __init__(self, api_url, access_token, token_type):
+    def __init__(self, partner, access_token, token_type):
         """
         Arguments:
-            api_url (str): URL of the API from which data is loaded
             access_token (str): OAuth2 access token
             token_type (str): The type of access token passed in (e.g. Bearer, JWT)
+            partner (Partner): The Partner which owns the APIs and data being loaded
         """
         token_type = token_type.lower()
 
@@ -46,11 +44,10 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
             raise ValueError('The token type {token_type} is invalid!'.format(token_type=token_type))
 
         self.access_token = access_token
-        self.api_url = api_url
         self.token_type = token_type
+        self.partner = partner
 
-    @cached_property
-    def api_client(self):
+    def get_api_client(self, api_url):
         """
         Returns an authenticated API client ready to call the API from which data is loaded.
 
@@ -64,7 +61,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
         else:
             kwargs['oauth_access_token'] = self.access_token
 
-        return EdxRestApiClient(self.api_url, **kwargs)
+        return EdxRestApiClient(api_url, **kwargs)
 
     @abc.abstractmethod
     def ingest(self):  # pragma: no cover
@@ -127,11 +124,12 @@ class OrganizationsApiDataLoader(AbstractDataLoader):
     """ Loads organizations from the Organizations API. """
 
     def ingest(self):
-        client = self.api_client
+        api_url = self.partner.organizations_api_url
+        client = self.get_api_client(api_url)
         count = None
         page = 1
 
-        logger.info('Refreshing Organizations from %s...', self.api_url)
+        logger.info('Refreshing Organizations from %s...', api_url)
 
         while page:
             response = client.organizations().get(page=page, page_size=self.PAGE_SIZE)
@@ -143,12 +141,11 @@ class OrganizationsApiDataLoader(AbstractDataLoader):
                 page += 1
             else:
                 page = None
-
             for body in results:
                 body = self.clean_strings(body)
                 self.update_organization(body)
 
-        logger.info('Retrieved %d organizations from %s.', count, self.api_url)
+        logger.info('Retrieved %d organizations from %s.', count, api_url)
 
         self.delete_orphans()
 
@@ -161,19 +158,22 @@ class OrganizationsApiDataLoader(AbstractDataLoader):
             'name': body['name'],
             'description': body['description'],
             'logo_image': image,
+            'partner': self.partner,
         }
         Organization.objects.update_or_create(key=body['short_name'], defaults=defaults)
+        logger.info('Created/updated organization "%s"', body['short_name'])
 
 
 class CoursesApiDataLoader(AbstractDataLoader):
     """ Loads course runs from the Courses API. """
 
     def ingest(self):
-        client = self.api_client
+        api_url = self.partner.courses_api_url
+        client = self.get_api_client(api_url)
         count = None
         page = 1
 
-        logger.info('Refreshing Courses and CourseRuns from %s...', self.api_url)
+        logger.info('Refreshing Courses and CourseRuns from %s...', api_url)
 
         while page:
             response = client.courses().get(page=page, page_size=self.PAGE_SIZE)
@@ -194,9 +194,13 @@ class CoursesApiDataLoader(AbstractDataLoader):
                     course = self.update_course(body)
                     self.update_course_run(course, body)
                 except:  # pylint: disable=bare-except
-                    logger.exception('An error occurred while updating [%s] from [%s]!', course_run_id, self.api_url)
+                    msg = 'An error occurred while updating {course_run} from {api_url}'.format(
+                        course_run=course_run_id,
+                        api_url=api_url
+                    )
+                    logger.exception(msg)
 
-        logger.info('Retrieved %d course runs from %s.', count, self.api_url)
+        logger.info('Retrieved %d course runs from %s.', count, api_url)
 
         self.delete_orphans()
 
@@ -205,10 +209,11 @@ class CoursesApiDataLoader(AbstractDataLoader):
         # which may not be unique for an organization.
         course_run_key_str = body['id']
         course_run_key = CourseKey.from_string(course_run_key_str)
-        organization, __ = Organization.objects.get_or_create(key=course_run_key.org)
+        organization, __ = Organization.objects.get_or_create(key=course_run_key.org, partner=self.partner)
         course_key = self.convert_course_run_key(course_run_key_str)
         defaults = {
-            'title': body['name']
+            'title': body['name'],
+            'partner': self.partner,
         }
         course, __ = Course.objects.update_or_create(key=course_key, defaults=defaults)
 
@@ -269,8 +274,9 @@ class DrupalApiDataLoader(AbstractDataLoader):
     """Loads course runs from the Drupal API."""
 
     def ingest(self):
-        client = self.api_client
-        logger.info('Refreshing Courses and CourseRuns from %s...', self.api_url)
+        api_url = self.partner.marketing_api_url
+        client = self.get_api_client(api_url)
+        logger.info('Refreshing Courses and CourseRuns from %s...', api_url)
         response = client.courses.get()
 
         data = response['items']
@@ -288,14 +294,18 @@ class DrupalApiDataLoader(AbstractDataLoader):
                 course = self.update_course(cleaned_body)
                 self.update_course_run(course, cleaned_body)
             except:  # pylint: disable=bare-except
-                logger.exception('An error occurred while updating [%s] from [%s]!', course_run_id, self.api_url)
+                msg = 'An error occurred while updating {course_run} from {api_url}'.format(
+                    course_run=course_run_id,
+                    api_url=api_url
+                )
+                logger.exception(msg)
 
         # Clean Organizations separately from other orphaned instances to avoid removing all orgnaziations
         # after an initial data load on an empty table.
         Organization.objects.filter(courseorganization__isnull=True).delete()
         self.delete_orphans()
 
-        logger.info('Retrieved %d course runs from %s.', len(data), self.api_url)
+        logger.info('Retrieved %d course runs from %s.', len(data), api_url)
 
     def update_course(self, body):
         """Create or update a course from Drupal data given by `body`."""
@@ -308,6 +318,7 @@ class DrupalApiDataLoader(AbstractDataLoader):
 
         course.full_description = self.clean_html(body['description'])
         course.short_description = self.clean_html(body['subtitle'])
+        course.partner = self.partner
 
         level_type, __ = LevelType.objects.get_or_create(name=body['level']['title'])
         course.level_type = level_type
@@ -335,7 +346,7 @@ class DrupalApiDataLoader(AbstractDataLoader):
             defaults = {
                 'name': sponsor_body['title'],
                 'logo_image': image,
-                'homepage_url': urljoin(settings.MARKETING_URL_ROOT, sponsor_body['uri'])
+                'homepage_url': urljoin(self.partner.marketing_url_root, sponsor_body['uri']),
             }
             organization, __ = Organization.objects.update_or_create(key=sponsor_body['uuid'], defaults=defaults)
             CourseOrganization.objects.create(
@@ -357,7 +368,7 @@ class DrupalApiDataLoader(AbstractDataLoader):
 
         course_run.language = self.get_language_tag(body)
         course_run.course = course
-        course_run.marketing_url = urljoin(settings.MARKETING_URL_ROOT, body['course_about_uri'])
+        course_run.marketing_url = urljoin(self.partner.marketing_url_root, body['course_about_uri'])
         course_run.start = self.parse_date(body['start'])
         course_run.end = self.parse_date(body['end'])
 
@@ -409,11 +420,12 @@ class EcommerceApiDataLoader(AbstractDataLoader):
     """ Loads course seats from the E-Commerce API. """
 
     def ingest(self):
-        client = self.api_client
+        api_url = self.partner.ecommerce_api_url
+        client = self.get_api_client(api_url)
         count = None
         page = 1
 
-        logger.info('Refreshing course seats from %s...', self.api_url)
+        logger.info('Refreshing course seats from %s...', api_url)
 
         while page:
             response = client.courses().get(page=page, page_size=self.PAGE_SIZE, include_products=True)
@@ -430,7 +442,7 @@ class EcommerceApiDataLoader(AbstractDataLoader):
                 body = self.clean_strings(body)
                 self.update_seats(body)
 
-        logger.info('Retrieved %d course seats from %s.', count, self.api_url)
+        logger.info('Retrieved %d course seats from %s.', count, api_url)
 
         self.delete_orphans()
 
@@ -495,11 +507,12 @@ class ProgramsApiDataLoader(AbstractDataLoader):
     image_height = 145
 
     def ingest(self):
-        client = self.api_client
+        api_url = self.partner.programs_api_url
+        client = self.get_api_client(api_url)
         count = None
         page = 1
 
-        logger.info('Refreshing programs from %s...', self.api_url)
+        logger.info('Refreshing programs from %s...', api_url)
 
         while page:
             response = client.programs.get(page=page, page_size=self.PAGE_SIZE)
@@ -516,7 +529,7 @@ class ProgramsApiDataLoader(AbstractDataLoader):
                 program = self.clean_strings(program)
                 self.update_program(program)
 
-        logger.info('Retrieved %d programs from %s.', count, self.api_url)
+        logger.info('Retrieved %d programs from %s.', count, api_url)
 
     def update_program(self, body):
         defaults = {
@@ -526,13 +539,14 @@ class ProgramsApiDataLoader(AbstractDataLoader):
             'status': body['status'],
             'marketing_slug': body['marketing_slug'],
             'image': self._get_image(body),
+            'partner': self.partner,
         }
         program, __ = Program.objects.update_or_create(uuid=body['uuid'], defaults=defaults)
 
         organizations = []
         for org in body['organizations']:
             organization, __ = Organization.objects.get_or_create(
-                key=org['key'], defaults={'name': org['display_name']}
+                key=org['key'], defaults={'name': org['display_name'], 'partner': self.partner}
             )
             organizations.append(organization)
 

course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py

 import logging
 
-from django.conf import settings
 from django.core.management import BaseCommand, CommandError
 from edx_rest_api_client.client import EdxRestApiClient
 
 from course_discovery.apps.course_metadata.data_loaders import (
     CoursesApiDataLoader, DrupalApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader
 )
+from course_discovery.apps.core.models import Partner
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +31,28 @@ class Command(BaseCommand):
             help='The type of access token being passed  (e.g. Bearer, JWT).'
         )
 
+        parser.add_argument(
+            '--partner_code',
+            action='store',
+            dest='partner_code',
+            default=None,
+            help='The short code for a specific partner to refresh.'
+        )
+
     def handle(self, *args, **options):
+        # For each partner defined...
+        partners = Partner.objects.all()
+
+        # If a specific partner was indicated, filter down the set
+        partner_code = options.get('partner_code')
+        if partner_code:
+            partners = partners.filter(short_code=partner_code)
+
+        if not partners:
+            raise CommandError('No partners available!')
+
+        for partner in partners:
+
             access_token = options.get('access_token')
             token_type = options.get('token_type')
 
@@ -44,25 +65,35 @@ class Command(BaseCommand):
 
                 try:
                     access_token, __ = EdxRestApiClient.get_oauth_access_token(
-                    '{root}/access_token'.format(root=settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT),
-                    settings.SOCIAL_AUTH_EDX_OIDC_KEY,
-                    settings.SOCIAL_AUTH_EDX_OIDC_SECRET,
+                        '{root}/access_token'.format(root=partner.social_auth_edx_oidc_url_root.strip('/')),
+                        partner.social_auth_edx_oidc_key,
+                        partner.social_auth_edx_oidc_secret,
                         token_type=token_type
                     )
                 except Exception:
                     logger.exception('No access token provided or acquired through client_credential flow.')
                     raise
 
-        loaders = (
-            (OrganizationsApiDataLoader, settings.ORGANIZATIONS_API_URL,),
-            (CoursesApiDataLoader, settings.COURSES_API_URL,),
-            (EcommerceApiDataLoader, settings.ECOMMERCE_API_URL,),
-            (DrupalApiDataLoader, settings.MARKETING_API_URL,),
-            (ProgramsApiDataLoader, settings.PROGRAMS_API_URL,),
-        )
+            loaders = []
+
+            if partner.organizations_api_url:
+                loaders.append(OrganizationsApiDataLoader)
+            if partner.courses_api_url:
+                loaders.append(CoursesApiDataLoader)
+            if partner.ecommerce_api_url:
+                loaders.append(EcommerceApiDataLoader)
+            if partner.marketing_api_url:
+                loaders.append(DrupalApiDataLoader)
+            if partner.programs_api_url:
+                loaders.append(ProgramsApiDataLoader)
 
-        for loader_class, api_url in loaders:
+            if loaders:
+                for loader_class in loaders:
                     try:
-                loader_class(api_url, access_token, token_type).ingest()
-            except Exception:
+                        loader_class(
+                            partner,
+                            access_token,
+                            token_type,
+                        ).ingest()
+                    except Exception:  # pylint: disable=broad-except
                         logger.exception('%s failed!', loader_class.__name__)

course_discovery/apps/course_metadata/management/commands/tests/test_refresh_course_metadata.py

+import json
+
+import responses
+from django.core.management import call_command, CommandError
+from django.test import TestCase
+
+from course_discovery.apps.core.tests.factories import PartnerFactory
+from course_discovery.apps.core.tests.utils import mock_api_callback
+from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Partner, Program
+from course_discovery.apps.course_metadata.tests import mock_data
+
+ACCESS_TOKEN = 'secret'
+ACCESS_TOKEN_TYPE = 'Bearer'
+JSON = 'application/json'
+LOGGER_NAME = 'course_metadata.management.commands.refresh_course_metadata'
+
+
+class RefreshCourseMetadataCommandTests(TestCase):
+
+    def setUp(self):
+        super(RefreshCourseMetadataCommandTests, self).setUp()
+        self.partner = PartnerFactory()
+
+    def mock_access_token_api(self):
+        body = {
+            'access_token': ACCESS_TOKEN,
+            'expires_in': 30
+        }
+
+        url = self.partner.social_auth_edx_oidc_url_root.strip('/') + '/access_token'
+        responses.add_callback(
+            responses.POST,
+            url,
+            callback=mock_api_callback(url, body, results_key=False),
+            content_type=JSON
+        )
+
+        return body
+
+    def mock_organizations_api(self):
+        bodies = mock_data.ORGANIZATIONS_API_BODIES
+        url = self.partner.organizations_api_url + 'organizations/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies),
+            content_type=JSON
+        )
+        return bodies
+
+    def mock_lms_courses_api(self):
+        bodies = mock_data.COURSES_API_BODIES
+        url = self.partner.courses_api_url + 'courses/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies, pagination=True),
+            content_type=JSON
+        )
+        return bodies
+
+    def mock_ecommerce_courses_api(self):
+
+        bodies = mock_data.ECOMMERCE_API_BODIES
+        url = self.partner.ecommerce_api_url + 'courses/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies),
+            content_type=JSON
+        )
+        return bodies
+
+    def mock_marketing_courses_api(self):
+        """Mock out the Marketing API. Returns a list of mocked-out course runs."""
+        body = mock_data.MARKETING_API_BODY
+        responses.add(
+            responses.GET,
+            self.partner.marketing_api_url + 'courses/',
+            body=json.dumps(body),
+            status=200,
+            content_type='application/json'
+        )
+        return body['items']
+
+    def mock_programs_api(self):
+        bodies = mock_data.PROGRAMS_API_BODIES
+        url = self.partner.programs_api_url + 'programs/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies),
+            content_type=JSON
+        )
+        return bodies
+
+    @responses.activate
+    def test_refresh_course_metadata(self):
+        """ Verify """
+        self.mock_access_token_api()
+        self.mock_organizations_api()
+        self.mock_lms_courses_api()
+        self.mock_ecommerce_courses_api()
+        self.mock_marketing_courses_api()
+        self.mock_programs_api()
+
+        call_command('refresh_course_metadata')
+
+        partners = Partner.objects.all()
+        self.assertEqual(len(partners), 1)
+
+        organizations = Organization.objects.all()
+        self.assertEqual(len(organizations), 3)
+
+        for organization in organizations:
+            self.assertEqual(organization.partner.short_code, self.partner.short_code)
+
+        courses = Course.objects.all()
+        self.assertEqual(len(courses), 2)
+        for course in courses:
+            self.assertEqual(course.partner.short_code, self.partner.short_code)
+
+        course_runs = CourseRun.objects.all()
+        self.assertEqual(len(course_runs), 3)
+        for course_run in course_runs:
+            self.assertEqual(course_run.course.partner.short_code, self.partner.short_code)
+
+        programs = Program.objects.all()
+        self.assertEqual(len(programs), 2)
+        for program in programs:
+            self.assertEqual(program.partner.short_code, self.partner.short_code)
+
+        # Refresh only a specific partner
+        command_args = ['--partner_code={0}'.format(partners[0].short_code)]
+        call_command('refresh_course_metadata', *command_args)
+
+        # Invalid partner code
+        with self.assertRaises(CommandError):
+            command_args = ['--partner_code=invalid']
+            call_command('refresh_course_metadata', *command_args)
+
+        # Access token but no token type
+        with self.assertRaises(CommandError):
+            command_args = ['--access_token=test-access-token']
+            call_command('refresh_course_metadata', *command_args)

course_discovery/apps/course_metadata/migrations/0009_auto_20160725_1751.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0008_partner'),
+        ('course_metadata', '0008_program_image'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='course',
+            name='partner',
+            field=models.ForeignKey(null=True, to='core.Partner'),
+        ),
+        migrations.AddField(
+            model_name='historicalcourse',
+            name='partner',
+            field=models.ForeignKey(related_name='+', null=True, on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False, blank=True, to='core.Partner'),
+        ),
+        migrations.AddField(
+            model_name='historicalorganization',
+            name='partner',
+            field=models.ForeignKey(related_name='+', null=True, on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False, blank=True, to='core.Partner'),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='partner',
+            field=models.ForeignKey(null=True, to='core.Partner'),
+        ),
+        migrations.AddField(
+            model_name='program',
+            name='partner',
+            field=models.ForeignKey(null=True, to='core.Partner'),
+        ),
+    ]

course_discovery/apps/course_metadata/models.py

@@ -4,7 +4,6 @@ from urllib.parse import urljoin
 from uuid import uuid4
 
 import pytz
-from django.conf import settings
 from django.db import models
 from django.db.models.query_utils import Q
 from django.utils.translation import ugettext_lazy as _
@@ -13,7 +12,7 @@ from haystack.query import SearchQuerySet
 from simple_history.models import HistoricalRecords
 from sortedm2m.fields import SortedManyToManyField
 
-from course_discovery.apps.core.models import Currency
+from course_discovery.apps.core.models import Currency, Partner
 from course_discovery.apps.course_metadata.query import CourseQuerySet
 from course_discovery.apps.course_metadata.utils import clean_query
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
@@ -132,6 +131,7 @@ class Organization(TimeStampedModel):
     description = models.TextField(null=True, blank=True)
     homepage_url = models.URLField(max_length=255, null=True, blank=True)
     logo_image = models.ForeignKey(Image, null=True, blank=True)
+    partner = models.ForeignKey(Partner, null=True, blank=False)
 
     history = HistoricalRecords()
 
@@ -189,6 +189,7 @@ class Course(TimeStampedModel):
 
     history = HistoricalRecords()
     objects = CourseQuerySet.as_manager()
+    partner = models.ForeignKey(Partner, null=True, blank=False)
 
     @property
     def owners(self):
@@ -496,6 +497,8 @@ class Program(TimeStampedModel):
 
     organizations = models.ManyToManyField(Organization, blank=True)
 
+    partner = models.ForeignKey(Partner, null=True, blank=False)
+
     def __str__(self):
         return self.title
 
@@ -503,7 +506,7 @@ class Program(TimeStampedModel):
     def marketing_url(self):
         if self.marketing_slug:
             path = '{category}/{slug}'.format(category=self.category, slug=self.marketing_slug)
-            return urljoin(settings.MARKETING_URL_ROOT, path)
+            return urljoin(self.partner.marketing_url_root, path)
 
         return None
 

course_discovery/apps/course_metadata/tests/factories.py

@@ -3,10 +3,12 @@ from uuid import uuid4
 
 import factory
 from factory.fuzzy import (
-    BaseFuzzyAttribute, FuzzyText, FuzzyChoice, FuzzyDateTime, FuzzyInteger, FuzzyDecimal
+    FuzzyText, FuzzyChoice, FuzzyDateTime, FuzzyInteger, FuzzyDecimal
 )
 from pytz import UTC
 
+from course_discovery.apps.core.tests.factories import PartnerFactory
+from course_discovery.apps.core.tests.utils import FuzzyURL
 from course_discovery.apps.core.models import Currency
 from course_discovery.apps.course_metadata.models import (
     Course, CourseRun, Organization, Person, Image, Video, Subject, Seat, Prerequisite, LevelType, Program,
@@ -15,22 +17,6 @@ from course_discovery.apps.course_metadata.models import (
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
 
-class FuzzyURL(BaseFuzzyAttribute):
-    def fuzz(self):
-        protocol = FuzzyChoice(('http', 'https',))
-        subdomain = FuzzyText()
-        domain = FuzzyText()
-        tld = FuzzyChoice(('com', 'net', 'org', 'biz', 'pizza', 'coffee', 'diamonds', 'fail', 'win', 'wtf',))
-        resource = FuzzyText()
-        return "{protocol}://{subdomain}.{domain}.{tld}/{resource}".format(
-            protocol=protocol.fuzz(),
-            subdomain=subdomain.fuzz(),
-            domain=domain.fuzz(),
-            tld=tld.fuzz(),
-            resource=resource.fuzz()
-        )
-
-
 class AbstractMediaModelFactory(factory.DjangoModelFactory):
     src = FuzzyURL()
     description = FuzzyText()
@@ -89,6 +75,7 @@ class CourseFactory(factory.DjangoModelFactory):
     image = factory.SubFactory(ImageFactory)
     video = factory.SubFactory(VideoFactory)
     marketing_url = FuzzyText(prefix='https://example.com/test-course-url')
+    partner = factory.SubFactory(PartnerFactory)
 
     class Meta:
         model = Course
@@ -123,6 +110,7 @@ class OrganizationFactory(factory.DjangoModelFactory):
     description = FuzzyText()
     homepage_url = FuzzyURL()
     logo_image = factory.SubFactory(ImageFactory)
+    partner = factory.SubFactory(PartnerFactory)
 
     class Meta:
         model = Organization
@@ -150,6 +138,7 @@ class ProgramFactory(factory.django.DjangoModelFactory):
     status = 'unpublished'
     marketing_slug = factory.Sequence(lambda n: 'test-slug-{}'.format(n))  # pylint: disable=unnecessary-lambda
     image = factory.SubFactory(ImageFactory)
+    partner = factory.SubFactory(PartnerFactory)
 
 
 class AbstractSocialNetworkModelFactory(factory.DjangoModelFactory):

course_discovery/apps/course_metadata/tests/mock_data.py

+# A course which exists, but has no associated runs
+EXISTING_COURSE = {
+    'course_key': 'PartialX+P102',
+    'title': 'A partial course',
+}
+
+EXISTING_COURSE_AND_RUN_DATA = (
+    {
+        'course_run_key': 'course-v1:SC+BreadX+3T2015',
+        'course_key': 'SC+BreadX',
+        'title': 'Bread Baking 101',
+        'current_language': 'en-us',
+    },
+    {
+        'course_run_key': 'course-v1:TX+T201+3T2015',
+        'course_key': 'TX+T201',
+        'title': 'Testing 201',
+        'current_language': ''
+    }
+)
+
+
+ORPHAN_ORGANIZATION_KEY = 'orphan_org'
+
+
+ORPHAN_STAFF_KEY = 'orphan_staff'
+
+
+COURSES_API_BODIES = [
+    {
+        'end': '2015-08-08T00:00:00Z',
+        'enrollment_start': '2015-05-15T13:00:00Z',
+        'enrollment_end': '2015-06-29T13:00:00Z',
+        'id': 'course-v1:MITx+0.111x+2T2015',
+        'media': {
+            'image': {
+                'raw': 'http://example.com/image.jpg',
+            },
+        },
+        'name': 'Making Science and Engineering Pictures: A Practical Guide to Presenting Your Work',
+        'number': '0.111x',
+        'org': 'MITx',
+        'short_description': '',
+        'start': '2015-06-15T13:00:00Z',
+        'pacing': 'self',
+    },
+    {
+        'effort': None,
+        'end': '2015-12-11T06:00:00Z',
+        'enrollment_start': None,
+        'enrollment_end': None,
+        'id': 'course-v1:KyotoUx+000x+2T2016',
+        'media': {
+            'course_image': {
+                'uri': '/asset-v1:KyotoUx+000x+2T2016+type@asset+block@000x-course_imagec-378x225.jpg'
+            },
+            'course_video': {
+                'uri': None
+            }
+        },
+        'name': 'Evolution of the Human Sociality: A Quest for the Origin of Our Social Behavior',
+        'number': '000x',
+        'org': 'KyotoUx',
+        'short_description': '',
+        'start': '2015-10-29T09:00:00Z',
+        'pacing': 'instructor,'
+    },
+    {
+        # Add a second run of KyotoUx+000x (3T2016) to test merging data across
+        # multiple course runs into a single course.
+        'effort': None,
+        'end': None,
+        'enrollment_start': None,
+        'enrollment_end': None,
+        'id': 'course-v1:KyotoUx+000x+3T2016',
+        'media': {
+            'course_image': {
+                'uri': '/asset-v1:KyotoUx+000x+3T2016+type@asset+block@000x-course_imagec-378x225.jpg'
+            },
+            'course_video': {
+                'uri': None
+            }
+        },
+        'name': 'Evolution of the Human Sociality: A Quest for the Origin of Our Social Behavior',
+        'number': '000x',
+        'org': 'KyotoUx',
+        'short_description': '',
+        'start': None,
+    },
+]
+
+
+ECOMMERCE_API_BODIES = [
+    {
+        "id": "audit/course/run",
+        "products": [
+            {
+                "structure": "parent",
+                "price": None,
+                "expires": None,
+                "attribute_values": [],
+                "is_available_to_buy": False,
+                "stockrecords": []
+            },
+            {
+                "structure": "child",
+                "expires": None,
+                "attribute_values": [],
+                "stockrecords": [
+                    {
+                        "price_currency": "USD",
+                        "price_excl_tax": "0.00",
+                    }
+                ]
+            }
+        ]
+    },
+    {
+        "id": "verified/course/run",
+        "products": [
+            {
+                "structure": "parent",
+                "price": None,
+                "expires": None,
+                "attribute_values": [],
+                "is_available_to_buy": False,
+                "stockrecords": []
+            },
+            {
+                "structure": "child",
+                "expires": None,
+                "attribute_values": [
+                    {
+                        "name": "certificate_type",
+                        "value": "honor"
+                    }
+                ],
+                "stockrecords": [
+                    {
+                        "price_currency": "EUR",
+                        "price_excl_tax": "0.00",
+                    }
+                ]
+            },
+            {
+                "structure": "child",
+                "expires": "2017-01-01T12:00:00Z",
+                "attribute_values": [
+                    {
+                        "name": "certificate_type",
+                        "value": "verified"
+                    }
+                ],
+                "stockrecords": [
+                    {
+                        "price_currency": "EUR",
+                        "price_excl_tax": "25.00",
+                    }
+                ]
+            }
+        ]
+    },
+    {
+        # This credit course has two credit seats to verify we are correctly finding/updating using the credit
+        # provider field.
+        "id": "credit/course/run",
+        "products": [
+            {
+                "structure": "parent",
+                "price": None,
+                "expires": None,
+                "attribute_values": [],
+                "is_available_to_buy": False,
+                "stockrecords": []
+            },
+            {
+                "structure": "child",
+                "expires": None,
+                "attribute_values": [],
+                "stockrecords": [
+                    {
+                        "price_currency": "USD",
+                        "price_excl_tax": "0.00",
+                    }
+                ]
+            },
+            {
+                "structure": "child",
+                "expires": "2017-01-01T12:00:00Z",
+                "attribute_values": [
+                    {
+                        "name": "certificate_type",
+                        "value": "verified"
+                    }
+                ],
+                "stockrecords": [
+                    {
+                        "price_currency": "USD",
+                        "price_excl_tax": "25.00",
+                    }
+                ]
+            },
+            {
+                "structure": "child",
+                "expires": "2017-06-01T12:00:00Z",
+                "attribute_values": [
+                    {
+                        "name": "certificate_type",
+                        "value": "credit"
+                    },
+                    {
+                        "name": "credit_hours",
+                        "value": 2
+                    },
+                    {
+                        "name": "credit_provider",
+                        "value": "asu"
+                    },
+                    {
+                        "name": "verification_required",
+                        "value": False
+                    },
+                ],
+                "stockrecords": [
+                    {
+                        "price_currency": "USD",
+                        "price_excl_tax": "250.00",
+                    }
+                ]
+            },
+            {
+                "structure": "child",
+                "expires": "2017-06-01T12:00:00Z",
+                "attribute_values": [
+                    {
+                        "name": "certificate_type",
+                        "value": "credit"
+                    },
+                    {
+                        "name": "credit_hours",
+                        "value": 2
+                    },
+                    {
+                        "name": "credit_provider",
+                        "value": "acme"
+                    },
+                    {
+                        "name": "verification_required",
+                        "value": False
+                    },
+                ],
+                "stockrecords": [
+                    {
+                        "price_currency": "USD",
+                        "price_excl_tax": "250.00",
+                    }
+                ]
+            }
+        ]
+    },
+    {  # Course with a currency not found in the database
+        "id": "nocurrency/course/run",
+        "products": [
+            {
+                "structure": "parent",
+                "price": None,
+                "expires": None,
+                "attribute_values": [],
+                "is_available_to_buy": False,
+                "stockrecords": []
+            },
+            {
+                "structure": "child",
+                "expires": None,
+                "attribute_values": [],
+                "stockrecords": [
+                    {
+                        "price_currency": "123",
+                        "price_excl_tax": "0.00",
+                    }
+                ]
+            }
+        ]
+    },
+    {  # Course which does not exist in LMS
+        "id": "fake-course-does-not-exist",
+        "products": [
+            {
+                "structure": "parent",
+                "price": None,
+                "expires": None,
+                "attribute_values": [],
+                "is_available_to_buy": False,
+                "stockrecords": []
+            },
+            {
+                "structure": "child",
+                "expires": None,
+                "attribute_values": [],
+                "stockrecords": [
+                    {
+                        "price_currency": "USD",
+                        "price_excl_tax": "0.00",
+                    }
+                ]
+            }
+        ]
+    }
+]
+
+
+MARKETING_API_BODY = {
+    'items': [
+        {
+            'title': EXISTING_COURSE_AND_RUN_DATA[0]['title'],
+            'start': '2015-06-15T13:00:00Z',
+            'end': '2015-12-15T13:00:00Z',
+            'level': {
+                'title': 'Introductory',
+            },
+            'course_about_uri': '/course/bread-baking-101',
+            'course_id': EXISTING_COURSE_AND_RUN_DATA[0]['course_run_key'],
+            'subjects': [{
+                'title': 'Bread baking',
+            }],
+            'current_language': EXISTING_COURSE_AND_RUN_DATA[0]['current_language'],
+            'subtitle': 'Learn about Bread',
+            'description': '<p>Bread is a <a href="/wiki/Staple_food" title="Staple food">staple food</a>.',
+            'sponsors': [{
+                'uuid': 'abc123',
+                'title': 'Tatte',
+                'image': 'http://example.com/tatte.jpg',
+                'uri': 'sponsor/tatte'
+            }],
+            'staff': [{
+                'uuid': 'staff123',
+                'title': 'The Muffin Man',
+                'image': 'http://example.com/muffinman.jpg',
+                'display_position': {
+                    'title': 'Baker'
+                }
+            }, {
+                'uuid': 'staffZYX',
+                'title': 'Arthur',
+                'image': 'http://example.com/kingarthur.jpg',
+                'display_position': {
+                    'title': 'King'
+                }
+            }]
+        },
+        {
+            'title': EXISTING_COURSE_AND_RUN_DATA[1]['title'],
+            'start': '2015-06-15T13:00:00Z',
+            'end': '2015-12-15T13:00:00Z',
+            'level': {
+                'title': 'Intermediate',
+            },
+            'course_about_uri': '/course/testing-201',
+            'course_id': EXISTING_COURSE_AND_RUN_DATA[1]['course_run_key'],
+            'subjects': [{
+                'title': 'testing',
+            }],
+            'current_language': EXISTING_COURSE_AND_RUN_DATA[1]['current_language'],
+            'subtitle': 'Testing 201',
+            'description': "how to test better",
+            'sponsors': [],
+            'staff': [{
+                'uuid': '432staff',
+                'title': 'Test',
+                'image': 'http://example.com/test.jpg',
+                'display_position': {
+                    'title': 'Tester'
+                }
+            }]
+        },
+        {  # Create a course which exists in LMS/Otto, but without course runs
+            'title': EXISTING_COURSE['title'],
+            'start': '2015-06-15T13:00:00Z',
+            'end': '2015-12-15T13:00:00Z',
+            'level': {
+                'title': 'Advanced',
+            },
+            'course_about_uri': '/course/partial-101',
+            'course_id': 'course-v1:{course_key}+run'.format(course_key=EXISTING_COURSE['course_key']),
+            'subjects': [{
+                'title': 'partially fake',
+            }],
+            'current_language': 'en-us',
+            'subtitle': 'Nope',
+            'description': 'what is fake?',
+            'sponsors': [{
+                'uuid': '123abc',
+                'title': 'Fake',
+                'image': 'http://example.com/fake.jpg',
+                'uri': 'sponsor/fake'
+            }, {
+                'uuid': 'qwertyuiop',
+                'title': 'Faux',
+                'image': 'http://example.com/faux.jpg',
+                'uri': 'sponsor/faux'
+            }],
+            'staff': [],
+        },
+        {  # Create a fake course run which doesn't exist in LMS/Otto
+            'title': 'A partial course',
+            'start': '2015-06-15T13:00:00Z',
+            'end': '2015-12-15T13:00:00Z',
+            'level': {
+                'title': 'Advanced',
+            },
+            'course_about_uri': '/course/partial-101',
+            'course_id': 'course-v1:fakeX+fake+reallyfake',
+            'subjects': [{
+                'title': 'seriously fake',
+            }],
+            'current_language': 'en-us',
+            'subtitle': 'Nope',
+            'description': 'what is real?',
+            'sponsors': [],
+            'staff': [],
+        },
+        # NOTE (CCB): Some of the entries are empty arrays. Remove this as part of ECOM-4493.
+        [],
+    ]
+}
+
+
+ORGANIZATIONS_API_BODIES = [
+    {
+        'name': 'edX',
+        'short_name': ' edX ',
+        'description': 'edX',
+        'logo': 'https://example.com/edx.jpg',
+    },
+    {
+        'name': 'Massachusetts Institute of Technology ',
+        'short_name': 'MITx',
+        'description': ' ',
+        'logo': '',
+    }
+]
+
+
+PROGRAMS_API_BODIES = [
+    {
+        'uuid': 'd9ee1a73-d82d-4ed7-8eb1-80ea2b142ad6',
+        'id': 1,
+        'name': 'Water Management',
+        'subtitle': 'Explore water management concepts and technologies',
+        'category': 'xseries',
+        'status': 'active',
+        'marketing_slug': 'water-management',
+        'organizations': [
+            {
+                'display_name': 'Delft University of Technology',
+                'key': 'DelftX'
+            }
+        ],
+        'banner_image_urls': {
+            'w1440h480': 'https://example.com/delft-water__1440x480.jpg',
+            'w348h116': 'https://example.com/delft-water__348x116.jpg',
+            'w726h242': 'https://example.com/delft-water__726x242.jpg',
+            'w435h145': 'https://example.com/delft-water__435x145.jpg'
+        }
+    },
+    {
+        'uuid': 'b043f467-5e80-4225-93d2-248a93a8556a',
+        'id': 2,
+        'name': 'Supply Chain Management',
+        'subtitle': 'Learn how to design and optimize the supply chain to enhance business performance.',
+        'category': 'xseries',
+        'status': 'active',
+        'marketing_slug': 'supply-chain-management-0',
+        'organizations': [
+            {
+                'display_name': 'Massachusetts Institute of Technology',
+                'key': 'MITx'
+            }
+        ],
+        'banner_image_urls': {},
+    },
+]

course_discovery/apps/course_metadata/tests/test_data_loaders.py

@@ -2,37 +2,33 @@
 import datetime
 import json
 from decimal import Decimal
-from urllib.parse import parse_qs, urlparse
 
 import ddt
 import mock
 import responses
-from django.conf import settings
-from django.test import TestCase, override_settings
+from django.test import TestCase
 from edx_rest_api_client.auth import BearerAuth, SuppliedJwtAuth
 from edx_rest_api_client.client import EdxRestApiClient
 from opaque_keys.edx.keys import CourseKey
 from pytz import UTC
 
+from course_discovery.apps.core.tests.factories import PartnerFactory
+from course_discovery.apps.core.tests.utils import mock_api_callback
 from course_discovery.apps.course_metadata.data_loaders import (
     OrganizationsApiDataLoader, CoursesApiDataLoader, DrupalApiDataLoader, EcommerceApiDataLoader, AbstractDataLoader,
     ProgramsApiDataLoader)
 from course_discovery.apps.course_metadata.models import (
     Course, CourseOrganization, CourseRun, Image, LanguageTag, Organization, Person, Seat, Subject,
     Program)
+from course_discovery.apps.course_metadata.tests import mock_data
 from course_discovery.apps.course_metadata.tests.factories import (
-    CourseRunFactory, SeatFactory, ImageFactory, PersonFactory, VideoFactory
+    CourseRunFactory, SeatFactory, ImageFactory, PartnerFactory, PersonFactory, VideoFactory
 )
 
 ACCESS_TOKEN = 'secret'
 ACCESS_TOKEN_TYPE = 'Bearer'
-COURSES_API_URL = 'https://lms.example.com/api/courses/v1'
-ECOMMERCE_API_URL = 'https://ecommerce.example.com/api/v2'
 ENGLISH_LANGUAGE_TAG = LanguageTag(code='en-us', name='English - United States')
 JSON = 'application/json'
-MARKETING_API_URL = 'https://example.com/api/catalog/v2/'
-ORGANIZATIONS_API_URL = 'https://lms.example.com/api/organizations/v0'
-PROGRAMS_API_URL = 'https://programs.example.com/api/v1'
 
 
 class AbstractDataLoaderTest(TestCase):
@@ -73,12 +69,13 @@ class AbstractDataLoaderTest(TestCase):
 # pylint: disable=not-callable
 @ddt.ddt
 class DataLoaderTestMixin(object):
-    api_url = None
     loader_class = None
+    partner = None
 
     def setUp(self):
         super(DataLoaderTestMixin, self).setUp()
-        self.loader = self.loader_class(self.api_url, ACCESS_TOKEN, ACCESS_TOKEN_TYPE)
+        self.partner = PartnerFactory()
+        self.loader = self.loader_class(self.partner, ACCESS_TOKEN, ACCESS_TOKEN_TYPE)
 
     def assert_api_called(self, expected_num_calls, check_auth=True):
         """ Asserts the API was called with the correct number of calls, and the appropriate Authorization header. """
@@ -88,24 +85,24 @@ class DataLoaderTestMixin(object):
 
     def test_init(self):
         """ Verify the constructor sets the appropriate attributes. """
-        self.assertEqual(self.loader.api_url, self.api_url)
+        self.assertEqual(self.loader.partner.short_code, self.partner.short_code)
         self.assertEqual(self.loader.access_token, ACCESS_TOKEN)
         self.assertEqual(self.loader.token_type, ACCESS_TOKEN_TYPE.lower())
 
     def test_init_with_unsupported_token_type(self):
         """ Verify the constructor raises an error if an unsupported token type is passed in. """
         with self.assertRaises(ValueError):
-            self.loader_class(self.api_url, ACCESS_TOKEN, 'not-supported')
+            self.loader_class(self.partner, ACCESS_TOKEN, 'not-supported')
 
     @ddt.unpack
     @ddt.data(
         ('Bearer', BearerAuth),
         ('JWT', SuppliedJwtAuth),
     )
-    def test_api_client(self, token_type, expected_auth_class):
+    def test_get_api_client(self, token_type, expected_auth_class):
         """ Verify the property returns an API client with the correct authentication. """
-        loader = self.loader_class(self.api_url, ACCESS_TOKEN, token_type)
-        client = loader.api_client
+        loader = self.loader_class(self.partner, ACCESS_TOKEN, token_type)
+        client = loader.get_api_client(self.partner.programs_api_url)
         self.assertIsInstance(client, EdxRestApiClient)
         # NOTE (CCB): My initial preference was to mock the constructor and ensure the correct auth arguments
         # were passed. However, that seems nearly impossible. This is the next best alternative. It is brittle, and
@@ -114,55 +111,18 @@ class DataLoaderTestMixin(object):
 
 
 @ddt.ddt
-@override_settings(ORGANIZATIONS_API_URL=ORGANIZATIONS_API_URL)
 class OrganizationsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
-    api_url = ORGANIZATIONS_API_URL
     loader_class = OrganizationsApiDataLoader
 
     def mock_api(self):
-        bodies = [
-            {
-                'name': 'edX',
-                'short_name': ' edX ',
-                'description': 'edX',
-                'logo': 'https://example.com/edx.jpg',
-            },
-            {
-                'name': 'Massachusetts Institute of Technology ',
-                'short_name': 'MITx',
-                'description': ' ',
-                'logo': '',
-            }
-        ]
-
-        def organizations_api_callback(url, data):
-            def request_callback(request):
-                # pylint: disable=redefined-builtin
-                next = None
-                count = len(bodies)
-
-                # Use the querystring to determine which page should be returned. Default to page 1.
-                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
-                qs = parse_qs(urlparse(request.path_url).query)
-                page = int(qs.get('page', [1])[0])
-
-                if page < count:
-                    next = '{}?page={}'.format(url, page)
-
-                body = {
-                    'count': count,
-                    'next': next,
-                    'previous': None,
-                    'results': [data[page - 1]]
-                }
-
-                return 200, {}, json.dumps(body)
-
-            return request_callback
-
-        url = '{host}/organizations/'.format(host=self.api_url)
-        responses.add_callback(responses.GET, url, callback=organizations_api_callback(url, bodies), content_type=JSON)
-
+        bodies = mock_data.ORGANIZATIONS_API_BODIES
+        url = self.partner.organizations_api_url + 'organizations/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies),
+            content_type=JSON
+        )
         return bodies
 
     def assert_organization_loaded(self, body):
@@ -181,19 +141,19 @@ class OrganizationsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     @responses.activate
     def test_ingest(self):
         """ Verify the method ingests data from the Organizations API. """
-        data = self.mock_api()
+        api_data = self.mock_api()
         self.assertEqual(Organization.objects.count(), 0)
 
         self.loader.ingest()
 
         # Verify the API was called with the correct authorization header
-        expected_num_orgs = len(data)
-        self.assert_api_called(expected_num_orgs)
+        self.assert_api_called(1)
 
         # Verify the Organizations were created correctly
+        expected_num_orgs = len(api_data)
         self.assertEqual(Organization.objects.count(), expected_num_orgs)
 
-        for datum in data:
+        for datum in api_data:
             self.assert_organization_loaded(datum)
 
         # Verify multiple calls to ingest data do NOT result in data integrity errors.
@@ -201,106 +161,18 @@ class OrganizationsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
 
 @ddt.ddt
-@override_settings(COURSES_API_URL=COURSES_API_URL)
 class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase):
-    api_url = COURSES_API_URL
     loader_class = CoursesApiDataLoader
 
     def mock_api(self):
-        bodies = [
-            {
-                'end': '2015-08-08T00:00:00Z',
-                'enrollment_start': '2015-05-15T13:00:00Z',
-                'enrollment_end': '2015-06-29T13:00:00Z',
-                'id': 'course-v1:MITx+0.111x+2T2015',
-                'media': {
-                    'image': {
-                        'raw': 'http://example.com/image.jpg',
-                    },
-                },
-                'name': 'Making Science and Engineering Pictures: A Practical Guide to Presenting Your Work',
-                'number': '0.111x',
-                'org': 'MITx',
-                'short_description': '',
-                'start': '2015-06-15T13:00:00Z',
-                'pacing': 'self',
-            },
-            {
-                'effort': None,
-                'end': '2015-12-11T06:00:00Z',
-                'enrollment_start': None,
-                'enrollment_end': None,
-                'id': 'course-v1:KyotoUx+000x+2T2016',
-                'media': {
-                    'course_image': {
-                        'uri': '/asset-v1:KyotoUx+000x+2T2016+type@asset+block@000x-course_imagec-378x225.jpg'
-                    },
-                    'course_video': {
-                        'uri': None
-                    }
-                },
-                'name': 'Evolution of the Human Sociality: A Quest for the Origin of Our Social Behavior',
-                'number': '000x',
-                'org': 'KyotoUx',
-                'short_description': '',
-                'start': '2015-10-29T09:00:00Z',
-                'pacing': 'instructor,'
-            },
-            {
-                # Add a second run of KyotoUx+000x (3T2016) to test merging data across
-                # multiple course runs into a single course.
-                'effort': None,
-                'end': None,
-                'enrollment_start': None,
-                'enrollment_end': None,
-                'id': 'course-v1:KyotoUx+000x+3T2016',
-                'media': {
-                    'course_image': {
-                        'uri': '/asset-v1:KyotoUx+000x+3T2016+type@asset+block@000x-course_imagec-378x225.jpg'
-                    },
-                    'course_video': {
-                        'uri': None
-                    }
-                },
-                'name': 'Evolution of the Human Sociality: A Quest for the Origin of Our Social Behavior',
-                'number': '000x',
-                'org': 'KyotoUx',
-                'short_description': '',
-                'start': None,
-            },
-        ]
-
-        def courses_api_callback(url, data):
-            def request_callback(request):
-                # pylint: disable=redefined-builtin
-                next = None
-                count = len(bodies)
-
-                # Use the querystring to determine which page should be returned. Default to page 1.
-                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
-                qs = parse_qs(urlparse(request.path_url).query)
-                page = int(qs.get('page', [1])[0])
-
-                if page < count:
-                    next = '{}?page={}'.format(url, page)
-
-                body = {
-                    'pagination': {
-                        'count': count,
-                        'next': next,
-                        'num_pages': len(data),
-                        'previous': None,
-                    },
-                    'results': [data[page - 1]]
-                }
-
-                return 200, {}, json.dumps(body)
-
-            return request_callback
-
-        url = '{host}/courses/'.format(host=settings.COURSES_API_URL)
-        responses.add_callback(responses.GET, url, callback=courses_api_callback(url, bodies), content_type=JSON)
-
+        bodies = mock_data.COURSES_API_BODIES
+        url = self.partner.courses_api_url + 'courses/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies, pagination=True),
+            content_type=JSON
+        )
         return bodies
 
     def assert_course_run_loaded(self, body):
@@ -330,20 +202,20 @@ class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     @responses.activate
     def test_ingest(self):
         """ Verify the method ingests data from the Courses API. """
-        data = self.mock_api()
+        api_data = self.mock_api()
         self.assertEqual(Course.objects.count(), 0)
         self.assertEqual(CourseRun.objects.count(), 0)
 
         self.loader.ingest()
 
         # Verify the API was called with the correct authorization header
-        expected_num_course_runs = len(data)
-        self.assert_api_called(expected_num_course_runs)
+        self.assert_api_called(1)
 
         # Verify the CourseRuns were created correctly
+        expected_num_course_runs = len(api_data)
         self.assertEqual(CourseRun.objects.count(), expected_num_course_runs)
 
-        for datum in data:
+        for datum in api_data:
             self.assert_course_run_loaded(datum)
 
         # Verify multiple calls to ingest data do NOT result in data integrity errors.
@@ -352,15 +224,17 @@ class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     @responses.activate
     def test_ingest_exception_handling(self):
         """ Verify the data loader properly handles exceptions during processing of the data from the API. """
-        data = self.mock_api()
+        api_data = self.mock_api()
 
         with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
             with mock.patch('course_discovery.apps.course_metadata.data_loaders.logger') as mock_logger:
                 self.loader.ingest()
-                self.assertEqual(mock_logger.exception.call_count, len(data))
-                mock_logger.exception.assert_called_with(
-                    'An error occurred while updating [%s] from [%s]!', data[-1]['id'], self.api_url
+                self.assertEqual(mock_logger.exception.call_count, len(api_data))
+                msg = 'An error occurred while updating {0} from {1}'.format(
+                    api_data[-1]['id'],
+                    self.partner.courses_api_url
                 )
+                mock_logger.exception.assert_called_with(msg)
 
     def test_get_pacing_type_field_missing(self):
         """ Verify the method returns None if the API response does not include a pacing field. """
@@ -422,38 +296,12 @@ class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
 
 @ddt.ddt
-@override_settings(MARKETING_API_URL=MARKETING_API_URL)
 class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
-    EXISTING_COURSE_AND_RUN_DATA = (
-        {
-            'course_run_key': 'course-v1:SC+BreadX+3T2015',
-            'course_key': 'SC+BreadX',
-            'title': 'Bread Baking 101',
-            'current_language': 'en-us',
-        },
-        {
-            'course_run_key': 'course-v1:TX+T201+3T2015',
-            'course_key': 'TX+T201',
-            'title': 'Testing 201',
-            'current_language': ''
-        }
-    )
-
-    # A course which exists, but has no associated runs
-    EXISTING_COURSE = {
-        'course_key': 'PartialX+P102',
-        'title': 'A partial course',
-    }
-
-    ORPHAN_ORGANIZATION_KEY = 'orphan_org'
-    ORPHAN_STAFF_KEY = 'orphan_staff'
-
-    api_url = MARKETING_API_URL
     loader_class = DrupalApiDataLoader
 
     def setUp(self):
         super(DrupalApiDataLoaderTests, self).setUp()
-        for course_dict in self.EXISTING_COURSE_AND_RUN_DATA:
+        for course_dict in mock_data.EXISTING_COURSE_AND_RUN_DATA:
             course = Course.objects.create(key=course_dict['course_key'], title=course_dict['title'])
             course_run = CourseRun.objects.create(
                 key=course_dict['course_run_key'],
@@ -471,130 +319,16 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                 relation_type=CourseOrganization.SPONSOR
             )
 
-        Course.objects.create(key=self.EXISTING_COURSE['course_key'], title=self.EXISTING_COURSE['title'])
-        Person.objects.create(key=self.ORPHAN_STAFF_KEY)
-        Organization.objects.create(key=self.ORPHAN_ORGANIZATION_KEY)
+        Course.objects.create(key=mock_data.EXISTING_COURSE['course_key'], title=mock_data.EXISTING_COURSE['title'])
+        Person.objects.create(key=mock_data.ORPHAN_STAFF_KEY)
+        Organization.objects.create(key=mock_data.ORPHAN_ORGANIZATION_KEY)
 
     def mock_api(self):
         """Mock out the Drupal API. Returns a list of mocked-out course runs."""
-        body = {
-            'items': [
-                {
-                    'title': self.EXISTING_COURSE_AND_RUN_DATA[0]['title'],
-                    'start': '2015-06-15T13:00:00Z',
-                    'end': '2015-12-15T13:00:00Z',
-                    'level': {
-                        'title': 'Introductory',
-                    },
-                    'course_about_uri': '/course/bread-baking-101',
-                    'course_id': self.EXISTING_COURSE_AND_RUN_DATA[0]['course_run_key'],
-                    'subjects': [{
-                        'title': 'Bread baking',
-                    }],
-                    'current_language': self.EXISTING_COURSE_AND_RUN_DATA[0]['current_language'],
-                    'subtitle': 'Learn about Bread',
-                    'description': '<p>Bread is a <a href="/wiki/Staple_food" title="Staple food">staple food</a>.',
-                    'sponsors': [{
-                        'uuid': 'abc123',
-                        'title': 'Tatte',
-                        'image': 'http://example.com/tatte.jpg',
-                        'uri': 'sponsor/tatte'
-                    }],
-                    'staff': [{
-                        'uuid': 'staff123',
-                        'title': 'The Muffin Man',
-                        'image': 'http://example.com/muffinman.jpg',
-                        'display_position': {
-                            'title': 'Baker'
-                        }
-                    }, {
-                        'uuid': 'staffZYX',
-                        'title': 'Arthur',
-                        'image': 'http://example.com/kingarthur.jpg',
-                        'display_position': {
-                            'title': 'King'
-                        }
-                    }]
-                },
-                {
-                    'title': self.EXISTING_COURSE_AND_RUN_DATA[1]['title'],
-                    'start': '2015-06-15T13:00:00Z',
-                    'end': '2015-12-15T13:00:00Z',
-                    'level': {
-                        'title': 'Intermediate',
-                    },
-                    'course_about_uri': '/course/testing-201',
-                    'course_id': self.EXISTING_COURSE_AND_RUN_DATA[1]['course_run_key'],
-                    'subjects': [{
-                        'title': 'testing',
-                    }],
-                    'current_language': self.EXISTING_COURSE_AND_RUN_DATA[1]['current_language'],
-                    'subtitle': 'Testing 201',
-                    'description': "how to test better",
-                    'sponsors': [],
-                    'staff': [{
-                        'uuid': '432staff',
-                        'title': 'Test',
-                        'image': 'http://example.com/test.jpg',
-                        'display_position': {
-                            'title': 'Tester'
-                        }
-                    }]
-                },
-                {  # Create a course which exists in LMS/Otto, but without course runs
-                    'title': self.EXISTING_COURSE['title'],
-                    'start': '2015-06-15T13:00:00Z',
-                    'end': '2015-12-15T13:00:00Z',
-                    'level': {
-                        'title': 'Advanced',
-                    },
-                    'course_about_uri': '/course/partial-101',
-                    'course_id': 'course-v1:{course_key}+run'.format(course_key=self.EXISTING_COURSE['course_key']),
-                    'subjects': [{
-                        'title': 'partially fake',
-                    }],
-                    'current_language': 'en-us',
-                    'subtitle': 'Nope',
-                    'description': 'what is fake?',
-                    'sponsors': [{
-                        'uuid': '123abc',
-                        'title': 'Fake',
-                        'image': 'http://example.com/fake.jpg',
-                        'uri': 'sponsor/fake'
-                    }, {
-                        'uuid': 'qwertyuiop',
-                        'title': 'Faux',
-                        'image': 'http://example.com/faux.jpg',
-                        'uri': 'sponsor/faux'
-                    }],
-                    'staff': [],
-                },
-                {  # Create a fake course run which doesn't exist in LMS/Otto
-                    'title': 'A partial course',
-                    'start': '2015-06-15T13:00:00Z',
-                    'end': '2015-12-15T13:00:00Z',
-                    'level': {
-                        'title': 'Advanced',
-                    },
-                    'course_about_uri': '/course/partial-101',
-                    'course_id': 'course-v1:fakeX+fake+reallyfake',
-                    'subjects': [{
-                        'title': 'seriously fake',
-                    }],
-                    'current_language': 'en-us',
-                    'subtitle': 'Nope',
-                    'description': 'what is real?',
-                    'sponsors': [],
-                    'staff': [],
-                },
-                # NOTE (CCB): Some of the entries are empty arrays. Remove this as part of ECOM-4493.
-                [],
-            ]
-        }
-
+        body = mock_data.MARKETING_API_BODY
         responses.add(
             responses.GET,
-            settings.MARKETING_API_URL + 'courses/',
+            self.partner.marketing_api_url + 'courses/',
             body=json.dumps(body),
             status=200,
             content_type='application/json'
@@ -624,6 +358,7 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
     def assert_staff_loaded(self, course_run, body):
         """Verify that staff have been loaded correctly."""
+
         course_run_staff = course_run.staff.all()
         api_staff = body['staff']
         self.assertEqual(len(course_run_staff), len(api_staff))
@@ -662,10 +397,10 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     @responses.activate
     def test_ingest(self):
         """Verify the data loader ingests data from Drupal."""
-        data = self.mock_api()
+        api_data = self.mock_api()
         # Neither the faked course, nor the empty array, should not be loaded from Drupal.
         # Change this back to -2 as part of ECOM-4493.
-        loaded_data = data[:-3]
+        loaded_data = api_data[:-3]
 
         self.loader.ingest()
 
@@ -674,27 +409,28 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
         # Assert that the fake course was not created
         self.assertEqual(CourseRun.objects.count(), len(loaded_data))
+
         for datum in loaded_data:
             self.assert_course_run_loaded(datum)
 
-        Course.objects.get(key=self.EXISTING_COURSE['course_key'], title=self.EXISTING_COURSE['title'])
+        Course.objects.get(key=mock_data.EXISTING_COURSE['course_key'], title=mock_data.EXISTING_COURSE['title'])
 
         # Verify multiple calls to ingest data do NOT result in data integrity errors.
         self.loader.ingest()
 
         # Verify that orphan data is deleted
-        self.assertFalse(Person.objects.filter(key=self.ORPHAN_STAFF_KEY).exists())
-        self.assertFalse(Organization.objects.filter(key=self.ORPHAN_ORGANIZATION_KEY).exists())
+        self.assertFalse(Person.objects.filter(key=mock_data.ORPHAN_STAFF_KEY).exists())
+        self.assertFalse(Organization.objects.filter(key=mock_data.ORPHAN_ORGANIZATION_KEY).exists())
         self.assertFalse(Person.objects.filter(key__startswith='orphan_staff_').exists())
         self.assertFalse(Organization.objects.filter(key__startswith='orphan_org_').exists())
 
     @responses.activate
     def test_ingest_exception_handling(self):
         """ Verify the data loader properly handles exceptions during processing of the data from the API. """
-        data = self.mock_api()
+        api_data = self.mock_api()
         # Include all data, except the empty array.
         # TODO: Remove the -1 after ECOM-4493 is in production.
-        expected_call_count = len(data) - 1
+        expected_call_count = len(api_data) - 1
 
         with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
             with mock.patch('course_discovery.apps.course_metadata.data_loaders.logger') as mock_logger:
@@ -702,9 +438,11 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
                 self.assertEqual(mock_logger.exception.call_count, expected_call_count)
 
                 # TODO: Change the -2 to -1 after ECOM-4493 is in production.
-                mock_logger.exception.assert_called_with(
-                    'An error occurred while updating [%s] from [%s]!', data[-2]['course_id'], self.api_url
+                msg = 'An error occurred while updating {0} from {1}'.format(
+                    api_data[-2]['course_id'],
+                    self.partner.marketing_api_url
                 )
+                mock_logger.exception.assert_called_with(msg)
 
     @ddt.data(
         ('', ''),
@@ -733,274 +471,34 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
 
 @ddt.ddt
-@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL)
 class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase):
-    api_url = ECOMMERCE_API_URL
     loader_class = EcommerceApiDataLoader
 
     def mock_api(self):
-        course_run_audit = CourseRunFactory(title_override='audit')
-        course_run_verified = CourseRunFactory(title_override='verified')
-        course_run_credit = CourseRunFactory(title_override='credit')
-        course_run_no_currency = CourseRunFactory(title_override='no currency')
 
         # create existing seats to be removed by ingest
-        SeatFactory(course_run=course_run_audit, type=Seat.PROFESSIONAL)
-        SeatFactory(course_run=course_run_verified, type=Seat.PROFESSIONAL)
-        SeatFactory(course_run=course_run_credit, type=Seat.PROFESSIONAL)
-        SeatFactory(course_run=course_run_no_currency, type=Seat.PROFESSIONAL)
-
-        bodies = [
-            {
-                "id": course_run_audit.key,
-                "products": [
-                    {
-                        "structure": "parent",
-                        "price": None,
-                        "expires": None,
-                        "attribute_values": [],
-                        "is_available_to_buy": False,
-                        "stockrecords": []
-                    },
-                    {
-                        "structure": "child",
-                        "expires": None,
-                        "attribute_values": [],
-                        "stockrecords": [
-                            {
-                                "price_currency": "USD",
-                                "price_excl_tax": "0.00",
-                            }
-                        ]
-                    }
-                ]
-            },
-            {
-                "id": course_run_verified.key,
-                "products": [
-                    {
-                        "structure": "parent",
-                        "price": None,
-                        "expires": None,
-                        "attribute_values": [],
-                        "is_available_to_buy": False,
-                        "stockrecords": []
-                    },
-                    {
-                        "structure": "child",
-                        "expires": None,
-                        "attribute_values": [
-                            {
-                                "name": "certificate_type",
-                                "value": "honor"
-                            }
-                        ],
-                        "stockrecords": [
-                            {
-                                "price_currency": "EUR",
-                                "price_excl_tax": "0.00",
-                            }
-                        ]
-                    },
-                    {
-                        "structure": "child",
-                        "expires": "2017-01-01T12:00:00Z",
-                        "attribute_values": [
-                            {
-                                "name": "certificate_type",
-                                "value": "verified"
-                            }
-                        ],
-                        "stockrecords": [
-                            {
-                                "price_currency": "EUR",
-                                "price_excl_tax": "25.00",
-                            }
-                        ]
-                    }
-                ]
-            },
-            {
-                # This credit course has two credit seats to verify we are correctly finding/updating using the credit
-                # provider field.
-                "id": course_run_credit.key,
-                "products": [
-                    {
-                        "structure": "parent",
-                        "price": None,
-                        "expires": None,
-                        "attribute_values": [],
-                        "is_available_to_buy": False,
-                        "stockrecords": []
-                    },
-                    {
-                        "structure": "child",
-                        "expires": None,
-                        "attribute_values": [],
-                        "stockrecords": [
-                            {
-                                "price_currency": "USD",
-                                "price_excl_tax": "0.00",
-                            }
-                        ]
-                    },
-                    {
-                        "structure": "child",
-                        "expires": "2017-01-01T12:00:00Z",
-                        "attribute_values": [
-                            {
-                                "name": "certificate_type",
-                                "value": "verified"
-                            }
-                        ],
-                        "stockrecords": [
-                            {
-                                "price_currency": "USD",
-                                "price_excl_tax": "25.00",
-                            }
-                        ]
-                    },
-                    {
-                        "structure": "child",
-                        "expires": "2017-06-01T12:00:00Z",
-                        "attribute_values": [
-                            {
-                                "name": "certificate_type",
-                                "value": "credit"
-                            },
-                            {
-                                "name": "credit_hours",
-                                "value": 2
-                            },
-                            {
-                                "name": "credit_provider",
-                                "value": "asu"
-                            },
-                            {
-                                "name": "verification_required",
-                                "value": False
-                            },
-                        ],
-                        "stockrecords": [
-                            {
-                                "price_currency": "USD",
-                                "price_excl_tax": "250.00",
-                            }
-                        ]
-                    },
-                    {
-                        "structure": "child",
-                        "expires": "2017-06-01T12:00:00Z",
-                        "attribute_values": [
-                            {
-                                "name": "certificate_type",
-                                "value": "credit"
-                            },
-                            {
-                                "name": "credit_hours",
-                                "value": 2
-                            },
-                            {
-                                "name": "credit_provider",
-                                "value": "acme"
-                            },
-                            {
-                                "name": "verification_required",
-                                "value": False
-                            },
-                        ],
-                        "stockrecords": [
-                            {
-                                "price_currency": "USD",
-                                "price_excl_tax": "250.00",
-                            }
-                        ]
-                    }
-                ]
-            },
-            {  # Course with a currency not found in the database
-                "id": course_run_no_currency.key,
-                "products": [
-                    {
-                        "structure": "parent",
-                        "price": None,
-                        "expires": None,
-                        "attribute_values": [],
-                        "is_available_to_buy": False,
-                        "stockrecords": []
-                    },
-                    {
-                        "structure": "child",
-                        "expires": None,
-                        "attribute_values": [],
-                        "stockrecords": [
-                            {
-                                "price_currency": "123",
-                                "price_excl_tax": "0.00",
-                            }
-                        ]
-                    }
-                ]
-            },
-            {  # Course which does not exist in LMS
-                "id": "fake-course-does-not-exist",
-                "products": [
-                    {
-                        "structure": "parent",
-                        "price": None,
-                        "expires": None,
-                        "attribute_values": [],
-                        "is_available_to_buy": False,
-                        "stockrecords": []
-                    },
-                    {
-                        "structure": "child",
-                        "expires": None,
-                        "attribute_values": [],
-                        "stockrecords": [
-                            {
-                                "price_currency": "USD",
-                                "price_excl_tax": "0.00",
-                            }
-                        ]
-                    }
-                ]
-            }
-        ]
-
-        def courses_api_callback(url, data):
-            def request_callback(request):
-                # pylint: disable=redefined-builtin
-                next = None
-                count = len(bodies)
-
-                # Use the querystring to determine which page should be returned. Default to page 1.
-                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
-                qs = parse_qs(urlparse(request.path_url).query)
-                page = int(qs.get('page', [1])[0])
-
-                if page < count:
-                    next = '{}?page={}'.format(url, page)
-
-                body = {
-                    'count': count,
-                    'next': next,
-                    'previous': None,
-                    'results': [data[page - 1]]
-                }
-
-                return 200, {}, json.dumps(body)
-
-            return request_callback
-
-        url = '{host}/courses/'.format(host=settings.ECOMMERCE_API_URL)
-        responses.add_callback(responses.GET, url, callback=courses_api_callback(url, bodies), content_type=JSON)
-
+        audit_run = CourseRunFactory(title_override='audit', key='audit/course/run')
+        verified_run = CourseRunFactory(title_override='verified', key='verified/course/run')
+        credit_run = CourseRunFactory(title_override='credit', key='credit/course/run')
+        no_currency_run = CourseRunFactory(title_override='no currency', key='nocurrency/course/run')
+
+        SeatFactory(course_run=audit_run, type=Seat.PROFESSIONAL)
+        SeatFactory(course_run=verified_run, type=Seat.PROFESSIONAL)
+        SeatFactory(course_run=credit_run, type=Seat.PROFESSIONAL)
+        SeatFactory(course_run=no_currency_run, type=Seat.PROFESSIONAL)
+
+        bodies = mock_data.ECOMMERCE_API_BODIES
+        url = self.partner.ecommerce_api_url + 'courses/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies),
+            content_type=JSON
+        )
         return bodies
 
     def assert_seats_loaded(self, body):
         """ Assert a Seat corresponding to the specified data body was properly loaded into the database. """
-
         course_run = CourseRun.objects.get(key=body['id'])
         products = [p for p in body['products'] if p['structure'] == 'child']
         # Verify that the old seat is removed
@@ -1042,9 +540,9 @@ class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase):
     @responses.activate
     def test_ingest(self):
         """ Verify the method ingests data from the E-Commerce API. """
-        data = self.mock_api()
-        loaded_course_run_data = data[:-1]
-        loaded_seat_data = data[:-2]
+        api_data = self.mock_api()
+        loaded_course_run_data = api_data[:-1]
+        loaded_seat_data = api_data[:-2]
 
         self.assertEqual(CourseRun.objects.count(), len(loaded_course_run_data))
 
@@ -1055,8 +553,7 @@ class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase):
         self.loader.ingest()
 
         # Verify the API was called with the correct authorization header
-        expected_num_course_runs = len(data)
-        self.assert_api_called(expected_num_course_runs)
+        self.assert_api_called(1)
 
         for datum in loaded_seat_data:
             self.assert_seats_loaded(datum)
@@ -1085,80 +582,18 @@ class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
 
 @ddt.ddt
-@override_settings(PROGRAMS_API_URL=PROGRAMS_API_URL)
 class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
-    api_url = PROGRAMS_API_URL
     loader_class = ProgramsApiDataLoader
 
     def mock_api(self):
-        bodies = [
-            {
-                'uuid': 'd9ee1a73-d82d-4ed7-8eb1-80ea2b142ad6',
-                'id': 1,
-                'name': 'Water Management',
-                'subtitle': 'Explore water management concepts and technologies',
-                'category': 'xseries',
-                'status': 'active',
-                'marketing_slug': 'water-management',
-                'organizations': [
-                    {
-                        'display_name': 'Delft University of Technology',
-                        'key': 'DelftX'
-                    }
-                ],
-                'banner_image_urls': {
-                    'w1440h480': 'https://example.com/delft-water__1440x480.jpg',
-                    'w348h116': 'https://example.com/delft-water__348x116.jpg',
-                    'w726h242': 'https://example.com/delft-water__726x242.jpg',
-                    'w435h145': 'https://example.com/delft-water__435x145.jpg'
-                }
-            },
-            {
-                'uuid': 'b043f467-5e80-4225-93d2-248a93a8556a',
-                'id': 2,
-                'name': 'Supply Chain Management',
-                'subtitle': 'Learn how to design and optimize the supply chain to enhance business performance.',
-                'category': 'xseries',
-                'status': 'active',
-                'marketing_slug': 'supply-chain-management-0',
-                'organizations': [
-                    {
-                        'display_name': 'Massachusetts Institute of Technology',
-                        'key': 'MITx'
-                    }
-                ],
-                'banner_image_urls': {},
-            },
-        ]
-
-        def programs_api_callback(url, data):
-            def request_callback(request):
-                # pylint: disable=redefined-builtin
-                next = None
-                count = len(bodies)
-
-                # Use the querystring to determine which page should be returned. Default to page 1.
-                # Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
-                qs = parse_qs(urlparse(request.path_url).query)
-                page = int(qs.get('page', [1])[0])
-
-                if page < count:
-                    next = '{}?page={}'.format(url, page)
-
-                body = {
-                    'count': count,
-                    'next': next,
-                    'previous': None,
-                    'results': [data[page - 1]]
-                }
-
-                return 200, {}, json.dumps(body)
-
-            return request_callback
-
-        url = '{host}/programs/'.format(host=self.api_url)
-        responses.add_callback(responses.GET, url, callback=programs_api_callback(url, bodies), content_type=JSON)
-
+        bodies = mock_data.PROGRAMS_API_BODIES
+        url = self.partner.programs_api_url + 'programs/'
+        responses.add_callback(
+            responses.GET,
+            url,
+            callback=mock_api_callback(url, bodies),
+            content_type=JSON
+        )
         return bodies
 
     def assert_program_loaded(self, body):
@@ -1176,23 +611,26 @@ class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
 
         image_url = body.get('banner_image_urls', {}).get('w435h145')
         if image_url:
-            image = Image.objects.get(src=image_url, width=self.loader_class.image_width,
-                                      height=self.loader_class.image_height)
+            image = Image.objects.get(src=image_url, width=self.loader.image_width,
+                                      height=self.loader.image_height)
             self.assertEqual(program.image, image)
 
     @responses.activate
     def test_ingest(self):
         """ Verify the method ingests data from the Organizations API. """
-        data = self.mock_api()
+        api_data = self.mock_api()
         self.assertEqual(Program.objects.count(), 0)
 
         self.loader.ingest()
 
-        expected_num_programs = len(data)
-        self.assert_api_called(expected_num_programs)
+        # Verify the API was called with the correct authorization header
+        self.assert_api_called(1)
+
+        # Verify the Programs were created correctly
+        expected_num_programs = len(api_data)
         self.assertEqual(Program.objects.count(), expected_num_programs)
 
-        for datum in data:
+        for datum in api_data:
             self.assert_program_loaded(datum)
 
         self.loader.ingest()

course_discovery/apps/course_metadata/tests/test_models.py

@@ -3,8 +3,8 @@ import datetime
 import ddt
 import mock
 import pytz
+
 from dateutil.parser import parse
-from django.conf import settings
 from django.db import IntegrityError
 from django.test import TestCase
 from freezegun import freeze_time
@@ -278,7 +278,7 @@ class ProgramTests(TestCase):
 
     def test_marketing_url(self):
         """ Verify the property creates a complete marketing URL. """
-        expected = '{root}/{category}/{slug}'.format(root=settings.MARKETING_URL_ROOT.strip('/'),
+        expected = '{root}/{category}/{slug}'.format(root=self.program.partner.marketing_url_root.strip('/'),
                                                      category=self.program.category, slug=self.program.marketing_slug)
         self.assertEqual(self.program.marketing_url, expected)
 

course_discovery/settings/base.py

@@ -354,13 +354,10 @@ HAYSTACK_CONNECTIONS = {
 
 HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 
-COURSES_API_URL = 'http://127.0.0.1:8000/api/courses/v1/'
-ECOMMERCE_API_URL = 'http://127.0.0.1:8002/api/v2/'
-ORGANIZATIONS_API_URL = 'http://127.0.0.1:8000/api/organizations/v0/'
-PROGRAMS_API_URL = 'http://127.0.0.1:8003/api/v1/'
-MARKETING_API_URL = 'http://example.org/api/catalog/v2/'
-MARKETING_URL_ROOT = 'http://example.org/'
 
 COMPRESS_PRECOMPILERS = (
     ('text/x-scss', 'django_libsass.SassCompiler'),
 )
+
+
+DEFAULT_PARTNER_ID = None

course_discovery/settings/test.py

@@ -42,3 +42,5 @@ JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
 EDX_DRF_EXTENSIONS = {
     'OAUTH2_USER_INFO_URL': 'http://example.com/oauth2/user_info',
 }
+
+DEFAULT_PARTNER_ID = 1