Skip to content

C
course-discovery
Commit 5e8c2ad7 by Clinton Blackburn Committed by GitHub
Options

Added new marketing site data loader (#193) 

This will eventually replace the existing Drupal data loader. A more extensive API is now used to get much more data.

ECOM-5099
parent a0dbf812
Showing with 518 additions and 30 deletions
course_discovery/apps/core/admin.py
...@@ -53,7 +53,8 @@ class PartnerAdmin(admin.ModelAdmin): ...@@ -53,7 +53,8 @@ class PartnerAdmin(admin.ModelAdmin):
}), }),
(_('Marketing Site Configuration'), { (_('Marketing Site Configuration'), {
'description': _('Configure the marketing site URLs that will be used to retrieve data and create URLs.'), 'description': _('Configure the marketing site URLs that will be used to retrieve data and create URLs.'),
'fields': ('marketing_site_url_root', 'marketing_site_api_url',) 'fields': ('marketing_site_url_root', 'marketing_site_api_url', 'marketing_site_api_username',
'marketing_site_api_password',)
}), }),
) )
list_display = ('name', 'short_code',) list_display = ('name', 'short_code',)
......
course_discovery/apps/core/migrations/0010_auto_20160731_0023.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0009_auto_20160730_2131'),
]
operations = [
migrations.AddField(
model_name='partner',
name='marketing_site_api_password',
field=models.CharField(verbose_name='Marketing Site API Password', blank=True, max_length=255, null=True),
),
migrations.AddField(
model_name='partner',
name='marketing_site_api_username',
field=models.CharField(verbose_name='Marketing Site API Username', blank=True, max_length=255, null=True),
),
]
course_discovery/apps/core/models.py
...@@ -70,6 +70,10 @@ class Partner(TimeStampedModel): ...@@ -70,6 +70,10 @@ class Partner(TimeStampedModel):
verbose_name=_('Marketing Site API URL')) verbose_name=_('Marketing Site API URL'))
marketing_site_url_root = models.URLField(max_length=255, null=True, blank=True, marketing_site_url_root = models.URLField(max_length=255, null=True, blank=True,
verbose_name=_('Marketing Site URL')) verbose_name=_('Marketing Site URL'))
marketing_site_api_username = models.CharField(max_length=255, null=True, blank=True,
verbose_name=_('Marketing Site API Username'))
marketing_site_api_password = models.CharField(max_length=255, null=True, blank=True,
verbose_name=_('Marketing Site API Password'))
oidc_url_root = models.CharField(max_length=255, null=True, verbose_name=_('OpenID Connect URL')) oidc_url_root = models.CharField(max_length=255, null=True, verbose_name=_('OpenID Connect URL'))
oidc_key = models.CharField(max_length=255, null=True, verbose_name=_('OpenID Connect Key')) oidc_key = models.CharField(max_length=255, null=True, verbose_name=_('OpenID Connect Key'))
oidc_secret = models.CharField(max_length=255, null=True, verbose_name=_('OpenID Connect Secret')) oidc_secret = models.CharField(max_length=255, null=True, verbose_name=_('OpenID Connect Secret'))
......
course_discovery/apps/core/tests/factories.py
...@@ -27,6 +27,8 @@ class PartnerFactory(factory.DjangoModelFactory): ...@@ -27,6 +27,8 @@ class PartnerFactory(factory.DjangoModelFactory):
programs_api_url = '{root}/api/programs/v1/'.format(root=FuzzyUrlRoot().fuzz()) programs_api_url = '{root}/api/programs/v1/'.format(root=FuzzyUrlRoot().fuzz())
marketing_site_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz()) marketing_site_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
marketing_site_url_root = '{root}/'.format(root=FuzzyUrlRoot().fuzz()) marketing_site_url_root = '{root}/'.format(root=FuzzyUrlRoot().fuzz())
marketing_site_api_username = FuzzyText().fuzz()
marketing_site_api_password = FuzzyText().fuzz()
oidc_url_root = '{root}'.format(root=FuzzyUrlRoot().fuzz()) oidc_url_root = '{root}'.format(root=FuzzyUrlRoot().fuzz())
oidc_key = FuzzyText().fuzz() oidc_key = FuzzyText().fuzz()
oidc_secret = FuzzyText().fuzz() oidc_secret = FuzzyText().fuzz()
......
course_discovery/apps/course_metadata/data_loaders.py
...@@ -2,9 +2,10 @@ ...@@ -2,9 +2,10 @@
import abc import abc
import logging import logging
from decimal import Decimal from decimal import Decimal
from urllib.parse import urljoin from urllib.parse import urljoin, urlencode
import html2text import html2text
import requests
from dateutil.parser import parse from dateutil.parser import parse
from django.utils.functional import cached_property from django.utils.functional import cached_property
from edx_rest_api_client.client import EdxRestApiClient from edx_rest_api_client.client import EdxRestApiClient
...@@ -33,7 +34,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta): ...@@ -33,7 +34,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
PAGE_SIZE = 50 PAGE_SIZE = 50
SUPPORTED_TOKEN_TYPES = ('bearer', 'jwt',) SUPPORTED_TOKEN_TYPES = ('bearer', 'jwt',)
def __init__(self, partner, api_url, access_token, token_type): def __init__(self, partner, api_url, access_token=None, token_type=None):
""" """
Arguments: Arguments:
partner (Partner): Partner which owns the APIs and data being loaded partner (Partner): Partner which owns the APIs and data being loaded
...@@ -41,6 +42,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta): ...@@ -41,6 +42,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
access_token (str): OAuth2 access token access_token (str): OAuth2 access token
token_type (str): The type of access token passed in (e.g. Bearer, JWT) token_type (str): The type of access token passed in (e.g. Bearer, JWT)
""" """
if token_type:
token_type = token_type.lower() token_type = token_type.lower()
if token_type not in self.SUPPORTED_TOKEN_TYPES: if token_type not in self.SUPPORTED_TOKEN_TYPES:
...@@ -49,7 +51,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta): ...@@ -49,7 +51,7 @@ class AbstractDataLoader(metaclass=abc.ABCMeta):
self.access_token = access_token self.access_token = access_token
self.token_type = token_type self.token_type = token_type
self.partner = partner self.partner = partner
self.api_url = api_url self.api_url = api_url.strip('/')
@cached_property @cached_property
def api_client(self): def api_client(self):
...@@ -573,3 +575,101 @@ class ProgramsApiDataLoader(AbstractDataLoader): ...@@ -573,3 +575,101 @@ class ProgramsApiDataLoader(AbstractDataLoader):
image, __ = Image.objects.update_or_create(src=image_url, defaults=defaults) image, __ = Image.objects.update_or_create(src=image_url, defaults=defaults)
return image return image
class MarketingSiteDataLoader(AbstractDataLoader):
def __init__(self, partner, api_url, access_token=None, token_type=None):
super(MarketingSiteDataLoader, self).__init__(partner, api_url, access_token, token_type)
if not (self.partner.marketing_site_api_username and self.partner.marketing_site_api_password):
msg = 'Marketing Site API credentials are not properly configured for Partner [{partner}]!'.format(
partner=partner.short_code)
raise Exception(msg)
@cached_property
def api_client(self):
username = self.partner.marketing_site_api_username
# Login by posting to the login form
login_data = {
'name': username,
'pass': self.partner.marketing_site_api_password,
'form_id': 'user_login',
'op': 'Log in',
}
session = requests.Session()
login_url = '{root}/user'.format(root=self.api_url)
response = session.post(login_url, data=login_data)
expected_url = '{root}/users/{username}'.format(root=self.api_url, username=username)
if not (response.status_code == 200 and response.url == expected_url):
raise Exception('Login failed!')
return session
def ingest(self): # pragma: no cover
""" Load data for all supported objects (e.g. courses, runs). """
# TODO Ingest schools
# TODO Ingest instructors
# TODO Ingest course runs (courses)
self.retrieve_and_ingest_node_type('xseries', self.update_xseries)
def retrieve_and_ingest_node_type(self, node_type, update_method):
"""
Retrieves all nodes of the specified type, and calls `update_method` for each node.
Args:
node_type (str): Type of node to retrieve (e.g. course, xseries, school, instructor)
update_method: Method to which the retrieved data should be passed.
"""
page = 0
while page is not None and page >= 0:
kwargs = {
'type': node_type,
'max-depth': 2,
'load-entity-refs': 'subject,file,taxonomy_term,taxonomy_vocabulary,node,field_collection_item',
'page': page,
}
qs = urlencode(kwargs)
url = '{root}/node.json?{qs}'.format(root=self.api_url, qs=qs)
response = self.api_client.get(url)
status_code = response.status_code
if status_code is not 200:
msg = 'Failed to retrieve data from {url}\nStatus Code: {status}\nBody: {body}'.format(
url=url, status=status_code, body=response.content)
logger.error(msg)
raise Exception(msg)
data = response.json()
for datum in data['list']:
try:
url = datum['url']
datum = self.clean_strings(datum)
update_method(datum)
except: # pylint: disable=bare-except
logger.exception('Failed to load %s.', url)
if 'next' in data:
page += 1
else:
break
def update_xseries(self, data):
marketing_slug = data['url'].split('/')[-1]
card_image_url = data.get('field_card_image', {}).get('url')
defaults = {
'title': data['title'],
'subtitle': data.get('field_xseries_subtitle_short'),
'category': 'XSeries',
'partner': self.partner,
}
if card_image_url:
card_image, __ = Image.objects.get_or_create(src=card_image_url)
defaults['image'] = card_image
Program.objects.update_or_create(marketing_slug=marketing_slug, defaults=defaults)
course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py
...@@ -5,7 +5,8 @@ from edx_rest_api_client.client import EdxRestApiClient ...@@ -5,7 +5,8 @@ from edx_rest_api_client.client import EdxRestApiClient
from course_discovery.apps.core.models import Partner from course_discovery.apps.core.models import Partner
from course_discovery.apps.course_metadata.data_loaders import ( from course_discovery.apps.course_metadata.data_loaders import (
CoursesApiDataLoader, DrupalApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader CoursesApiDataLoader, DrupalApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader,
ProgramsApiDataLoader, MarketingSiteDataLoader
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -80,6 +81,7 @@ class Command(BaseCommand): ...@@ -80,6 +81,7 @@ class Command(BaseCommand):
(partner.ecommerce_api_url, EcommerceApiDataLoader,), (partner.ecommerce_api_url, EcommerceApiDataLoader,),
(partner.marketing_site_api_url, DrupalApiDataLoader,), (partner.marketing_site_api_url, DrupalApiDataLoader,),
(partner.programs_api_url, ProgramsApiDataLoader,), (partner.programs_api_url, ProgramsApiDataLoader,),
(partner.marketing_site_url_root, MarketingSiteDataLoader,),
) )
for api_url, loader_class in data_loaders: for api_url, loader_class in data_loaders:
......
course_discovery/apps/course_metadata/migrations/0010_auto_20160731_0226.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('course_metadata', '0009_auto_20160725_1751'),
]
operations = [
migrations.AlterField(
model_name='program',
name='marketing_slug',
field=models.CharField(db_index=True, blank=True, help_text='Slug used to generate links to the marketing site', max_length=255),
),
]
course_discovery/apps/course_metadata/models.py
...@@ -490,7 +490,8 @@ class Program(TimeStampedModel): ...@@ -490,7 +490,8 @@ class Program(TimeStampedModel):
marketing_slug = models.CharField( marketing_slug = models.CharField(
help_text=_('Slug used to generate links to the marketing site'), help_text=_('Slug used to generate links to the marketing site'),
blank=True, blank=True,
max_length=255 max_length=255,
db_index=True
) )
image = models.ForeignKey(Image, default=None, null=True, blank=True) image = models.ForeignKey(Image, default=None, null=True, blank=True)
......
course_discovery/apps/course_metadata/tests/mock_data.py
...@@ -492,3 +492,210 @@ PROGRAMS_API_BODIES = [ ...@@ -492,3 +492,210 @@ PROGRAMS_API_BODIES = [
'banner_image_urls': {}, 'banner_image_urls': {},
}, },
] ]
MARKETING_SITE_API_XSERIES_BODIES = [
{
'field_course_effort': 'self-paced: 3 hours per week',
'body': {
'value': '<p>The Astrophysics XSeries Program consists of four foundational courses in astrophysics taught '
'by prestigious leaders in the field, including Nobel Prize winners. You will be taught by Brian '
'Schmidt, who led the team that discovered dark energy – work which won him the 2011 Nobel Prize '
'for Physics, and by prize-winning educator, science communicator and astrophysics researcher '
'Paul Francis, who will take you through an incredible journey where you learn about the unsolved '
'mysteries of the universe, exoplanets, black holes and supernovae, and general cosmology. '
'Astronomy and astrophysics is the study of everything beyond Earth. Astronomers work in '
'universities, at observatories, for various space agencies like NASA, and more. The study of '
'astronomy provides you with a wide range of skills in math, engineering, and computation which '
'are sought after skills across many occupations. This XSeries Program is great for anyone to '
'start their studies in astronomy and astrophysics or individuals simply interested in what lies '
'beyond Earth.</p>',
'summary': '',
'format': 'standard_html'
},
'field_xseries_banner_image': {
'fid': '65336',
'name': 'aat075a_72.jpg',
'mime': 'image/jpeg',
'size': '146765',
'url': 'https://stage.edx.org/sites/default/files/xseries/image/banner/aat075a_72.jpg',
'timestamp': '1438027131',
'owner': {
'uri': 'https://stage.edx.org/user/9761',
'id': '9761',
'resource': 'user',
'uuid': '4af80bce-a315-4ea2-8eb2-a65d03014673'
},
'uuid': 'd2a87930-2d6a-4f2b-867b-8711d981404a'
},
'field_course_level': 'Intermediate',
'field_xseries_institutions': [
{
'field_school_description': {
'value': '<p>The Australian National University (ANU) is a celebrated place of intensive '
'research, education and policy engagement. Our research has always been central to '
'everything we do, shaping a holistic learning experience that goes beyond the classroom, '
'giving students access to researchers who are among the best in their fields and to '
'opportunities for development around Australia and the world.</p>',
'format': 'standard_html'
},
'field_school_name': 'Australian National University',
'field_school_image_banner': {
'fid': '31524',
'name': 'anu-home-banner.jpg',
'mime': 'image/jpeg',
'size': '30181',
'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu-home-banner_0.jpg',
'timestamp': '1384283150',
'owner': {
'uri': 'https://stage.edx.org/user/1',
'id': '1',
'resource': 'user',
'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
},
'uuid': 'f7fca9c1-078b-45bd-b4c9-ae5a927ba632'
},
'field_school_image_logo': {
'fid': '31526',
'name': 'anu_logo_200x101.png',
'mime': 'image/png',
'size': '13977',
'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu_logo_200x101_0.png',
'timestamp': '1384283150',
'owner': {
'uri': 'https://stage.edx.org/user/1',
'id': '1',
'resource': 'user',
'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
},
'uuid': '74a40d7e-e81f-4de0-9733-04ca12d25605'
},
'field_school_image_logo_thumb': {
'fid': '31525',
'name': 'anu_logo_185x48.png',
'mime': 'image/png',
'size': '2732',
'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu_logo_185x48_0.png',
'timestamp': '1384283150',
'owner': {
'uri': 'https://stage.edx.org/user/1',
'id': '1',
'resource': 'user',
'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
},
'uuid': '14fbc10e-c6a8-499f-a53c-032f92c9da32'
},
'field_school_image_logo_sub': {
'fid': '31527',
'name': 'anu-on-edx-logo.png',
'mime': 'image/png',
'size': '4517',
'url': 'https://stage.edx.org/sites/default/files/school/image/banner/anu-on-edx-logo_0.png',
'timestamp': '1384283150',
'owner': {
'uri': 'https://stage.edx.org/user/1',
'id': '1',
'resource': 'user',
'uuid': '434dea4f-7b93-4cba-9965-fe4856062a4f'
},
'uuid': 'ea74abe3-66ce-48ba-bf6d-34b2e109fbeb'
},
'field_school_description_private': [],
'field_school_subdomain_prefix': None,
'field_school_url_slug': 'anux',
'field_school_is_school': True,
'field_school_is_partner': False,
'field_school_is_contributor': True,
'field_school_is_charter': True,
'field_school_is_founder': False,
'field_school_is_display': True,
'field_school_freeform': [],
'field_school_is_affiliate': False,
'field_school_display_name': None,
'field_school_catalog_heading': None,
'field_school_catalog_subheading': None,
'field_school_subtitle': None,
'nid': '635',
'vid': '7917',
'is_new': False,
'type': 'school',
'title': 'ANUx',
'language': 'und',
'url': 'https://stage.edx.org/school/anux',
'edit_url': 'https://stage.edx.org/node/635/edit',
'status': '1',
'promote': '0',
'sticky': '0',
'created': '1384283059',
'changed': '1426706369',
'author': {
'uri': 'https://stage.edx.org/user/143',
'id': '143',
'resource': 'user',
'uuid': '8ed4adee-6f84-4bec-8b64-20f9bfe7af0c'
},
'log': 'Updated by FeedsNodeProcessor',
'revision': None,
'body': [],
'uuid': '1e6df8ed-a3fe-4307-99b9-775af509fcba',
'vuuid': '98f08316-2d87-4412-8e03-838fa94a7f03'
}
],
'field_card_image': {
'fid': '65346',
'name': 'anu_astrophys_xseries_card.jpg',
'mime': 'image/jpeg',
'size': '53246',
'url': 'https://stage.edx.org/sites/default/files/card/images/anu_astrophys_xseries_card.jpg',
'timestamp': '1438043010',
'owner': {
'uri': 'https://stage.edx.org/user/9761',
'id': '9761',
'resource': 'user',
'uuid': '4af80bce-a315-4ea2-8eb2-a65d03014673'
},
'uuid': '820b05ad-1283-47ab-a123-6a7a17868a37'
},
'field_xseries_length': 'self-paced: ~9 weeks per course',
'field_xseries_overview': {
'value': '<h3>What You\'ll Learn</h3> <ul><li>An understanding of the biggest unsolved mysteries in '
'astrophysics and how researchers are attempting to answer them</li> <li>Methods used to find '
'and study exoplanets</li> <li>How scientists tackle challenging problems</li> <li>About white '
'dwarfs, novae, supernovae, neutro stars and black holes and how quantum mechanics and relativity '
'help explain these objects</li> <li>How astrophysicists investigate the origin, nature and fate '
'of our universe</li> </ul>',
'format': 'expanded_html'
},
'field_xseries_price': '$50/Course',
'field_xseries_subtitle': 'Learn contemporary astrophysics from the leaders in the field.',
'field_xseries_subtitle_short': 'Learn contemporary astrophysics from the leaders in the field.',
'field_xseries_outcome': None,
'field_xseries_required_weeks': None,
'field_xseries_required_hours': None,
'nid': '7046',
'vid': '130386',
'type': 'xseries',
'title': 'Astrophysics',
'language': 'und',
'url': 'https://stage.edx.org/xseries/astrophysics'
},
{
'body': {
'value': '<p>In this XSeries, you will find all of the content required to be successful on the AP '
'Biology exam including genetics, the cell, ecology, diversity and evolution. You will also '
'find practice AP-style multiple choice and free response questions, tutorials on how to '
'formulate great responses and lab experiences that will be crucial to your success on the AP '
'exam.<br /> </p> <p><span>This XSeries consists of 5 courses.</span> The cost is $25 per '
'course. The total cost of this XSeries is $125. The component courses for this XSeries may be '
'taken individually.</p>',
'summary': '',
'format': 'standard_html'
},
'field_xseries_banner_image': {
'url': 'https://stage.edx.org/sites/default/files/xseries/image/banner/ap-biology-exam.jpg'
},
'field_xseries_subtitle_short': 'Learn Biology!',
'type': 'xseries',
'title': 'Biology',
'url': 'https://stage.edx.org/xseries/biology'
},
]
course_discovery/apps/course_metadata/tests/test_data_loaders.py
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
import datetime import datetime
import json import json
from decimal import Decimal from decimal import Decimal
from urllib.parse import parse_qs
from urllib.parse import urlparse
import ddt import ddt
import mock import mock
...@@ -15,7 +17,7 @@ from pytz import UTC ...@@ -15,7 +17,7 @@ from pytz import UTC
from course_discovery.apps.core.tests.utils import mock_api_callback from course_discovery.apps.core.tests.utils import mock_api_callback
from course_discovery.apps.course_metadata.data_loaders import ( from course_discovery.apps.course_metadata.data_loaders import (
OrganizationsApiDataLoader, CoursesApiDataLoader, DrupalApiDataLoader, EcommerceApiDataLoader, AbstractDataLoader, OrganizationsApiDataLoader, CoursesApiDataLoader, DrupalApiDataLoader, EcommerceApiDataLoader, AbstractDataLoader,
ProgramsApiDataLoader ProgramsApiDataLoader, MarketingSiteDataLoader
) )
from course_discovery.apps.course_metadata.models import ( from course_discovery.apps.course_metadata.models import (
Course, CourseOrganization, CourseRun, Image, LanguageTag, Organization, Person, Seat, Subject, Program Course, CourseOrganization, CourseRun, Image, LanguageTag, Organization, Person, Seat, Subject, Program
...@@ -66,8 +68,25 @@ class AbstractDataLoaderTest(TestCase): ...@@ -66,8 +68,25 @@ class AbstractDataLoaderTest(TestCase):
self.assertFalse(instance.__class__.objects.filter(pk=instance.pk).exists()) # pylint: disable=no-member self.assertFalse(instance.__class__.objects.filter(pk=instance.pk).exists()) # pylint: disable=no-member
# pylint: disable=not-callable
@ddt.ddt @ddt.ddt
class ApiClientTestMixin(object):
@ddt.unpack
@ddt.data(
('Bearer', BearerAuth),
('JWT', SuppliedJwtAuth),
)
def test_api_client(self, token_type, expected_auth_class):
""" Verify the property returns an API client with the correct authentication. """
loader = self.loader_class(self.partner, self.api_url, ACCESS_TOKEN, token_type)
client = loader.api_client
self.assertIsInstance(client, EdxRestApiClient)
# NOTE (CCB): My initial preference was to mock the constructor and ensure the correct auth arguments
# were passed. However, that seems nearly impossible. This is the next best alternative. It is brittle, and
# may break if we ever change the underlying request class of EdxRestApiClient.
self.assertIsInstance(client._store['session'].auth, expected_auth_class) # pylint: disable=protected-access
# pylint: disable=not-callable
class DataLoaderTestMixin(object): class DataLoaderTestMixin(object):
loader_class = None loader_class = None
partner = None partner = None
...@@ -98,24 +117,9 @@ class DataLoaderTestMixin(object): ...@@ -98,24 +117,9 @@ class DataLoaderTestMixin(object):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
self.loader_class(self.partner, self.api_url, ACCESS_TOKEN, 'not-supported') self.loader_class(self.partner, self.api_url, ACCESS_TOKEN, 'not-supported')
@ddt.unpack
@ddt.data(
('Bearer', BearerAuth),
('JWT', SuppliedJwtAuth),
)
def test_api_client(self, token_type, expected_auth_class):
""" Verify the property returns an API client with the correct authentication. """
loader = self.loader_class(self.partner, self.api_url, ACCESS_TOKEN, token_type)
client = loader.api_client
self.assertIsInstance(client, EdxRestApiClient)
# NOTE (CCB): My initial preference was to mock the constructor and ensure the correct auth arguments
# were passed. However, that seems nearly impossible. This is the next best alternative. It is brittle, and
# may break if we ever change the underlying request class of EdxRestApiClient.
self.assertIsInstance(client._store['session'].auth, expected_auth_class) # pylint: disable=protected-access
@ddt.ddt @ddt.ddt
class OrganizationsApiDataLoaderTests(DataLoaderTestMixin, TestCase): class OrganizationsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase):
loader_class = OrganizationsApiDataLoader loader_class = OrganizationsApiDataLoader
@property @property
...@@ -169,7 +173,7 @@ class OrganizationsApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -169,7 +173,7 @@ class OrganizationsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
@ddt.ddt @ddt.ddt
class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase): class CoursesApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase):
loader_class = CoursesApiDataLoader loader_class = CoursesApiDataLoader
@property @property
...@@ -308,7 +312,7 @@ class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -308,7 +312,7 @@ class CoursesApiDataLoaderTests(DataLoaderTestMixin, TestCase):
@ddt.ddt @ddt.ddt
class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase): class DrupalApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase):
loader_class = DrupalApiDataLoader loader_class = DrupalApiDataLoader
@property @property
...@@ -487,7 +491,7 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -487,7 +491,7 @@ class DrupalApiDataLoaderTests(DataLoaderTestMixin, TestCase):
@ddt.ddt @ddt.ddt
class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase): class EcommerceApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase):
loader_class = EcommerceApiDataLoader loader_class = EcommerceApiDataLoader
@property @property
...@@ -601,7 +605,7 @@ class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -601,7 +605,7 @@ class EcommerceApiDataLoaderTests(DataLoaderTestMixin, TestCase):
@ddt.ddt @ddt.ddt
class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase): class ProgramsApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase):
loader_class = ProgramsApiDataLoader loader_class = ProgramsApiDataLoader
@property @property
...@@ -652,10 +656,134 @@ class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase): ...@@ -652,10 +656,134 @@ class ProgramsApiDataLoaderTests(DataLoaderTestMixin, TestCase):
self.assert_api_called(1) self.assert_api_called(1)
# Verify the Programs were created correctly # Verify the Programs were created correctly
expected_num_programs = len(api_data) self.assertEqual(Program.objects.count(), len(api_data))
self.assertEqual(Program.objects.count(), expected_num_programs)
for datum in api_data: for datum in api_data:
self.assert_program_loaded(datum) self.assert_program_loaded(datum)
self.loader.ingest() self.loader.ingest()
class MarketingSiteDataLoaderTests(DataLoaderTestMixin, TestCase):
loader_class = MarketingSiteDataLoader
LOGIN_COOKIE = ('session_id', 'abc123')
@property
def api_url(self):
return self.partner.marketing_site_url_root
def mock_login_response(self, failure=False):
url = self.api_url + 'user'
landing_url = '{base}users/{username}'.format(base=self.api_url,
username=self.partner.marketing_site_api_username)
status = 500 if failure else 302
adding_headers = {}
if not failure:
adding_headers['Location'] = landing_url
responses.add(responses.POST, url, status=status, adding_headers=adding_headers)
responses.add(responses.GET, landing_url)
def mock_api_callback(self, url, data):
""" Paginate the data, one item per page. """
def request_callback(request):
count = len(data)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [0])[0])
page_size = 1
body = {
'list': [data[page]]
}
if (page * page_size) < count - 1:
next_page = page + 1
next_url = '{}?page={}'.format(url, next_page)
body['next'] = next_url
return 200, {}, json.dumps(body)
return request_callback
def mock_api(self):
bodies = mock_data.MARKETING_SITE_API_XSERIES_BODIES
url = self.api_url + 'node.json'
responses.add_callback(
responses.GET,
url,
callback=self.mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def mock_api_failure(self):
url = self.api_url + 'node.json'
responses.add(responses.GET, url, status=500)
def assert_program_loaded(self, data):
marketing_slug = data['url'].split('/')[-1]
program = Program.objects.get(marketing_slug=marketing_slug)
self.assertEqual(program.title, data['title'])
self.assertEqual(program.subtitle, data.get('field_xseries_subtitle_short'))
self.assertEqual(program.category, 'XSeries')
self.assertEqual(program.partner, self.partner)
card_image_url = data.get('field_card_image', {}).get('url')
if card_image_url:
card_image = Image.objects.get(src=card_image_url)
self.assertEqual(program.image, card_image)
else:
self.assertIsNone(program.image)
def test_constructor_without_credentials(self):
""" Verify the constructor raises an exception if the Partner has no marketing site credentials set. """
self.partner.marketing_site_api_username = None
with self.assertRaises(Exception):
self.loader_class(self.partner, self.api_url)
@responses.activate
def test_api_client_login_failure(self):
self.mock_login_response(failure=True)
with self.assertRaises(Exception):
self.loader.api_client # pylint: disable=pointless-statement
@responses.activate
def test_ingest(self):
self.mock_login_response()
api_data = self.mock_api()
self.assertEqual(Program.objects.count(), 0)
self.loader.ingest()
for datum in api_data:
self.assert_program_loaded(datum)
@responses.activate
def test_ingest_with_api_failure(self):
self.mock_login_response()
self.mock_api_failure()
with self.assertRaises(Exception):
self.loader.ingest()
@responses.activate
def test_ingest_exception_handling(self):
""" Verify the data loader properly handles exceptions during processing of the data from the API. """
self.mock_login_response()
api_data = self.mock_api()
with mock.patch.object(self.loader, 'clean_strings', side_effect=Exception):
with mock.patch('course_discovery.apps.course_metadata.data_loaders.logger') as mock_logger:
self.loader.ingest()
self.assertEqual(mock_logger.exception.call_count, len(api_data))
calls = [mock.call('Failed to load %s.', datum['url']) for datum in api_data]
mock_logger.exception.assert_has_calls(calls)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment