Commit 85d5a892 by Matt Drayer

mattdrayer/SOL-1929.2: Add Partner metadata model

parent c7179fce
......@@ -5,7 +5,7 @@ from django.contrib.auth.admin import UserAdmin
from django.utils.translation import ugettext_lazy as _
from course_discovery.apps.core.forms import UserThrottleRateForm
from course_discovery.apps.core.models import User, UserThrottleRate, Currency
from course_discovery.apps.core.models import User, UserThrottleRate, Currency, Partner
@admin.register(User)
......@@ -34,3 +34,10 @@ class CurrencyAdmin(admin.ModelAdmin):
list_display = ('code', 'name',)
ordering = ('code', 'name',)
search_fields = ('code', 'name',)
@admin.register(Partner)
class PartnerAdmin(admin.ModelAdmin):
list_display = ('name', 'short_code',)
ordering = ('name', 'short_code',)
search_fields = ('name', 'short_code',)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import django_extensions.db.fields
class Migration(migrations.Migration):
dependencies = [
('core', '0007_auto_20160510_2017'),
]
operations = [
migrations.CreateModel(
name='Partner',
fields=[
('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
('created', django_extensions.db.fields.CreationDateTimeField(verbose_name='created', auto_now_add=True)),
('modified', django_extensions.db.fields.ModificationDateTimeField(verbose_name='modified', auto_now=True)),
('name', models.CharField(max_length=128, unique=True)),
('short_code', models.CharField(max_length=8, unique=True)),
('courses_api_url', models.URLField(max_length=255, null=True)),
('ecommerce_api_url', models.URLField(max_length=255, null=True)),
('organizations_api_url', models.URLField(max_length=255, null=True)),
('programs_api_url', models.URLField(max_length=255, null=True)),
('marketing_api_url', models.URLField(max_length=255, null=True)),
('marketing_url_root', models.URLField(max_length=255, null=True)),
('social_auth_edx_oidc_url_root', models.CharField(max_length=255, null=True)),
('social_auth_edx_oidc_key', models.CharField(max_length=255, null=True)),
('social_auth_edx_oidc_secret', models.CharField(max_length=255, null=True)),
],
options={
'verbose_name': 'Partner',
'verbose_name_plural': 'Partners',
},
),
]
......@@ -3,6 +3,7 @@
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django_extensions.db.models import TimeStampedModel
from guardian.mixins import GuardianUserMixin
......@@ -53,3 +54,24 @@ class Currency(models.Model):
class Meta(object):
verbose_name_plural = 'Currencies'
class Partner(TimeStampedModel):
name = models.CharField(max_length=128, unique=True, null=False, blank=False)
short_code = models.CharField(max_length=8, unique=True, null=False, blank=False)
courses_api_url = models.URLField(max_length=255, null=True)
ecommerce_api_url = models.URLField(max_length=255, null=True)
organizations_api_url = models.URLField(max_length=255, null=True)
programs_api_url = models.URLField(max_length=255, null=True)
marketing_api_url = models.URLField(max_length=255, null=True)
marketing_url_root = models.URLField(max_length=255, null=True)
social_auth_edx_oidc_url_root = models.CharField(max_length=255, null=True)
social_auth_edx_oidc_key = models.CharField(max_length=255, null=True)
social_auth_edx_oidc_secret = models.CharField(max_length=255, null=True)
def __str__(self):
return '{name} ({code})'.format(name=self.name, code=self.short_code)
class Meta:
verbose_name = _('Partner')
verbose_name_plural = _('Partners')
import factory
from factory.fuzzy import FuzzyText
from course_discovery.apps.core.models import User
from course_discovery.apps.core.models import User, Partner
from course_discovery.apps.core.tests.utils import FuzzyUrlRoot
USER_PASSWORD = 'password'
......@@ -14,3 +16,20 @@ class UserFactory(factory.DjangoModelFactory):
class Meta:
model = User
class PartnerFactory(factory.DjangoModelFactory):
name = factory.Sequence(lambda n: 'test-partner-{}'.format(n)) # pylint: disable=unnecessary-lambda
short_code = factory.Sequence(lambda n: 'test{}'.format(n)) # pylint: disable=unnecessary-lambda
courses_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
ecommerce_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
organizations_api_url = '{root}/api/organizations/v1/'.format(root=FuzzyUrlRoot().fuzz())
programs_api_url = '{root}/api/programs/v1/'.format(root=FuzzyUrlRoot().fuzz())
marketing_api_url = '{root}/api/courses/v1/'.format(root=FuzzyUrlRoot().fuzz())
marketing_url_root = '{root}/'.format(root=FuzzyUrlRoot().fuzz())
social_auth_edx_oidc_url_root = '{root}'.format(root=FuzzyUrlRoot().fuzz())
social_auth_edx_oidc_key = FuzzyText().fuzz()
social_auth_edx_oidc_secret = FuzzyText().fuzz()
class Meta(object):
model = Partner
......@@ -3,7 +3,7 @@
from django.test import TestCase
from social.apps.django_app.default.models import UserSocialAuth
from course_discovery.apps.core.models import Currency
from course_discovery.apps.core.models import Currency, Partner
from course_discovery.apps.core.tests.factories import UserFactory
......@@ -54,3 +54,17 @@ class CurrencyTests(TestCase):
name = 'U.S. Dollar'
instance = Currency(code=code, name=name)
self.assertEqual(str(instance), '{code} - {name}'.format(code=code, name=name))
class PartnerTests(TestCase):
""" Tests for the Partner class. """
def test_str(self):
"""
Verify casting an instance to a string returns a string containing the name and short code of the partner.
"""
code = 'test'
name = 'Test Partner'
instance = Partner(name=name, short_code=code)
self.assertEqual(str(instance), '{name} ({code})'.format(name=name, code=code))
import json
from urllib.parse import parse_qs, urlparse
from factory.fuzzy import (
BaseFuzzyAttribute, FuzzyText, FuzzyChoice
)
class FuzzyDomain(BaseFuzzyAttribute):
def fuzz(self):
subdomain = FuzzyText()
domain = FuzzyText()
tld = FuzzyChoice(('com', 'net', 'org', 'biz', 'pizza', 'coffee', 'diamonds', 'fail', 'win', 'wtf',))
return "{subdomain}.{domain}.{tld}".format(
subdomain=subdomain.fuzz(),
domain=domain.fuzz(),
tld=tld.fuzz()
)
class FuzzyUrlRoot(BaseFuzzyAttribute):
def fuzz(self):
protocol = FuzzyChoice(('http', 'https',))
domain = FuzzyDomain()
return "{protocol}://{domain}".format(
protocol=protocol.fuzz(),
domain=domain.fuzz()
)
class FuzzyURL(BaseFuzzyAttribute):
def fuzz(self):
root = FuzzyUrlRoot()
resource = FuzzyText()
return "{root}/{resource}".format(
root=root.fuzz(),
resource=resource.fuzz()
)
def mock_api_callback(url, data, results_key=True, pagination=False):
def request_callback(request):
# pylint: disable=redefined-builtin
count = len(data)
next_url = None
previous_url = None
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
page_size = int(qs.get('page_size', [1])[0])
if (page * page_size) < count:
next_page = page + 1
next_url = '{}?page={}'.format(url, next_page)
if page > 1:
previous_page = page - 1
previous_url = '{}?page={}'.format(url, previous_page)
body = {
'count': count,
'next': next_url,
'previous': previous_url,
}
if pagination:
body = {
'pagination': body
}
if results_key:
body['results'] = data
else:
body.update(data)
return 200, {}, json.dumps(body)
return request_callback
import logging
from django.conf import settings
from django.core.management import BaseCommand, CommandError
from edx_rest_api_client.client import EdxRestApiClient
from course_discovery.apps.course_metadata.data_loaders import (
CoursesApiDataLoader, DrupalApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader
)
from course_discovery.apps.core.models import Partner
logger = logging.getLogger(__name__)
......@@ -31,38 +31,69 @@ class Command(BaseCommand):
help='The type of access token being passed (e.g. Bearer, JWT).'
)
def handle(self, *args, **options):
access_token = options.get('access_token')
token_type = options.get('token_type')
if access_token and not token_type:
raise CommandError('The token_type must be specified when passing in an access token!')
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
token_type = 'JWT'
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT),
settings.SOCIAL_AUTH_EDX_OIDC_KEY,
settings.SOCIAL_AUTH_EDX_OIDC_SECRET,
token_type=token_type
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
loaders = (
(OrganizationsApiDataLoader, settings.ORGANIZATIONS_API_URL,),
(CoursesApiDataLoader, settings.COURSES_API_URL,),
(EcommerceApiDataLoader, settings.ECOMMERCE_API_URL,),
(DrupalApiDataLoader, settings.MARKETING_API_URL,),
(ProgramsApiDataLoader, settings.PROGRAMS_API_URL,),
parser.add_argument(
'--partner_code',
action='store',
dest='partner_code',
default=None,
help='The short code for a specific partner to refresh.'
)
for loader_class, api_url in loaders:
try:
loader_class(api_url, access_token, token_type).ingest()
except Exception:
logger.exception('%s failed!', loader_class.__name__)
def handle(self, *args, **options):
# For each partner defined...
partners = Partner.objects.all()
# If a specific partner was indicated, filter down the set
partner_code = options.get('partner_code')
if partner_code:
partners = partners.filter(short_code=partner_code)
if not partners:
raise CommandError('No partners available!')
for partner in partners:
access_token = options.get('access_token')
token_type = options.get('token_type')
if access_token and not token_type:
raise CommandError('The token_type must be specified when passing in an access token!')
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
token_type = 'JWT'
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=partner.social_auth_edx_oidc_url_root.strip('/')),
partner.social_auth_edx_oidc_key,
partner.social_auth_edx_oidc_secret,
token_type=token_type
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
loaders = []
if partner.organizations_api_url:
loaders.append(OrganizationsApiDataLoader)
if partner.courses_api_url:
loaders.append(CoursesApiDataLoader)
if partner.ecommerce_api_url:
loaders.append(EcommerceApiDataLoader)
if partner.marketing_api_url:
loaders.append(DrupalApiDataLoader)
if partner.programs_api_url:
loaders.append(ProgramsApiDataLoader)
if loaders:
for loader_class in loaders:
try:
loader_class(
partner,
access_token,
token_type,
).ingest()
except Exception: # pylint: disable=broad-except
logger.exception('%s failed!', loader_class.__name__)
import json
import responses
from django.core.management import call_command, CommandError
from django.test import TestCase
from course_discovery.apps.core.tests.factories import PartnerFactory
from course_discovery.apps.core.tests.utils import mock_api_callback
from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Partner, Program
from course_discovery.apps.course_metadata.tests import mock_data
ACCESS_TOKEN = 'secret'
ACCESS_TOKEN_TYPE = 'Bearer'
JSON = 'application/json'
LOGGER_NAME = 'course_metadata.management.commands.refresh_course_metadata'
class RefreshCourseMetadataCommandTests(TestCase):
def setUp(self):
super(RefreshCourseMetadataCommandTests, self).setUp()
self.partner = PartnerFactory()
def mock_access_token_api(self):
body = {
'access_token': ACCESS_TOKEN,
'expires_in': 30
}
url = self.partner.social_auth_edx_oidc_url_root.strip('/') + '/access_token'
responses.add_callback(
responses.POST,
url,
callback=mock_api_callback(url, body, results_key=False),
content_type=JSON
)
return body
def mock_organizations_api(self):
bodies = mock_data.ORGANIZATIONS_API_BODIES
url = self.partner.organizations_api_url + 'organizations/'
responses.add_callback(
responses.GET,
url,
callback=mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def mock_lms_courses_api(self):
bodies = mock_data.COURSES_API_BODIES
url = self.partner.courses_api_url + 'courses/'
responses.add_callback(
responses.GET,
url,
callback=mock_api_callback(url, bodies, pagination=True),
content_type=JSON
)
return bodies
def mock_ecommerce_courses_api(self):
bodies = mock_data.ECOMMERCE_API_BODIES
url = self.partner.ecommerce_api_url + 'courses/'
responses.add_callback(
responses.GET,
url,
callback=mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def mock_marketing_courses_api(self):
"""Mock out the Marketing API. Returns a list of mocked-out course runs."""
body = mock_data.MARKETING_API_BODY
responses.add(
responses.GET,
self.partner.marketing_api_url + 'courses/',
body=json.dumps(body),
status=200,
content_type='application/json'
)
return body['items']
def mock_programs_api(self):
bodies = mock_data.PROGRAMS_API_BODIES
url = self.partner.programs_api_url + 'programs/'
responses.add_callback(
responses.GET,
url,
callback=mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
@responses.activate
def test_refresh_course_metadata(self):
""" Verify """
self.mock_access_token_api()
self.mock_organizations_api()
self.mock_lms_courses_api()
self.mock_ecommerce_courses_api()
self.mock_marketing_courses_api()
self.mock_programs_api()
call_command('refresh_course_metadata')
partners = Partner.objects.all()
self.assertEqual(len(partners), 1)
organizations = Organization.objects.all()
self.assertEqual(len(organizations), 3)
for organization in organizations:
self.assertEqual(organization.partner.short_code, self.partner.short_code)
courses = Course.objects.all()
self.assertEqual(len(courses), 2)
for course in courses:
self.assertEqual(course.partner.short_code, self.partner.short_code)
course_runs = CourseRun.objects.all()
self.assertEqual(len(course_runs), 3)
for course_run in course_runs:
self.assertEqual(course_run.course.partner.short_code, self.partner.short_code)
programs = Program.objects.all()
self.assertEqual(len(programs), 2)
for program in programs:
self.assertEqual(program.partner.short_code, self.partner.short_code)
# Refresh only a specific partner
command_args = ['--partner_code={0}'.format(partners[0].short_code)]
call_command('refresh_course_metadata', *command_args)
# Invalid partner code
with self.assertRaises(CommandError):
command_args = ['--partner_code=invalid']
call_command('refresh_course_metadata', *command_args)
# Access token but no token type
with self.assertRaises(CommandError):
command_args = ['--access_token=test-access-token']
call_command('refresh_course_metadata', *command_args)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('core', '0008_partner'),
('course_metadata', '0008_program_image'),
]
operations = [
migrations.AddField(
model_name='course',
name='partner',
field=models.ForeignKey(null=True, to='core.Partner'),
),
migrations.AddField(
model_name='historicalcourse',
name='partner',
field=models.ForeignKey(related_name='+', null=True, on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False, blank=True, to='core.Partner'),
),
migrations.AddField(
model_name='historicalorganization',
name='partner',
field=models.ForeignKey(related_name='+', null=True, on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False, blank=True, to='core.Partner'),
),
migrations.AddField(
model_name='organization',
name='partner',
field=models.ForeignKey(null=True, to='core.Partner'),
),
migrations.AddField(
model_name='program',
name='partner',
field=models.ForeignKey(null=True, to='core.Partner'),
),
]
......@@ -4,7 +4,6 @@ from urllib.parse import urljoin
from uuid import uuid4
import pytz
from django.conf import settings
from django.db import models
from django.db.models.query_utils import Q
from django.utils.translation import ugettext_lazy as _
......@@ -13,7 +12,7 @@ from haystack.query import SearchQuerySet
from simple_history.models import HistoricalRecords
from sortedm2m.fields import SortedManyToManyField
from course_discovery.apps.core.models import Currency
from course_discovery.apps.core.models import Currency, Partner
from course_discovery.apps.course_metadata.query import CourseQuerySet
from course_discovery.apps.course_metadata.utils import clean_query
from course_discovery.apps.ietf_language_tags.models import LanguageTag
......@@ -132,6 +131,7 @@ class Organization(TimeStampedModel):
description = models.TextField(null=True, blank=True)
homepage_url = models.URLField(max_length=255, null=True, blank=True)
logo_image = models.ForeignKey(Image, null=True, blank=True)
partner = models.ForeignKey(Partner, null=True, blank=False)
history = HistoricalRecords()
......@@ -189,6 +189,7 @@ class Course(TimeStampedModel):
history = HistoricalRecords()
objects = CourseQuerySet.as_manager()
partner = models.ForeignKey(Partner, null=True, blank=False)
@property
def owners(self):
......@@ -496,6 +497,8 @@ class Program(TimeStampedModel):
organizations = models.ManyToManyField(Organization, blank=True)
partner = models.ForeignKey(Partner, null=True, blank=False)
def __str__(self):
return self.title
......@@ -503,7 +506,7 @@ class Program(TimeStampedModel):
def marketing_url(self):
if self.marketing_slug:
path = '{category}/{slug}'.format(category=self.category, slug=self.marketing_slug)
return urljoin(settings.MARKETING_URL_ROOT, path)
return urljoin(self.partner.marketing_url_root, path)
return None
......
......@@ -3,10 +3,12 @@ from uuid import uuid4
import factory
from factory.fuzzy import (
BaseFuzzyAttribute, FuzzyText, FuzzyChoice, FuzzyDateTime, FuzzyInteger, FuzzyDecimal
FuzzyText, FuzzyChoice, FuzzyDateTime, FuzzyInteger, FuzzyDecimal
)
from pytz import UTC
from course_discovery.apps.core.tests.factories import PartnerFactory
from course_discovery.apps.core.tests.utils import FuzzyURL
from course_discovery.apps.core.models import Currency
from course_discovery.apps.course_metadata.models import (
Course, CourseRun, Organization, Person, Image, Video, Subject, Seat, Prerequisite, LevelType, Program,
......@@ -15,22 +17,6 @@ from course_discovery.apps.course_metadata.models import (
from course_discovery.apps.ietf_language_tags.models import LanguageTag
class FuzzyURL(BaseFuzzyAttribute):
def fuzz(self):
protocol = FuzzyChoice(('http', 'https',))
subdomain = FuzzyText()
domain = FuzzyText()
tld = FuzzyChoice(('com', 'net', 'org', 'biz', 'pizza', 'coffee', 'diamonds', 'fail', 'win', 'wtf',))
resource = FuzzyText()
return "{protocol}://{subdomain}.{domain}.{tld}/{resource}".format(
protocol=protocol.fuzz(),
subdomain=subdomain.fuzz(),
domain=domain.fuzz(),
tld=tld.fuzz(),
resource=resource.fuzz()
)
class AbstractMediaModelFactory(factory.DjangoModelFactory):
src = FuzzyURL()
description = FuzzyText()
......@@ -89,6 +75,7 @@ class CourseFactory(factory.DjangoModelFactory):
image = factory.SubFactory(ImageFactory)
video = factory.SubFactory(VideoFactory)
marketing_url = FuzzyText(prefix='https://example.com/test-course-url')
partner = factory.SubFactory(PartnerFactory)
class Meta:
model = Course
......@@ -123,6 +110,7 @@ class OrganizationFactory(factory.DjangoModelFactory):
description = FuzzyText()
homepage_url = FuzzyURL()
logo_image = factory.SubFactory(ImageFactory)
partner = factory.SubFactory(PartnerFactory)
class Meta:
model = Organization
......@@ -150,6 +138,7 @@ class ProgramFactory(factory.django.DjangoModelFactory):
status = 'unpublished'
marketing_slug = factory.Sequence(lambda n: 'test-slug-{}'.format(n)) # pylint: disable=unnecessary-lambda
image = factory.SubFactory(ImageFactory)
partner = factory.SubFactory(PartnerFactory)
class AbstractSocialNetworkModelFactory(factory.DjangoModelFactory):
......
......@@ -3,8 +3,8 @@ import datetime
import ddt
import mock
import pytz
from dateutil.parser import parse
from django.conf import settings
from django.db import IntegrityError
from django.test import TestCase
from freezegun import freeze_time
......@@ -278,7 +278,7 @@ class ProgramTests(TestCase):
def test_marketing_url(self):
""" Verify the property creates a complete marketing URL. """
expected = '{root}/{category}/{slug}'.format(root=settings.MARKETING_URL_ROOT.strip('/'),
expected = '{root}/{category}/{slug}'.format(root=self.program.partner.marketing_url_root.strip('/'),
category=self.program.category, slug=self.program.marketing_slug)
self.assertEqual(self.program.marketing_url, expected)
......
......@@ -354,13 +354,10 @@ HAYSTACK_CONNECTIONS = {
HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
COURSES_API_URL = 'http://127.0.0.1:8000/api/courses/v1/'
ECOMMERCE_API_URL = 'http://127.0.0.1:8002/api/v2/'
ORGANIZATIONS_API_URL = 'http://127.0.0.1:8000/api/organizations/v0/'
PROGRAMS_API_URL = 'http://127.0.0.1:8003/api/v1/'
MARKETING_API_URL = 'http://example.org/api/catalog/v2/'
MARKETING_URL_ROOT = 'http://example.org/'
COMPRESS_PRECOMPILERS = (
('text/x-scss', 'django_libsass.SassCompiler'),
)
DEFAULT_PARTNER_ID = None
......@@ -42,3 +42,5 @@ JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
EDX_DRF_EXTENSIONS = {
'OAUTH2_USER_INFO_URL': 'http://example.com/oauth2/user_info',
}
DEFAULT_PARTNER_ID = 1
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment