Skip to content

C
course-discovery
Commit 32b7fd14 by Clinton Blackburn Committed by GitHub
Options

Merge pull request #242 from edx/clintonb/organizations-data-loader 

Added data loader for schools and sponsors (Organizations)
parents 33198721 48dd3f1a
Showing with 321 additions and 11 deletions
course_discovery/apps/api/serializers.py
......@@ -7,6 +7,7 @@ from django.utils.translation import ugettext_lazy as _
from drf_haystack.serializers import HaystackSerializer, HaystackFacetSerializer
from rest_framework import serializers
from rest_framework.fields import DictField
from taggit_serializer.serializers import TagListSerializerField, TaggitSerializer
from course_discovery.apps.catalogs.models import Catalog
from course_discovery.apps.course_metadata.models import (
......@@ -155,12 +156,13 @@ class PersonSerializer(serializers.ModelSerializer):
fields = ('key', 'name', 'title', 'bio', 'profile_image',)
class OrganizationSerializer(serializers.ModelSerializer):
class OrganizationSerializer(TaggitSerializer, serializers.ModelSerializer):
"""Serializer for the ``Organization`` model."""
tags = TagListSerializerField()
class Meta(object):
model = Organization
fields = ('key', 'name', 'description', 'homepage_url',)
fields = ('key', 'name', 'description', 'homepage_url', 'tags',)
class CatalogSerializer(serializers.ModelSerializer):
......
course_discovery/apps/api/tests/test_serializers.py
......@@ -387,6 +387,8 @@ class VideoSerializerTests(TestCase):
class OrganizationSerializerTests(TestCase):
def test_data(self):
organization = OrganizationFactory()
TAG = 'test'
organization.tags.add(TAG)
serializer = OrganizationSerializer(organization)
expected = {
......@@ -394,6 +396,7 @@ class OrganizationSerializerTests(TestCase):
'name': organization.name,
'description': organization.description,
'homepage_url': organization.homepage_url,
'tags': [TAG],
}
self.assertDictEqual(serializer.data, expected)
......
course_discovery/apps/course_metadata/data_loaders/marketing_site.py
......@@ -304,3 +304,65 @@ class SubjectMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
subject, __ = Subject.objects.update_or_create(slug=slug, partner=self.partner, defaults=defaults)
logger.info('Processed subject with slug [%s].', slug)
return subject
class SchoolMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
@property
def node_type(self):
return 'school'
def process_node(self, data):
key = data['title']
defaults = {
'uuid': data['uuid'],
'name': data['field_school_name'],
'description': self.clean_html(data['field_school_description']['value']),
'logo_image_url': self._get_nested_url(data.get('field_school_image_logo')),
'banner_image_url': self._get_nested_url(data.get('field_school_image_banner')),
'marketing_url_path': 'school/' + data['field_school_url_slug'],
}
school, __ = Organization.objects.update_or_create(key=key, partner=self.partner, defaults=defaults)
self.set_tags(school, data)
logger.info('Processed school with key [%s].', key)
return school
def set_tags(self, school, data):
tags = []
mapping = {
'field_school_is_founder': 'founder',
'field_school_is_charter': 'charter',
'field_school_is_contributor': 'contributor',
'field_school_is_partner': 'partner',
}
for field, tag in mapping.items():
if data.get(field, False):
tags.append(tag)
school.tags.set(*tags, clear=True)
class SponsorMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
@property
def node_type(self):
return 'sponsorer'
def process_node(self, data):
uuid = data['uuid']
body = (data['body'] or {}).get('value')
if body:
body = self.clean_html(body)
defaults = {
'key': data['url'].split('/')[-1],
'name': data['title'],
'description': body,
'logo_image_url': data['field_sponsorer_image']['url'],
}
sponsor, __ = Organization.objects.update_or_create(uuid=uuid, partner=self.partner, defaults=defaults)
logger.info('Processed sponsor with UUID [%s].', uuid)
return sponsor
course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py
......@@ -9,7 +9,8 @@ from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
SponsorMarketingSiteDataLoader,
)
from course_discovery.apps.course_metadata.data_loaders.tests import JSON
from course_discovery.apps.course_metadata.data_loaders.tests.mixins import ApiClientTestMixin, DataLoaderTestMixin
......@@ -415,3 +416,93 @@ class SubjectMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMix
for datum in api_data:
self.assert_subject_loaded(datum)
class SchoolMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
loader_class = SchoolMarketingSiteDataLoader
def mock_api(self):
bodies = mock_data.MARKETING_SITE_API_SCHOOL_BODIES
url = self.api_url + 'node.json'
responses.add_callback(
responses.GET,
url,
callback=self.mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def assert_school_loaded(self, data):
key = data['title']
school = Organization.objects.get(key=key, partner=self.partner)
expected_values = {
'uuid': UUID(data['uuid']),
'name': data['field_school_name'],
'description': self.loader.clean_html(data['field_school_description']['value']),
'logo_image_url': data['field_school_image_logo']['url'],
'banner_image_url': data['field_school_image_banner']['url'],
'marketing_url_path': 'school/' + data['field_school_url_slug'],
}
for field, value in expected_values.items():
self.assertEqual(getattr(school, field), value)
self.assertEqual(sorted(school.tags.names()), ['charter', 'founder'])
@responses.activate
def test_ingest(self):
self.mock_login_response()
schools = self.mock_api()
self.loader.ingest()
for school in schools:
self.assert_school_loaded(school)
class SponsorMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
loader_class = SponsorMarketingSiteDataLoader
def mock_api(self):
bodies = mock_data.MARKETING_SITE_API_SPONSOR_BODIES
url = self.api_url + 'node.json'
responses.add_callback(
responses.GET,
url,
callback=self.mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def assert_sponsor_loaded(self, data):
uuid = data['uuid']
school = Organization.objects.get(uuid=uuid, partner=self.partner)
body = (data['body'] or {}).get('value')
if body:
body = self.loader.clean_html(body)
expected_values = {
'key': data['url'].split('/')[-1],
'name': data['title'],
'description': body,
'logo_image_url': data['field_sponsorer_image']['url'],
}
for field, value in expected_values.items():
self.assertEqual(getattr(school, field), value)
@responses.activate
def test_ingest(self):
self.mock_login_response()
sponsors = self.mock_api()
self.loader.ingest()
for sponsor in sponsors:
self.assert_sponsor_loaded(sponsor)
course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py
......@@ -8,7 +8,8 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
)
from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader,
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
SponsorMarketingSiteDataLoader,
)
logger = logging.getLogger(__name__)
......@@ -79,6 +80,8 @@ class Command(BaseCommand):
data_loaders = (
(partner.marketing_site_url_root, SubjectMarketingSiteDataLoader,),
(partner.marketing_site_url_root, SchoolMarketingSiteDataLoader,),
(partner.marketing_site_url_root, SponsorMarketingSiteDataLoader,),
(partner.organizations_api_url, OrganizationsApiDataLoader,),
(partner.courses_api_url, CoursesApiDataLoader,),
(partner.ecommerce_api_url, EcommerceApiDataLoader,),
......
course_discovery/apps/course_metadata/migrations/0017_auto_20160815_2135.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import taggit.managers
class Migration(migrations.Migration):
dependencies = [
('taggit', '0002_auto_20150616_2121'),
('course_metadata', '0016_auto_20160815_1438'),
]
operations = [
migrations.AddField(
model_name='historicalorganization',
name='marketing_url_path',
field=models.CharField(null=True, max_length=255, blank=True),
),
migrations.AddField(
model_name='organization',
name='marketing_url_path',
field=models.CharField(null=True, max_length=255, blank=True),
),
migrations.AddField(
model_name='organization',
name='tags',
field=taggit.managers.TaggableManager(verbose_name='Tags', to='taggit.Tag', through='taggit.TaggedItem', help_text='A comma-separated list of tags.', blank=True),
),
]
course_discovery/apps/course_metadata/models.py
......@@ -14,6 +14,7 @@ from djchoices import DjangoChoices, ChoiceItem
from haystack.query import SearchQuerySet
from simple_history.models import HistoricalRecords
from sortedm2m.fields import SortedManyToManyField
from taggit.managers import TaggableManager
from course_discovery.apps.core.models import Currency, Partner
from course_discovery.apps.course_metadata.query import CourseQuerySet
......@@ -156,11 +157,14 @@ class Organization(TimeStampedModel):
uuid = models.UUIDField(blank=False, null=False, default=uuid4, editable=False, verbose_name=_('UUID'))
key = models.CharField(max_length=255)
name = models.CharField(max_length=255)
marketing_url_path = models.CharField(max_length=255, null=True, blank=True)
description = models.TextField(null=True, blank=True)
homepage_url = models.URLField(max_length=255, null=True, blank=True)
logo_image_url = models.URLField(null=True, blank=True)
banner_image_url = models.URLField(null=True, blank=True)
history = HistoricalRecords()
tags = TaggableManager(blank=True)
class Meta:
unique_together = (
......
course_discovery/apps/course_metadata/tests/mock_data.py
......@@ -808,7 +808,7 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'expanded_html'
},
'field_xseries_banner_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/cs-1440x210.jpg'
'url': 'https://www.edx.org/sites/default/files/cs-1440x210.jpg'
},
'field_subject_url_slug': 'computer-science',
'field_subject_subtitle': {
......@@ -816,11 +816,11 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'basic_html'
},
'field_subject_card_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/subject/image/card/computer-science.jpg',
'url': 'https://www.edx.org/sites/default/files/subject/image/card/computer-science.jpg',
},
'type': 'subject',
'title': 'Computer Science',
'url': 'https://prod-edx-mktg-edit.edx.org/course/subject/math',
'url': 'https://www.edx.org/course/subject/math',
'uuid': 'e52e2134-a4e4-4fcb-805f-cbef40812580',
},
{
......@@ -832,7 +832,7 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'basic_html'
},
'field_xseries_banner_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/mathemagical-1440x210.jpg',
'url': 'https://www.edx.org/sites/default/files/mathemagical-1440x210.jpg',
},
'field_subject_url_slug': 'math',
'field_subject_subtitle': {
......@@ -840,11 +840,119 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'basic_html'
},
'field_subject_card_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/subject/image/card/math.jpg',
'url': 'https://www.edx.org/sites/default/files/subject/image/card/math.jpg',
},
'type': 'subject',
'title': 'Math',
'url': 'https://prod-edx-mktg-edit.edx.org/course/subject/math',
'url': 'https://www.edx.org/course/subject/math',
'uuid': 'a669e004-cbc0-4b68-8882-234c12e1cce4',
},
]
MARKETING_SITE_API_SCHOOL_BODIES = [
{
'field_school_description': {
'value': '\u003Cp\u003EHarvard University is devoted to excellence in teaching, learning, and '
'research, and to developing leaders in many disciplines who make a difference globally. '
'Harvard faculty are engaged with teaching and research to push the boundaries of human '
'knowledge. The University has twelve degree-granting Schools in addition to the Radcliffe '
'Institute for Advanced Study.\u003C/p\u003E\n\n\u003Cp\u003EEstablished in 1636, Harvard '
'is the oldest institution of higher education in the United States. The University, which '
'is based in Cambridge and Boston, Massachusetts, has an enrollment of over 20,000 degree '
'candidates, including undergraduate, graduate, and professional students. Harvard has more '
'than 360,000 alumni around the world.\u003C/p\u003E',
'format': 'standard_html'
},
'field_school_name': 'Harvard University',
'field_school_image_banner': {
'url': 'https:www.edx.org/sites/default/files/school/image/banner/harvardx.jpg',
},
'field_school_image_logo': {
'url': 'https://www.edx.org/sites/default/files/school/image/banner/harvard_logo_200x101_0.png',
},
'field_school_subdomain_prefix': 'harvard',
'field_school_url_slug': 'harvardx',
'field_school_is_school': True,
'field_school_is_partner': False,
'field_school_is_contributor': False,
'field_school_is_charter': True,
'field_school_is_founder': True,
'field_school_is_display': True,
'field_school_is_affiliate': False,
'type': 'school',
'title': 'HarvardX',
'url': 'https://www.edx.org/school/harvardx',
'uuid': '44022f13-20df-4666-9111-cede3e5dc5b6',
},
{
'field_school_description': {
'value': '\u003Cp\u003EMassachusetts Institute of Technology \u2014 a coeducational, privately '
'endowed research university founded in 1861 \u2014 is dedicated to advancing knowledge '
'and educating students in science, technology, and other areas of scholarship that will '
'best serve the nation and the world in the 21st century. \u003Ca href=\u0022http://web.'
'mit.edu/aboutmit/\u0022 target=\u0022_blank\u0022\u003ELearn more about MIT\u003C/a\u003E'
'. Through MITx, the Institute furthers its commitment to improving education worldwide.'
'\u003C/p\u003E\n\n\u003Cp\u003E\u003Cstrong\u003EMITx Courses\u003C/strong\u003E\u003Cbr '
'/\u003E\nMITx courses embody the inventiveness, openness, rigor and quality that are '
'hallmarks of MIT, and many use materials developed for MIT residential courses in the '
'Institute\u0027s five schools and 33 academic disciplines. Browse MITx courses below.'
'\u003C/p\u003E\n\n\u003Cp\u003E\u00a0\u003C/p\u003E',
},
'field_school_name': 'MIT',
'field_school_image_banner': {
'url': 'https://www.edx.org/sites/default/files/school/image/banner/mit-home-banner_0.jpg',
},
'field_school_image_logo': {
'url': 'https://www.edx.org/sites/default/files/school/image/banner/mit_logo_200x101_0.png',
},
'field_school_url_slug': 'mitx',
'field_school_is_school': True,
'field_school_is_partner': False,
'field_school_is_contributor': False,
'field_school_is_charter': True,
'field_school_is_founder': True,
'field_school_is_display': True,
'field_school_is_affiliate': False,
'type': 'school',
'title': 'MITx',
'url': 'https://www.edx.org/school/mitx',
'uuid': '2a73d2ce-c34a-4e08-8223-83bca9d2f01d'
},
]
MARKETING_SITE_API_SPONSOR_BODIES = [
{
'body': [],
'field_sponsorer_image': {
'url': 'https://www.edx.org/sites/default/files/sponsorer/image/trkcll.jpg',
},
'type': 'sponsorer',
'title': 'Turkcell Akademi',
'url': 'https://www.edx.org/sponsorer/turkcell-akademi',
'uuid': 'fcb48e7e-8f1b-4d4b-8bb0-77617aaad9ba'
},
{
'body': [],
'field_sponsorer_image': {
'url': 'https://www.edx.org/sites/default/files/sponsorer/image/databricks.png'
},
'type': 'sponsorer',
'title': 'Databricks',
'url': 'https://www.edx.org/sponsorer/databricks',
'uuid': '1d86977a-0661-44c9-8f39-32bbf8ca7d4b',
},
{
'body': {
'value': 'UC Berkeley is partnering with the U.S. Department of State to extend the reach of College '
'Writing 2X',
},
'field_sponsorer_image': {
'url': 'https://www.edx.org/sites/default/files/sponsorer/image/usdos-logo-seal.png',
},
'type': 'sponsorer',
'title': 'The U.S. Department of State',
'url': 'https://www.edx.org/sponsorer/u-s-department-state',
'uuid': 'db53bc49-bac0-4efe-8d77-1a2d8d185024'
},
]
course_discovery/settings/base.py
......@@ -45,6 +45,9 @@ THIRD_PARTY_APPS = [
'django_filters',
'django_fsm',
'storages',
'django_comments',
'taggit',
'taggit_serializer',
]
PROJECT_APPS = [
......@@ -56,7 +59,6 @@ PROJECT_APPS = [
'course_discovery.apps.edx_haystack_extensions',
'course_discovery.apps.publisher',
'course_discovery.apps.publisher_comments',
'django_comments',
]
......@@ -359,3 +361,5 @@ DEFAULT_PARTNER_ID = None
# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1
COMMENTS_APP = 'course_discovery.apps.publisher_comments'
TAGGIT_CASE_INSENSITIVE = True
requirements/base.txt
......@@ -12,6 +12,8 @@ django-libsass==0.7
django-simple-history==1.8.1
django-sortedm2m==1.3.2
django-storages==1.5.0
django-taggit==0.20.2
django-taggit-serializer==0.1.5
django-waffle==0.11.1
djangorestframework==3.3.3
djangorestframework-csv==1.4.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment