Skip to content

C
course-discovery
Commit ea25f9b3 by Clinton Blackburn
Options

Added school (Organization) data loader 

ECOM-5192
parent 1cadc799
Showing with 162 additions and 8 deletions
course_discovery/apps/course_metadata/data_loaders/marketing_site.py
......@@ -304,3 +304,41 @@ class SubjectMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
subject, __ = Subject.objects.update_or_create(slug=slug, partner=self.partner, defaults=defaults)
logger.info('Processed subject with slug [%s].', slug)
return subject
class SchoolMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
@property
def node_type(self):
return 'school'
def process_node(self, data):
key = data['title']
defaults = {
'uuid': data['uuid'],
'name': data['field_school_name'],
'description': self.clean_html(data['field_school_description']['value']),
'logo_image_url': self._get_nested_url(data.get('field_school_image_logo')),
'banner_image_url': self._get_nested_url(data.get('field_school_image_banner')),
'marketing_url_path': 'school/' + data['field_school_url_slug'],
}
school, __ = Organization.objects.update_or_create(key=key, partner=self.partner, defaults=defaults)
self.set_tags(school, data)
logger.info('Processed school with key [%s].', key)
return school
def set_tags(self, school, data):
tags = []
mapping = {
'field_school_is_founder': 'founder',
'field_school_is_charter': 'charter',
'field_school_is_contributor': 'contributor',
'field_school_is_partner': 'partner',
}
for field, tag in mapping.items():
if data.get(field, False):
tags.append(tag)
school.tags.set(*tags, clear=True)
course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py
......@@ -9,7 +9,7 @@ from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
)
from course_discovery.apps.course_metadata.data_loaders.tests import JSON
from course_discovery.apps.course_metadata.data_loaders.tests.mixins import ApiClientTestMixin, DataLoaderTestMixin
......@@ -415,3 +415,47 @@ class SubjectMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMix
for datum in api_data:
self.assert_subject_loaded(datum)
class SchoolMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
loader_class = SchoolMarketingSiteDataLoader
def mock_api(self):
bodies = mock_data.MARKETING_SITE_API_SCHOOL_BODIES
url = self.api_url + 'node.json'
responses.add_callback(
responses.GET,
url,
callback=self.mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def assert_school_loaded(self, data):
key = data['title']
school = Organization.objects.get(key=key, partner=self.partner)
expected_values = {
'uuid': UUID(data['uuid']),
'name': data['field_school_name'],
'description': self.loader.clean_html(data['field_school_description']['value']),
'logo_image_url': data['field_school_image_logo']['url'],
'banner_image_url': data['field_school_image_banner']['url'],
'marketing_url_path': 'school/' + data['field_school_url_slug'],
}
for field, value in expected_values.items():
self.assertEqual(getattr(school, field), value)
self.assertEqual(sorted(school.tags.names()), ['charter', 'founder'])
@responses.activate
def test_ingest(self):
self.mock_login_response()
schools = self.mock_api()
self.loader.ingest()
for school in schools:
self.assert_school_loaded(school)
course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py
......@@ -8,7 +8,7 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
)
from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader,
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader, SchoolMarketingSiteDataLoader,
)
logger = logging.getLogger(__name__)
......@@ -79,6 +79,7 @@ class Command(BaseCommand):
data_loaders = (
(partner.marketing_site_url_root, SubjectMarketingSiteDataLoader,),
(partner.marketing_site_url_root, SchoolMarketingSiteDataLoader,),
(partner.organizations_api_url, OrganizationsApiDataLoader,),
(partner.courses_api_url, CoursesApiDataLoader,),
(partner.ecommerce_api_url, EcommerceApiDataLoader,),
......
course_discovery/apps/course_metadata/tests/mock_data.py
......@@ -808,7 +808,7 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'expanded_html'
},
'field_xseries_banner_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/cs-1440x210.jpg'
'url': 'https://www.edx.org/sites/default/files/cs-1440x210.jpg'
},
'field_subject_url_slug': 'computer-science',
'field_subject_subtitle': {
......@@ -816,11 +816,11 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'basic_html'
},
'field_subject_card_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/subject/image/card/computer-science.jpg',
'url': 'https://www.edx.org/sites/default/files/subject/image/card/computer-science.jpg',
},
'type': 'subject',
'title': 'Computer Science',
'url': 'https://prod-edx-mktg-edit.edx.org/course/subject/math',
'url': 'https://www.edx.org/course/subject/math',
'uuid': 'e52e2134-a4e4-4fcb-805f-cbef40812580',
},
{
......@@ -832,7 +832,7 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'basic_html'
},
'field_xseries_banner_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/mathemagical-1440x210.jpg',
'url': 'https://www.edx.org/sites/default/files/mathemagical-1440x210.jpg',
},
'field_subject_url_slug': 'math',
'field_subject_subtitle': {
......@@ -840,11 +840,82 @@ MARKETING_SITE_API_SUBJECT_BODIES = [
'format': 'basic_html'
},
'field_subject_card_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/subject/image/card/math.jpg',
'url': 'https://www.edx.org/sites/default/files/subject/image/card/math.jpg',
},
'type': 'subject',
'title': 'Math',
'url': 'https://prod-edx-mktg-edit.edx.org/course/subject/math',
'url': 'https://www.edx.org/course/subject/math',
'uuid': 'a669e004-cbc0-4b68-8882-234c12e1cce4',
},
]
MARKETING_SITE_API_SCHOOL_BODIES = [
{
'field_school_description': {
'value': '\u003Cp\u003EHarvard University is devoted to excellence in teaching, learning, and '
'research, and to developing leaders in many disciplines who make a difference globally. '
'Harvard faculty are engaged with teaching and research to push the boundaries of human '
'knowledge. The University has twelve degree-granting Schools in addition to the Radcliffe '
'Institute for Advanced Study.\u003C/p\u003E\n\n\u003Cp\u003EEstablished in 1636, Harvard '
'is the oldest institution of higher education in the United States. The University, which '
'is based in Cambridge and Boston, Massachusetts, has an enrollment of over 20,000 degree '
'candidates, including undergraduate, graduate, and professional students. Harvard has more '
'than 360,000 alumni around the world.\u003C/p\u003E',
'format': 'standard_html'
},
'field_school_name': 'Harvard University',
'field_school_image_banner': {
'url': 'https:www.edx.org/sites/default/files/school/image/banner/harvardx.jpg',
},
'field_school_image_logo': {
'url': 'https://www.edx.org/sites/default/files/school/image/banner/harvard_logo_200x101_0.png',
},
'field_school_subdomain_prefix': 'harvard',
'field_school_url_slug': 'harvardx',
'field_school_is_school': True,
'field_school_is_partner': False,
'field_school_is_contributor': False,
'field_school_is_charter': True,
'field_school_is_founder': True,
'field_school_is_display': True,
'field_school_is_affiliate': False,
'type': 'school',
'title': 'HarvardX',
'url': 'https://www.edx.org/school/harvardx',
'uuid': '44022f13-20df-4666-9111-cede3e5dc5b6',
},
{
'field_school_description': {
'value': '\u003Cp\u003EMassachusetts Institute of Technology \u2014 a coeducational, privately '
'endowed research university founded in 1861 \u2014 is dedicated to advancing knowledge '
'and educating students in science, technology, and other areas of scholarship that will '
'best serve the nation and the world in the 21st century. \u003Ca href=\u0022http://web.'
'mit.edu/aboutmit/\u0022 target=\u0022_blank\u0022\u003ELearn more about MIT\u003C/a\u003E'
'. Through MITx, the Institute furthers its commitment to improving education worldwide.'
'\u003C/p\u003E\n\n\u003Cp\u003E\u003Cstrong\u003EMITx Courses\u003C/strong\u003E\u003Cbr '
'/\u003E\nMITx courses embody the inventiveness, openness, rigor and quality that are '
'hallmarks of MIT, and many use materials developed for MIT residential courses in the '
'Institute\u0027s five schools and 33 academic disciplines. Browse MITx courses below.'
'\u003C/p\u003E\n\n\u003Cp\u003E\u00a0\u003C/p\u003E',
},
'field_school_name': 'MIT',
'field_school_image_banner': {
'url': 'https://www.edx.org/sites/default/files/school/image/banner/mit-home-banner_0.jpg',
},
'field_school_image_logo': {
'url': 'https://www.edx.org/sites/default/files/school/image/banner/mit_logo_200x101_0.png',
},
'field_school_url_slug': 'mitx',
'field_school_is_school': True,
'field_school_is_partner': False,
'field_school_is_contributor': False,
'field_school_is_charter': True,
'field_school_is_founder': True,
'field_school_is_display': True,
'field_school_is_affiliate': False,
'type': 'school',
'title': 'MITx',
'url': 'https://www.edx.org/school/mitx',
'uuid': '2a73d2ce-c34a-4e08-8223-83bca9d2f01d'
},
]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment