Skip to content

C
course-discovery
Commit b6376f1a by Clinton Blackburn Committed by GitHub
Options

Merge pull request #227 from edx/clintonb/subject 

Updated subject model and data loader
parents c9ea3ef5 39e572b9
Showing with 267 additions and 16 deletions
course_discovery/apps/course_metadata/admin.py
......@@ -74,6 +74,14 @@ class OrganizationAdmin(admin.ModelAdmin):
list_filter = ('partner',)
@admin.register(Subject)
class SubjectAdmin(admin.ModelAdmin):
list_display = ('uuid', 'name', 'slug',)
list_filter = ('partner',)
readonly_fields = ('uuid',)
search_fields = ('uuid', 'name', 'slug',)
class KeyNameAdmin(admin.ModelAdmin):
list_display = ('key', 'name',)
ordering = ('key', 'name',)
......@@ -91,7 +99,7 @@ for model in (Person,):
admin.site.register(model, KeyNameAdmin)
# Register children of AbstractNamedModel
for model in (LevelType, Subject, Prerequisite, Expertise, MajorWork):
for model in (LevelType, Prerequisite, Expertise, MajorWork):
admin.site.register(model, NamedModelAdmin)
# Register remaining models using basic ModelAdmin classes
......
course_discovery/apps/course_metadata/data_loaders/marketing_site.py
......@@ -77,10 +77,8 @@ class DrupalApiDataLoader(AbstractDataLoader):
"""Update `course` with subjects from `body`."""
course.subjects.clear()
subjects = (s['title'] for s in body['subjects'])
for subject_name in subjects:
# Normalize subject names with title case
subject, __ = Subject.objects.get_or_create(name=subject_name.title())
course.subjects.add(subject)
subjects = Subject.objects.filter(name__in=subjects, partner=self.partner)
course.subjects.add(*subjects)
def set_sponsors(self, course, body):
"""Update `course` with sponsors from `body`."""
......@@ -285,3 +283,25 @@ class XSeriesMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
program.save()
logger.info('Processed XSeries with marketing_slug [%s].', marketing_slug)
return program
class SubjectMarketingSiteDataLoader(AbstractMarketingSiteDataLoader):
@property
def node_type(self):
return 'subject'
def process_node(self, data):
slug = data['field_subject_url_slug']
defaults = {
'uuid': data['uuid'],
'name': data['title'],
'description': self.clean_html(data['body']['value']),
'subtitle': self.clean_html(data['field_subject_subtitle']['value']),
'card_image_url': self._get_nested_url(data.get('field_subject_card_image')),
# NOTE (CCB): This is not a typo. Yes, the banner image for subjects is in a field with xseries in the name.
'banner_image_url': self._get_nested_url(data.get('field_xseries_banner_image'))
}
subject, __ = Subject.objects.update_or_create(slug=slug, partner=self.partner, defaults=defaults)
logger.info('Processed subject with slug [%s].', slug)
return subject
course_discovery/apps/course_metadata/data_loaders/tests/test_marketing_site.py
import json
from urllib.parse import parse_qs, urlparse
from uuid import UUID
import ddt
import mock
......@@ -8,7 +9,7 @@ from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader,
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader
)
from course_discovery.apps.course_metadata.data_loaders.tests import JSON
from course_discovery.apps.course_metadata.data_loaders.tests.mixins import ApiClientTestMixin, DataLoaderTestMixin
......@@ -55,9 +56,19 @@ class DrupalApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase
Person.objects.create(key=mock_data.ORPHAN_STAFF_KEY)
Organization.objects.create(key=mock_data.ORPHAN_ORGANIZATION_KEY)
def create_mock_subjects(self, course_runs):
course_runs = course_runs['items']
for course_run in course_runs:
if course_run:
for subject in course_run['subjects']:
Subject.objects.get_or_create(name=subject['title'], partner=self.partner)
def mock_api(self):
"""Mock out the Drupal API. Returns a list of mocked-out course runs."""
body = mock_data.MARKETING_API_BODY
self.create_mock_subjects(body)
responses.add(
responses.GET,
self.api_url + 'courses/',
......@@ -111,11 +122,10 @@ class DrupalApiDataLoaderTests(ApiClientTestMixin, DataLoaderTestMixin, TestCase
def assert_subjects_loaded(self, course, body):
"""Verify that subjects have been loaded correctly."""
course_subjects = course.subjects.all()
api_subjects = body['subjects']
self.assertEqual(len(course_subjects), len(api_subjects))
for api_subject in api_subjects:
loaded_subject = Subject.objects.get(name=api_subject['title'].title())
self.assertIn(loaded_subject, course_subjects)
expected_subjects = body['subjects']
expected_subjects = [subject['title'] for subject in expected_subjects]
actual_subjects = list(course_subjects.values_list('name', flat=True))
self.assertEqual(actual_subjects, expected_subjects)
def assert_sponsors_loaded(self, course, body):
"""Verify that sponsors have been loaded correctly."""
......@@ -298,7 +308,6 @@ class AbstractMarketingSiteDataLoaderTestMixin(DataLoaderTestMixin):
class XSeriesMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
loader_class = XSeriesMarketingSiteDataLoader
LOGIN_COOKIE = ('session_id', 'abc123')
def create_mock_programs(self, programs):
for program in programs:
......@@ -364,3 +373,45 @@ class XSeriesMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMix
calls = [mock.call('Program [%s] exists on the marketing site, but not in the Programs Service!',
datum['url'].split('/')[-1]) for datum in api_data]
mock_logger.error.assert_has_calls(calls)
class SubjectMarketingSiteDataLoaderTests(AbstractMarketingSiteDataLoaderTestMixin, TestCase):
loader_class = SubjectMarketingSiteDataLoader
def mock_api(self):
bodies = mock_data.MARKETING_SITE_API_SUBJECT_BODIES
url = self.api_url + 'node.json'
responses.add_callback(
responses.GET,
url,
callback=self.mock_api_callback(url, bodies),
content_type=JSON
)
return bodies
def assert_subject_loaded(self, data):
slug = data['field_subject_url_slug']
subject = Subject.objects.get(slug=slug, partner=self.partner)
expected_values = {
'uuid': UUID(data['uuid']),
'name': data['title'],
'description': self.loader.clean_html(data['body']['value']),
'subtitle': self.loader.clean_html(data['field_subject_subtitle']['value']),
'card_image_url': data['field_subject_card_image']['url'],
'banner_image_url': data['field_xseries_banner_image']['url'],
}
for field, value in expected_values.items():
self.assertEqual(getattr(subject, field), value)
@responses.activate
def test_ingest(self):
self.mock_login_response()
api_data = self.mock_api()
self.loader.ingest()
for datum in api_data:
self.assert_subject_loaded(datum)
course_discovery/apps/course_metadata/management/commands/refresh_course_metadata.py
......@@ -8,7 +8,7 @@ from course_discovery.apps.course_metadata.data_loaders.api import (
CoursesApiDataLoader, OrganizationsApiDataLoader, EcommerceApiDataLoader, ProgramsApiDataLoader,
)
from course_discovery.apps.course_metadata.data_loaders.marketing_site import (
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader,
DrupalApiDataLoader, XSeriesMarketingSiteDataLoader, SubjectMarketingSiteDataLoader,
)
logger = logging.getLogger(__name__)
......@@ -78,6 +78,7 @@ class Command(BaseCommand):
raise
data_loaders = (
(partner.marketing_site_url_root, SubjectMarketingSiteDataLoader,),
(partner.organizations_api_url, OrganizationsApiDataLoader,),
(partner.courses_api_url, CoursesApiDataLoader,),
(partner.ecommerce_api_url, EcommerceApiDataLoader,),
......
course_discovery/apps/course_metadata/migrations/0013_auto_20160809_1259.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import uuid
import django_extensions.db.fields
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db import migrations, models
def update_subjects(apps, schema_editor):
Subject = apps.get_model('course_metadata', 'Subject')
subjects = Subject.objects.filter(partner__isnull=True)
if subjects.count() > 0:
# We perform this check here to avoid issues with migrations for empty databases
# (e.g. when running unit tests) that don't yet have a defined Partner.
if not settings.DEFAULT_PARTNER_ID:
raise ImproperlyConfigured('DEFAULT_PARTNER_ID must be defined!')
Partner = apps.get_model('core', 'Partner')
partner = Partner.objects.get(id=settings.DEFAULT_PARTNER_ID)
# We iterate over all subjects, instead of calling .update(), to trigger slug generation
for subject in subjects:
subject.partner = partner
subject.uuid = uuid.uuid4()
subject.save()
class Migration(migrations.Migration):
dependencies = [
('core', '0010_auto_20160731_0023'),
('course_metadata', '0012_create_seat_types'),
]
operations = [
migrations.AddField(
model_name='subject',
name='uuid',
field=models.UUIDField(verbose_name='UUID', editable=False, default=uuid.uuid4),
),
migrations.AddField(
model_name='subject',
name='banner_image_url',
field=models.URLField(blank=True, null=True),
),
migrations.AddField(
model_name='subject',
name='card_image_url',
field=models.URLField(blank=True, null=True),
),
migrations.AddField(
model_name='subject',
name='description',
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name='subject',
name='partner',
field=models.ForeignKey(to='core.Partner', null=True),
),
migrations.AddField(
model_name='subject',
name='slug',
field=django_extensions.db.fields.AutoSlugField(overwrite=True, editable=False, blank=True,
populate_from='name'),
),
migrations.AddField(
model_name='subject',
name='subtitle',
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AlterField(
model_name='subject',
name='name',
field=models.CharField(max_length=255),
),
migrations.AlterUniqueTogether(
name='subject',
unique_together=set([('partner', 'name'), ('partner', 'slug'), ('partner', 'uuid')]),
),
migrations.RunPython(update_subjects, lambda *args: None),
migrations.AlterField(
model_name='subject',
name='slug',
field=django_extensions.db.fields.AutoSlugField(populate_from='name', editable=False,
help_text='Leave this field blank to have the value generated automatically.',
blank=True),
),
migrations.AlterField(
model_name='subject',
name='partner',
field=models.ForeignKey(to='core.Partner'),
),
]
course_discovery/apps/course_metadata/models.py
......@@ -97,9 +97,27 @@ class LevelType(AbstractNamedModel):
pass
class Subject(AbstractNamedModel):
class Subject(TimeStampedModel):
""" Subject model. """
pass
uuid = models.UUIDField(blank=False, null=False, default=uuid4, editable=False, verbose_name=_('UUID'))
name = models.CharField(max_length=255, blank=False, null=False)
subtitle = models.CharField(max_length=255, blank=True, null=True)
description = models.TextField(blank=True, null=True)
banner_image_url = models.URLField(blank=True, null=True)
card_image_url = models.URLField(blank=True, null=True)
slug = AutoSlugField(populate_from='name', editable=True, blank=True,
help_text=_('Leave this field blank to have the value generated automatically.'))
partner = models.ForeignKey(Partner)
def __str__(self):
return self.name
class Meta:
unique_together = (
('partner', 'name'),
('partner', 'slug'),
('partner', 'uuid'),
)
class Prerequisite(AbstractNamedModel):
......
course_discovery/apps/course_metadata/tests/factories.py
......@@ -43,10 +43,16 @@ class VideoFactory(AbstractMediaModelFactory):
model = Video
class SubjectFactory(AbstractNamedModelFactory):
class SubjectFactory(factory.DjangoModelFactory):
class Meta:
model = Subject
name = FuzzyText()
description = FuzzyText()
banner_image_url = FuzzyURL()
card_image_url = FuzzyURL()
partner = factory.SubFactory(PartnerFactory)
class LevelTypeFactory(AbstractNamedModelFactory):
class Meta:
......
course_discovery/apps/course_metadata/tests/mock_data.py
......@@ -795,3 +795,52 @@ MARKETING_SITE_API_XSERIES_BODIES = [
'url': 'https://www.edx.org/xseries/supply-chain-management-0'
}
]
MARKETING_SITE_API_SUBJECT_BODIES = [
{
'body': {
'value': 'Yay! CS!',
'summary': '',
'format': 'expanded_html'
},
'field_xseries_banner_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/cs-1440x210.jpg'
},
'field_subject_url_slug': 'computer-science',
'field_subject_subtitle': {
'value': 'Learn about computer science from the best universities and institutions around the world.',
'format': 'basic_html'
},
'field_subject_card_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/subject/image/card/computer-science.jpg',
},
'type': 'subject',
'title': 'Computer Science',
'url': 'https://prod-edx-mktg-edit.edx.org/course/subject/math',
'uuid': 'e52e2134-a4e4-4fcb-805f-cbef40812580',
},
{
'body': {
'value': 'Take free online math courses from MIT, Caltech, Tsinghua and other leading math and science '
'institutions. Get introductions to algebra, geometry, trigonometry, precalculus and calculus '
'or get help with current math coursework and AP exam preparation.',
'summary': '',
'format': 'basic_html'
},
'field_xseries_banner_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/mathemagical-1440x210.jpg',
},
'field_subject_url_slug': 'math',
'field_subject_subtitle': {
'value': 'Learn about math and more from the best universities and institutions around the world.',
'format': 'basic_html'
},
'field_subject_card_image': {
'url': 'https://prod-edx-mktg-edit.edx.org/sites/default/files/subject/image/card/math.jpg',
},
'type': 'subject',
'title': 'Math',
'url': 'https://prod-edx-mktg-edit.edx.org/course/subject/math',
'uuid': 'a669e004-cbc0-4b68-8882-234c12e1cce4',
},
]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment