Skip to content

C
course-discovery
Commit 1f771669 by Bill DeRusha Committed by Clinton Blackburn
Options

Add Course Metadata models

parent d1d54641
Showing with 953 additions and 827 deletions
course_discovery/apps/api/pagination.py deleted 100644 → 0
from rest_framework.pagination import LimitOffsetPagination
class ElasticsearchLimitOffsetPagination(LimitOffsetPagination):
def paginate_queryset(self, queryset, request, view=None):
"""
Convert a paginated Elasticsearch response to a response suitable for DRF.
Args:
queryset (dict): Elasticsearch response
request (Request): HTTP request
Returns:
List of data.
"""
# pylint: disable=attribute-defined-outside-init
self.limit = self.get_limit(request)
self.offset = self.get_offset(request)
self.count = queryset['total']
self.request = request
if self.count > self.limit and self.template is not None:
self.display_page_controls = True
return queryset['results']
course_discovery/apps/api/serializers.py
......@@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from course_discovery.apps.catalogs.models import Catalog
from course_discovery.apps.course_metadata.models import Course
class CatalogSerializer(serializers.ModelSerializer):
......@@ -12,10 +13,13 @@ class CatalogSerializer(serializers.ModelSerializer):
fields = ('id', 'name', 'query', 'url',)
class CourseSerializer(serializers.Serializer): # pylint: disable=abstract-method
id = serializers.CharField(help_text=_('Course ID'))
name = serializers.CharField(help_text=_('Course name'))
url = serializers.HyperlinkedIdentityField(view_name='api:v1:course-detail', lookup_field='id')
class CourseSerializer(serializers.ModelSerializer):
key = serializers.CharField()
title = serializers.CharField()
class Meta(object):
model = Course
fields = ('key', 'title',)
class ContainedCoursesSerializer(serializers.Serializer): # pylint: disable=abstract-method
......
course_discovery/apps/api/tests/test_serializers.py
......@@ -25,14 +25,13 @@ class CatalogSerializerTests(TestCase):
class CourseSerializerTests(TestCase):
def test_data(self):
course = CourseFactory()
path = reverse('api:v1:course-detail', kwargs={'id': course.id})
path = reverse('api:v1:course-detail', kwargs={'key': course.key})
request = RequestFactory().get(path)
serializer = CourseSerializer(course, context={'request': request})
expected = {
'id': course.id,
'name': course.name,
'url': request.build_absolute_uri(),
'key': course.key,
'title': course.title,
}
self.assertDictEqual(serializer.data, expected)
......
course_discovery/apps/api/v1/tests/test_views.py
......@@ -2,6 +2,7 @@
import json
import urllib
from time import time
from unittest import skip
import ddt
import jwt
......@@ -70,21 +71,8 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
super(CatalogViewSetTests, self).setUp()
self.user = UserFactory(is_staff=True, is_superuser=True)
self.client.login(username=self.user.username, password=USER_PASSWORD)
query = {
'query': {
'bool': {
'must': [
{
'wildcard': {
'course.name': 'abc*'
}
}
]
}
}
}
self.catalog = CatalogFactory(query=json.dumps(query))
self.course = CourseFactory(id='a/b/c', name='ABC Test Course')
self.catalog = CatalogFactory(query='title:abc*')
self.course = CourseFactory(key='a/b/c', title='ABC Test Course')
self.refresh_index()
def generate_jwt_token_header(self, user):
......@@ -153,6 +141,7 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
self.mock_user_info_response(self.user)
self.assert_catalog_created(HTTP_AUTHORIZATION=self.generate_oauth2_token_header(self.user))
@skip('Re-enable once we switch to Haystack')
def test_courses(self):
""" Verify the endpoint returns the list of courses contained in the catalog. """
url = reverse('api:v1:catalog-courses', kwargs={'id': self.catalog.id})
......@@ -162,15 +151,16 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
self.assertEqual(response.status_code, 200)
self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
@skip('Re-enable once we switch to Haystack')
def test_contains(self):
""" Verify the endpoint returns a filtered list of courses contained in the catalog. """
course_id = self.course.id
qs = urllib.parse.urlencode({'course_id': course_id})
course_key = self.course.key
qs = urllib.parse.urlencode({'course_id': course_key})
url = '{}?{}'.format(reverse('api:v1:catalog-contains', kwargs={'id': self.catalog.id}), qs)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data, {'courses': {course_id: True}})
self.assertEqual(response.data, {'courses': {course_key: True}})
def test_get(self):
""" Verify the endpoint returns the details for a single catalog. """
......@@ -242,10 +232,9 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
def test_list(self, format):
""" Verify the endpoint returns a list of all courses. """
courses = CourseFactory.create_batch(10)
courses.sort(key=lambda course: course.id.lower())
courses.sort(key=lambda course: course.key.lower())
url = reverse('api:v1:course-list')
limit = 3
self.refresh_index()
response = self.client.get(url, {'format': format, 'limit': limit})
self.assertEqual(response.status_code, 200)
......@@ -253,38 +242,6 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
response.render()
def test_list_query(self):
""" Verify the endpoint returns a filtered list of courses. """
# Create courses that should NOT match our query
CourseFactory.create_batch(3)
# Create courses that SHOULD match our query
name = 'query test'
courses = [CourseFactory(name=name), CourseFactory(name=name)]
courses.sort(key=lambda course: course.id.lower())
self.refresh_index()
query = {
"query": {
"bool": {
"must": [
{
"term": {
"course.name": name
}
}
]
}
}
}
qs = urllib.parse.urlencode({'q': json.dumps(query)})
url = '{}?{}'.format(reverse('api:v1:course-list'), qs)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data['count'], len(courses))
self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
def test_retrieve(self):
""" Verify the endpoint returns a single course. """
self.assert_retrieve_success()
......@@ -292,7 +249,7 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
def assert_retrieve_success(self, **headers):
""" Asserts the endpoint returns details for a single course. """
course = CourseFactory()
url = reverse('api:v1:course-detail', kwargs={'id': course.id})
url = reverse('api:v1:course-detail', kwargs={'key': course.key})
response = self.client.get(url, format='json', **headers)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data, self.serialize_course(course))
......
course_discovery/apps/api/v1/views.py
import json
import logging
from django.db.models.functions import Lower
from rest_framework import viewsets
from rest_framework.decorators import detail_route
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from course_discovery.apps.api.pagination import ElasticsearchLimitOffsetPagination
from course_discovery.apps.api.serializers import CatalogSerializer, CourseSerializer, ContainedCoursesSerializer
from course_discovery.apps.catalogs.models import Catalog
from course_discovery.apps.course_metadata.constants import COURSE_ID_REGEX
......@@ -91,49 +90,15 @@ class CatalogViewSet(viewsets.ModelViewSet):
class CourseViewSet(viewsets.ReadOnlyModelViewSet):
""" Course resource. """
lookup_field = 'id'
lookup_field = 'key'
lookup_value_regex = COURSE_ID_REGEX
permission_classes = (IsAuthenticated,)
serializer_class = CourseSerializer
pagination_class = ElasticsearchLimitOffsetPagination
def get_object(self):
""" Return a single course. """
return Course.get(self.kwargs[self.lookup_url_kwarg or self.lookup_field])
def get_queryset(self):
# Note (CCB): This is solely here to appease DRF. It is not actually used.
return []
def get_data(self, limit, offset):
""" Return all courses. """
query = self.request.GET.get('q', None)
if query:
query = json.loads(query)
return Course.search(query, limit=limit, offset=offset)
else:
return Course.all(limit=limit, offset=offset)
def list(self, request, *args, **kwargs): # pylint: disable=unused-argument
"""
List all courses.
---
parameters:
- name: q
description: Query to filter the courses
required: false
type: string
paramType: query
multiple: false
"""
limit = self.paginator.get_limit(self.request)
offset = self.paginator.get_offset(self.request)
data = self.get_data(limit, offset)
queryset = Course.objects.all().order_by(Lower('key'))
page = self.paginate_queryset(data)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def list(self, request, *args, **kwargs):
""" List all courses. """
return super(CourseViewSet, self).list(request, *args, **kwargs)
def retrieve(self, request, *args, **kwargs):
""" Retrieve details for a course. """
......
course_discovery/apps/catalogs/models.py
......@@ -4,8 +4,6 @@ from django.db import models
from django.utils.translation import ugettext_lazy as _
from django_extensions.db.models import TimeStampedModel
from course_discovery.apps.course_metadata.models import Course
class Catalog(TimeStampedModel):
name = models.CharField(max_length=255, null=False, blank=False, help_text=_('Catalog name'))
......@@ -25,7 +23,8 @@ class Catalog(TimeStampedModel):
Course[]
"""
return Course.search(self.query_as_dict)['results']
# TODO: Course.search no longer exists. Figure out what goes here.
# return Course.search(self.query_as_dict)['results']
def contains(self, course_ids): # pylint: disable=unused-argument
""" Determines if the given courses are contained in this catalog.
......@@ -37,26 +36,29 @@ class Catalog(TimeStampedModel):
dict: Mapping of course IDs to booleans indicating if course is
contained in this catalog.
"""
query = self.query_as_dict['query']
# Create a filtered query that includes that uses the catalog's query against a
# collection of courses filtered using the passed in course IDs.
filtered_query = {
"query": {
"filtered": {
"query": query,
"filter": {
"ids": {
"values": course_ids
}
}
}
}
}
contains = {course_id: False for course_id in course_ids}
courses = Course.search(filtered_query)['results']
for course in courses:
contains[course.id] = True
return contains
# query = self.query_as_dict['query']
# # Create a filtered query that includes that uses the catalog's query against a
# # collection of courses filtered using the passed in course IDs.
# filtered_query = {
# "query": {
# "filtered": {
# "query": query,
# "filter": {
# "ids": {
# "values": course_ids
# }
# }
# }
# }
# }
# contains = {course_id: False for course_id in course_ids}
# TODO: Course.search no longer exists. Figure out what goes here.
# courses = Course.search(filtered_query)['results']
# for course in courses:
# contains[course.id] = True
# return contains
pass
course_discovery/apps/catalogs/tests/test_models.py
import json
from unittest import skip
from django.test import TestCase
......@@ -18,7 +19,7 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
'must': [
{
'wildcard': {
'course.name': 'abc*'
'course.title': 'abc*'
}
}
]
......@@ -26,7 +27,7 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
}
}
self.catalog = factories.CatalogFactory(query=json.dumps(query))
self.course = CourseFactory(id='a/b/c', name='ABCs of Ͳҽʂէìղց')
self.course = CourseFactory(key='a/b/c', title='ABCs of Ͳҽʂէìղց')
self.refresh_index()
def test_unicode(self):
......@@ -38,11 +39,16 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
expected = 'Catalog #{id}: {name}'.format(id=self.catalog.id, name=name)
self.assertEqual(str(self.catalog), expected)
@skip('Skip until searching in ES is resolved')
def test_courses(self):
""" Verify the method returns a list of courses contained in the catalog. """
self.assertEqual(self.catalog.courses(), [self.course])
@skip('Skip until searching in ES is resolved')
def test_contains(self):
""" Verify the method returns a mapping of course IDs to booleans. """
other_id = 'd/e/f'
self.assertDictEqual(self.catalog.contains([self.course.id, other_id]), {self.course.id: True, other_id: False})
self.assertDictEqual(
self.catalog.contains([self.course.key, other_id]),
{self.course.key: True, other_id: False}
)
course_discovery/apps/core/tests/mixins.py
......@@ -3,7 +3,7 @@ import logging
from django.conf import settings
from elasticsearch import Elasticsearch
from course_discovery.apps.course_metadata.utils import ElasticsearchUtils
from course_discovery.apps.core.utils import ElasticsearchUtils
logger = logging.getLogger(__name__)
......
course_discovery/apps/core/tests/test_throttles.py
......@@ -15,7 +15,7 @@ class RateLimitingTest(APITestCase):
def setUp(self):
super(RateLimitingTest, self).setUp()
self.url = reverse('api:v1:course-list')
self.url = reverse('api:v1:catalog-list')
self.user = UserFactory()
self.client.login(username=self.user.username, password=USER_PASSWORD)
......
course_discovery/apps/course_metadata/utils.py → course_discovery/apps/core/utils.py
import datetime
import logging
from course_discovery.apps.course_metadata.config import COURSES_INDEX_CONFIG
logger = logging.getLogger(__name__)
......@@ -18,7 +16,7 @@ class ElasticsearchUtils(object):
# Create an index with a unique (timestamped) name
timestamp = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
index = '{alias}_{timestamp}'.format(alias=alias, timestamp=timestamp)
es.indices.create(index=index, body=COURSES_INDEX_CONFIG)
es.indices.create(index=index)
logger.info('...index [%s] created.', index)
# Point the alias to the new index
......
course_discovery/apps/course_metadata/admin.py 0 → 100644
from django.contrib import admin
from course_discovery.apps.course_metadata.models import (
Seat, Image, Video, LevelType, Subject, Prerequisite, ExpectedLearningItem, Course, CourseRun, Organization, Person,
CourseOrganization, SyllabusItem
)
class CourseOrganizationInline(admin.TabularInline):
model = CourseOrganization
extra = 1
class SeatInline(admin.TabularInline):
model = Seat
extra = 1
@admin.register(Course)
class CourseAdmin(admin.ModelAdmin):
inlines = (CourseOrganizationInline,)
@admin.register(CourseRun)
class CourseRunAdmin(admin.ModelAdmin):
inlines = (SeatInline,)
# Register all models using basic ModelAdmin classes
models = (Image, Video, LevelType, Subject, Prerequisite, ExpectedLearningItem, Organization, Person, SyllabusItem)
for model in models:
admin.site.register(model)
course_discovery/apps/course_metadata/apps.py
......@@ -2,5 +2,5 @@ from django.apps import AppConfig
class CourseMetadataConfig(AppConfig):
name = 'course_metadata'
name = 'course_discovery.apps.course_metadata'
verbose_name = 'Course Metadata'
course_discovery/apps/course_metadata/config.py deleted 100644 → 0
COURSES_INDEX_CONFIG = {
'settings': {
'analysis': {
'analyzer': {
'lowercase_keyword': {
'tokenizer': 'keyword',
'filter': ['lowercase']
}
}
}
},
'mappings': {
'course': {
'properties': {
'id': {
'type': 'string',
'analyzer': 'lowercase_keyword'
},
'name': {
'type': 'string',
'analyzer': 'lowercase_keyword'
}
}
}
}
}
course_discovery/apps/course_metadata/constants.py
COURSE_ID_REGEX = r'[^/+]+(/|\+)[^/+]+(/|\+)[^/]+'
COURSE_ID_PATTERN = r'(?P<id>{})'.format(COURSE_ID_REGEX)
COURSE_ID_REGEX = r'[^/+]+(/|\+)[^/+]+'
COURSE_RUN_ID_REGEX = r'[^/+]+(/|\+)[^/+]+(/|\+)[^/]+'
course_discovery/apps/course_metadata/exceptions.py deleted 100644 → 0
class CourseNotFoundError(Exception):
""" The specified course was not found in the data store. """
pass
course_discovery/apps/course_metadata/management/commands/install_es_indexes.py
......@@ -4,7 +4,7 @@ from django.conf import settings
from django.core.management import BaseCommand
from elasticsearch import Elasticsearch
from course_discovery.apps.course_metadata.utils import ElasticsearchUtils
from course_discovery.apps.core.utils import ElasticsearchUtils
logger = logging.getLogger(__name__)
......
course_discovery/apps/course_metadata/management/commands/refresh_all_courses.py deleted 100644 → 0
import logging
from django.conf import settings
from django.core.management import BaseCommand
from edx_rest_api_client.client import EdxRestApiClient
from course_discovery.apps.course_metadata.models import Course
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'Refresh course data from external sources.'
def add_arguments(self, parser):
parser.add_argument(
'--access_token',
action='store',
dest='access_token',
default=None,
help='OAuth2 access token used to authenticate API calls.'
)
def handle(self, *args, **options):
access_token = options.get('access_token')
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT),
settings.SOCIAL_AUTH_EDX_OIDC_KEY,
settings.SOCIAL_AUTH_EDX_OIDC_SECRET
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
Course.refresh_all(access_token=access_token)
course_discovery/apps/course_metadata/migrations/0001_initial.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import sortedm2m.fields
import django.db.models.deletion
from django.conf import settings
import django_extensions.db.fields
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core', '0004_currency'),
('ietf_language_tags', '0002_language_tag_data_migration'),
]
operations = [
migrations.CreateModel(
name='AbstractMediaModel',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('src', models.URLField(max_length=255, unique=True)),
('description', models.CharField(max_length=255, blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='Course',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True, db_index=True)),
('title', models.CharField(max_length=255, default=None, blank=True, null=True)),
('short_description', models.CharField(max_length=255, default=None, blank=True, null=True)),
('full_description', models.TextField(default=None, blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='CourseOrganization',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('relation_type', models.CharField(choices=[('owner', 'Owner'), ('sponsor', 'Sponsor')], max_length=63)),
('course', models.ForeignKey(to='course_metadata.Course')),
],
),
migrations.CreateModel(
name='CourseRun',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True)),
('title_override', models.CharField(max_length=255, default=None, help_text="Title specific for this run of a course. Leave this value blank to default to the parent course's title.", blank=True, null=True)),
('start', models.DateTimeField(blank=True, null=True)),
('end', models.DateTimeField(blank=True, null=True)),
('enrollment_start', models.DateTimeField(blank=True, null=True)),
('enrollment_end', models.DateTimeField(blank=True, null=True)),
('announcement', models.DateTimeField(blank=True, null=True)),
('short_description_override', models.CharField(max_length=255, default=None, help_text="Short description specific for this run of a course. Leave this value blank to default to the parent course's short_description attribute.", blank=True, null=True)),
('full_description_override', models.TextField(default=None, help_text="Full description specific for this run of a course. Leave this value blank to default to the parent course's full_description attribute.", blank=True, null=True)),
('min_effort', models.PositiveSmallIntegerField(help_text='Estimated minimum number of hours per week needed to complete a course run.', blank=True, null=True)),
('max_effort', models.PositiveSmallIntegerField(help_text='Estimated maximum number of hours per week needed to complete a course run.', blank=True, null=True)),
('pacing_type', models.CharField(choices=[('self_paced', 'Self-paced'), ('instructor_paced', 'Instructor-paced')], max_length=255, db_index=True, blank=True, null=True)),
('course', models.ForeignKey(to='course_metadata.Course')),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='ExpectedLearningItem',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('value', models.CharField(max_length=255)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='HistoricalCourse',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('title', models.CharField(max_length=255, default=None, blank=True, null=True)),
('short_description', models.CharField(max_length=255, default=None, blank=True, null=True)),
('full_description', models.TextField(default=None, blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical course',
},
),
migrations.CreateModel(
name='HistoricalCourseRun',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('title_override', models.CharField(max_length=255, default=None, help_text="Title specific for this run of a course. Leave this value blank to default to the parent course's title.", blank=True, null=True)),
('start', models.DateTimeField(blank=True, null=True)),
('end', models.DateTimeField(blank=True, null=True)),
('enrollment_start', models.DateTimeField(blank=True, null=True)),
('enrollment_end', models.DateTimeField(blank=True, null=True)),
('announcement', models.DateTimeField(blank=True, null=True)),
('short_description_override', models.CharField(max_length=255, default=None, help_text="Short description specific for this run of a course. Leave this value blank to default to the parent course's short_description attribute.", blank=True, null=True)),
('full_description_override', models.TextField(default=None, help_text="Full description specific for this run of a course. Leave this value blank to default to the parent course's full_description attribute.", blank=True, null=True)),
('min_effort', models.PositiveSmallIntegerField(help_text='Estimated minimum number of hours per week needed to complete a course run.', blank=True, null=True)),
('max_effort', models.PositiveSmallIntegerField(help_text='Estimated maximum number of hours per week needed to complete a course run.', blank=True, null=True)),
('pacing_type', models.CharField(choices=[('self_paced', 'Self-paced'), ('instructor_paced', 'Instructor-paced')], max_length=255, db_index=True, blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('course', models.ForeignKey(db_constraint=False, to='course_metadata.Course', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
('language', models.ForeignKey(db_constraint=False, to='ietf_language_tags.LanguageTag', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical course run',
},
),
migrations.CreateModel(
name='HistoricalOrganization',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('description', models.TextField(blank=True, null=True)),
('homepage_url', models.URLField(max_length=255, blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical organization',
},
),
migrations.CreateModel(
name='HistoricalPerson',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('title', models.CharField(max_length=255, blank=True, null=True)),
('bio', models.TextField(blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical person',
},
),
migrations.CreateModel(
name='HistoricalSeat',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('type', models.CharField(choices=[('honor', 'Honor'), ('audit', 'Audit'), ('verified', 'Verified'), ('professional', 'Professional'), ('credit', 'Credit')], max_length=63)),
('price', models.DecimalField(decimal_places=2, max_digits=10)),
('upgrade_deadline', models.DateTimeField()),
('credit_provider', models.CharField(max_length=255)),
('credit_hours', models.IntegerField()),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('course_run', models.ForeignKey(db_constraint=False, to='course_metadata.CourseRun', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
('currency', models.ForeignKey(db_constraint=False, to='core.Currency', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical seat',
},
),
migrations.CreateModel(
name='LevelType',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('name', models.CharField(max_length=255, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Organization',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('description', models.TextField(blank=True, null=True)),
('homepage_url', models.URLField(max_length=255, blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='Person',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('title', models.CharField(max_length=255, blank=True, null=True)),
('bio', models.TextField(blank=True, null=True)),
('organizations', models.ManyToManyField(to='course_metadata.Organization', blank=True)),
],
options={
'verbose_name_plural': 'People',
},
),
migrations.CreateModel(
name='Prerequisite',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('name', models.CharField(max_length=255, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Seat',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('type', models.CharField(choices=[('honor', 'Honor'), ('audit', 'Audit'), ('verified', 'Verified'), ('professional', 'Professional'), ('credit', 'Credit')], max_length=63)),
('price', models.DecimalField(decimal_places=2, max_digits=10)),
('upgrade_deadline', models.DateTimeField()),
('credit_provider', models.CharField(max_length=255)),
('credit_hours', models.IntegerField()),
('course_run', models.ForeignKey(to='course_metadata.CourseRun', related_name='seats')),
('currency', models.ForeignKey(to='core.Currency')),
],
),
migrations.CreateModel(
name='Subject',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('name', models.CharField(max_length=255, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='SyllabusItem',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('value', models.CharField(max_length=255)),
('parent', models.ForeignKey(to='course_metadata.SyllabusItem', null=True, blank=True, related_name='children')),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='Image',
fields=[
('abstractmediamodel_ptr', models.OneToOneField(serialize=False, parent_link=True, to='course_metadata.AbstractMediaModel', primary_key=True, auto_created=True)),
('height', models.IntegerField(blank=True, null=True)),
('width', models.IntegerField(blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
bases=('course_metadata.abstractmediamodel',),
),
migrations.CreateModel(
name='Video',
fields=[
('abstractmediamodel_ptr', models.OneToOneField(serialize=False, parent_link=True, to='course_metadata.AbstractMediaModel', primary_key=True, auto_created=True)),
('image', models.ForeignKey(to='course_metadata.Image')),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
bases=('course_metadata.abstractmediamodel',),
),
migrations.AddField(
model_name='historicalcourserun',
name='syllabus',
field=models.ForeignKey(db_constraint=False, to='course_metadata.SyllabusItem', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourse',
name='level_type',
field=models.ForeignKey(db_constraint=False, to='course_metadata.LevelType', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='courserun',
name='instructors',
field=sortedm2m.fields.SortedManyToManyField(to='course_metadata.Person', help_text=None, blank=True, related_name='courses_instructed'),
),
migrations.AddField(
model_name='courserun',
name='language',
field=models.ForeignKey(to='ietf_language_tags.LanguageTag', null=True, blank=True),
),
migrations.AddField(
model_name='courserun',
name='staff',
field=sortedm2m.fields.SortedManyToManyField(to='course_metadata.Person', help_text=None, blank=True, related_name='courses_staffed'),
),
migrations.AddField(
model_name='courserun',
name='syllabus',
field=models.ForeignKey(default=None, null=True, to='course_metadata.SyllabusItem', blank=True),
),
migrations.AddField(
model_name='courserun',
name='transcript_languages',
field=models.ManyToManyField(to='ietf_language_tags.LanguageTag', blank=True, related_name='transcript_courses'),
),
migrations.AddField(
model_name='courseorganization',
name='organization',
field=models.ForeignKey(to='course_metadata.Organization'),
),
migrations.AddField(
model_name='course',
name='expected_learning_items',
field=sortedm2m.fields.SortedManyToManyField(to='course_metadata.ExpectedLearningItem', help_text=None, blank=True),
),
migrations.AddField(
model_name='course',
name='level_type',
field=models.ForeignKey(default=None, null=True, to='course_metadata.LevelType', blank=True),
),
migrations.AddField(
model_name='course',
name='organizations',
field=models.ManyToManyField(to='course_metadata.Organization', through='course_metadata.CourseOrganization', blank=True),
),
migrations.AddField(
model_name='course',
name='prerequisites',
field=models.ManyToManyField(to='course_metadata.Prerequisite', blank=True),
),
migrations.AddField(
model_name='course',
name='subjects',
field=models.ManyToManyField(to='course_metadata.Subject', blank=True),
),
migrations.AlterUniqueTogether(
name='seat',
unique_together=set([('course_run', 'type', 'currency', 'credit_provider')]),
),
migrations.AddField(
model_name='person',
name='profile_image',
field=models.ForeignKey(to='course_metadata.Image', null=True, blank=True),
),
migrations.AddField(
model_name='organization',
name='logo_image',
field=models.ForeignKey(to='course_metadata.Image', null=True, blank=True),
),
migrations.AddField(
model_name='historicalperson',
name='profile_image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalorganization',
name='logo_image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourserun',
name='image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourserun',
name='video',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Video', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourse',
name='image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourse',
name='video',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Video', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='courserun',
name='image',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Image', blank=True),
),
migrations.AddField(
model_name='courserun',
name='video',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Video', blank=True),
),
migrations.AlterUniqueTogether(
name='courseorganization',
unique_together=set([('course', 'relation_type', 'relation_type')]),
),
migrations.AlterIndexTogether(
name='courseorganization',
index_together=set([('course', 'relation_type')]),
),
migrations.AddField(
model_name='course',
name='image',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Image', blank=True),
),
migrations.AddField(
model_name='course',
name='video',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Video', blank=True),
),
]
course_discovery/apps/course_metadata/models.py
import logging
from django.conf import settings
from edx_rest_api_client.client import EdxRestApiClient
from elasticsearch import Elasticsearch, NotFoundError
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django_extensions.db.models import TimeStampedModel
from simple_history.models import HistoricalRecords
from sortedm2m.fields import SortedManyToManyField
from course_discovery.apps.course_metadata.exceptions import CourseNotFoundError
from course_discovery.apps.core.models import Currency
from course_discovery.apps.ietf_language_tags.models import LanguageTag
logger = logging.getLogger(__name__)
class Course(object):
"""
Course model.
This model is backed by Elasticsearch.
"""
# Elasticsearch document type for courses.
doc_type = 'course'
# Elasticsearch index where course data is stored
_index = settings.ELASTICSEARCH['index']
@classmethod
def _es_client(cls):
""" Elasticsearch client. """
return Elasticsearch(settings.ELASTICSEARCH['host'])
@classmethod
def _hit_to_course(cls, hit):
return Course(hit['_source']['id'], hit['_source'])
@classmethod
def all(cls, limit=10, offset=0):
"""
Return a list of all courses.
Args:
limit (int): Maximum number of results to return
offset (int): Starting index from which to return results
Returns:
dict: Representation of data suitable for pagination
Examples:
{
'limit': 10,
'offset': 0,
'total': 2,
'results': [`Course`, `Course`],
}
"""
query = {
'query': {
'match_all': {}
}
}
return cls.search(query, limit=limit, offset=offset)
@classmethod
def get(cls, id): # pylint: disable=redefined-builtin
"""
Retrieve a single course.
Args:
id (str): Course ID
Returns:
Course: The course corresponding to the given ID.
Raises:
CourseNotFoundError: if the course is not found.
"""
try:
response = cls._es_client().get(index=cls._index, doc_type=cls.doc_type, id=id)
return cls._hit_to_course(response)
except NotFoundError:
raise CourseNotFoundError('Course [{}] was not found in the data store.'.format(id))
@classmethod
def search(cls, query, limit=10, offset=0):
"""
Search the data store for courses.
Args:
query (dict): Elasticsearch query used to find courses.
limit (int): Maximum number of results to return
offset (int): Index of first result to return
Returns:
dict: Representation of data suitable for pagination
Examples:
{
'limit': 10,
'offset': 0,
'total': 2,
'results': [`Course`, `Course`],
}
"""
query.setdefault('from', offset)
query.setdefault('size', limit)
query.setdefault('sort', {'id': 'asc'})
logger.debug('Querying [%s]: %s', cls._index, query)
response = cls._es_client().search(index=cls._index, doc_type=cls.doc_type, body=query)
hits = response['hits']
total = hits['total']
logger.info('Course search returned [%d] courses.', total)
return {
'limit': limit,
'offset': offset,
'total': total,
'results': [cls._hit_to_course(hit) for hit in hits['hits']]
}
@classmethod
def refresh(cls, course_id, access_token):
"""
Refresh the course data from the raw data sources.
Args:
course_id (str): Course ID
access_token (str): OAuth access token
Returns:
Course
"""
client = EdxRestApiClient(settings.ECOMMERCE_API_URL, oauth_access_token=access_token)
body = client.courses(course_id).get(include_products=True)
course = Course(course_id, body)
course.save()
return course
@classmethod
def refresh_all(cls, access_token):
"""
Refresh all course data.
Args:
access_token (str): OAuth access token
Returns:
None
"""
cls.refresh_all_ecommerce_data(access_token)
cls.refresh_all_course_api_data(access_token)
@classmethod
def refresh_all_ecommerce_data(cls, access_token):
ecommerce_api_url = settings.ECOMMERCE_API_URL
client = EdxRestApiClient(ecommerce_api_url, oauth_access_token=access_token)
count = None
page = 1
logger.info('Refreshing ecommerce data from %s....', ecommerce_api_url)
while page:
response = client.courses().get(include_products=True, page=page, page_size=50)
count = response['count']
results = response['results']
logger.info('Retrieved %d courses...', len(results))
if response['next']:
page += 1
else:
page = None
for body in results:
Course(body['id']).update(body)
logger.info('Retrieved %d courses from %s.', count, ecommerce_api_url)
@classmethod
def refresh_all_course_api_data(cls, access_token):
course_api_url = settings.COURSES_API_URL
client = EdxRestApiClient(course_api_url, oauth_access_token=access_token)
count = None
page = 1
logger.info('Refreshing course api data from %s....', course_api_url)
while page:
# TODO Update API to not require username?
response = client.courses().get(page=page, page_size=50, username='ecommerce_worker')
count = response['pagination']['count']
results = response['results']
logger.info('Retrieved %d courses...', len(results))
if response['pagination']['next']:
page += 1
else:
page = None
for body in results:
Course(body['id']).update(body)
logger.info('Retrieved %d courses from %s.', count, course_api_url)
def __init__(self, id, body=None): # pylint: disable=redefined-builtin
if not id:
raise ValueError('Course ID cannot be empty or None.')
self.id = id
self.body = body or {}
def __eq__(self, other):
"""
Determine if this Course equals another.
Args:
other (Course): object with which to compare
Returns: True iff. the two Course objects have the same `id` value; otherwise, False.
class AbstractNamedModel(TimeStampedModel):
""" Abstract base class for models with only a name field. """
name = models.CharField(max_length=255, unique=True)
"""
return self.id is not None \
and isinstance(other, Course) \
and self.id == getattr(other, 'id', None) \
and self.body == getattr(other, 'body', None)
def __repr__(self):
return 'Course {id}: {name}'.format(id=self.id, name=self.name)
def __str__(self):
return self.name
class Meta(object):
abstract = True
class AbstractMediaModel(TimeStampedModel):
""" Abstract base class for media-related (e.g. image, video) models. """
src = models.URLField(max_length=255, unique=True)
description = models.CharField(max_length=255, null=True, blank=True)
def __str__(self):
return self.src
class Image(AbstractMediaModel):
""" Image model. """
height = models.IntegerField(null=True, blank=True)
width = models.IntegerField(null=True, blank=True)
class Video(AbstractMediaModel):
""" Video model. """
image = models.ForeignKey(Image)
class LevelType(AbstractNamedModel):
""" LevelType model. """
pass
class Subject(AbstractNamedModel):
""" Subject model. """
pass
class Prerequisite(AbstractNamedModel):
""" Prerequisite model. """
pass
class ExpectedLearningItem(TimeStampedModel):
""" ExpectedLearningItem model. """
value = models.CharField(max_length=255)
class SyllabusItem(TimeStampedModel):
""" SyllabusItem model. """
parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
value = models.CharField(max_length=255)
class Organization(TimeStampedModel):
""" Organization model. """
key = models.CharField(max_length=255, unique=True)
name = models.CharField(max_length=255, null=True, blank=True)
description = models.TextField(null=True, blank=True)
homepage_url = models.URLField(max_length=255, null=True, blank=True)
logo_image = models.ForeignKey(Image, null=True, blank=True)
history = HistoricalRecords()
def __str__(self):
return '{key}: {name}'.format(key=self.key, name=self.name)
class Person(TimeStampedModel):
""" Person model. """
key = models.CharField(max_length=255, unique=True)
name = models.CharField(max_length=255, null=True, blank=True)
title = models.CharField(max_length=255, null=True, blank=True)
bio = models.TextField(null=True, blank=True)
profile_image = models.ForeignKey(Image, null=True, blank=True)
organizations = models.ManyToManyField(Organization, blank=True)
history = HistoricalRecords()
def __str__(self):
return '{key}: {name}'.format(key=self.key, name=self.name)
class Meta(object):
verbose_name_plural = 'People'
class Course(TimeStampedModel):
""" Course model. """
key = models.CharField(max_length=255, db_index=True, unique=True)
title = models.CharField(max_length=255, default=None, null=True, blank=True)
short_description = models.CharField(max_length=255, default=None, null=True, blank=True)
full_description = models.TextField(default=None, null=True, blank=True)
organizations = models.ManyToManyField('Organization', through='CourseOrganization', blank=True)
subjects = models.ManyToManyField(Subject, blank=True)
prerequisites = models.ManyToManyField(Prerequisite, blank=True)
level_type = models.ForeignKey(LevelType, default=None, null=True, blank=True)
expected_learning_items = SortedManyToManyField(ExpectedLearningItem, blank=True)
image = models.ForeignKey(Image, default=None, null=True, blank=True)
video = models.ForeignKey(Video, default=None, null=True, blank=True)
history = HistoricalRecords()
def __str__(self):
return '{key}: {title}'.format(key=self.key, title=self.title)
class CourseRun(TimeStampedModel):
""" CourseRun model. """
SELF_PACED = 'self_paced'
INSTRUCTOR_PACED = 'instructor_paced'
PACING_CHOICES = (
# Translators: Self-paced refers to course runs that operate on the student's schedule.
(SELF_PACED, _('Self-paced')),
# Translators: Instructor-paced refers to course runs that operate on a schedule set by the instructor,
# similar to a normal university course.
(INSTRUCTOR_PACED, _('Instructor-paced')),
)
course = models.ForeignKey(Course)
key = models.CharField(max_length=255, unique=True)
title_override = models.CharField(
max_length=255, default=None, null=True, blank=True,
help_text=_(
"Title specific for this run of a course. Leave this value blank to default to the parent course's title."))
start = models.DateTimeField(null=True, blank=True)
end = models.DateTimeField(null=True, blank=True)
enrollment_start = models.DateTimeField(null=True, blank=True)
enrollment_end = models.DateTimeField(null=True, blank=True)
announcement = models.DateTimeField(null=True, blank=True)
short_description_override = models.CharField(
max_length=255, default=None, null=True, blank=True,
help_text=_(
"Short description specific for this run of a course. Leave this value blank to default to "
"the parent course's short_description attribute."))
full_description_override = models.TextField(
default=None, null=True, blank=True,
help_text=_(
"Full description specific for this run of a course. Leave this value blank to default to "
"the parent course's full_description attribute."))
instructors = SortedManyToManyField(Person, blank=True, related_name='courses_instructed')
staff = SortedManyToManyField(Person, blank=True, related_name='courses_staffed')
min_effort = models.PositiveSmallIntegerField(
null=True, blank=True,
help_text=_('Estimated minimum number of hours per week needed to complete a course run.'))
max_effort = models.PositiveSmallIntegerField(
null=True, blank=True,
help_text=_('Estimated maximum number of hours per week needed to complete a course run.'))
language = models.ForeignKey(LanguageTag, null=True, blank=True)
transcript_languages = models.ManyToManyField(LanguageTag, blank=True, related_name='transcript_courses')
pacing_type = models.CharField(max_length=255, choices=PACING_CHOICES, db_index=True, null=True, blank=True)
syllabus = models.ForeignKey(SyllabusItem, default=None, null=True, blank=True)
image = models.ForeignKey(Image, default=None, null=True, blank=True)
video = models.ForeignKey(Video, default=None, null=True, blank=True)
history = HistoricalRecords()
@property
def title(self):
return self.title_override or self.course.title
@title.setter
def title(self, value):
# Treat empty strings as NULL
value = value or None
self.title_override = value
@property
def short_description(self):
return self.short_description_override or self.course.short_description
@short_description.setter
def short_description(self, value):
# Treat empty strings as NULL
value = value or None
self.short_description_override = value
@property
def name(self):
return self.body.get('name')
def save(self):
""" Save the course to the data store. """
logger.info('Indexing course %s...', self.id)
self._es_client().index(index=self._index, doc_type=self.doc_type, id=self.id, body=self.body)
logger.info('Finished indexing course %s.', self.id)
def update(self, body):
""" Updates (merges) the data in the index with the provided data.
Args:
body (dict): Data to be merged into the index.
Returns:
None
"""
body = {
'doc': body,
'doc_as_upsert': True,
}
logger.info('Updating course %s...', self.id)
self._es_client().update(index=self._index, doc_type=self.doc_type, id=self.id, body=body)
logger.info('Finished updating course %s.', self.id)
def full_description(self):
return self.full_description_override or self.course.full_description
@full_description.setter
def full_description(self, value):
# Treat empty strings as NULL
value = value or None
self.full_description_override = value
def __str__(self):
return '{key}: {title}'.format(key=self.key, title=self.title)
class Seat(TimeStampedModel):
""" Seat model. """
HONOR = 'honor'
AUDIT = 'audit'
VERIFIED = 'verified'
PROFESSIONAL = 'professional'
CREDIT = 'credit'
SEAT_TYPE_CHOICES = (
(HONOR, _('Honor')),
(AUDIT, _('Audit')),
(VERIFIED, _('Verified')),
(PROFESSIONAL, _('Professional')),
(CREDIT, _('Credit')),
)
course_run = models.ForeignKey(CourseRun, related_name='seats')
type = models.CharField(max_length=63, choices=SEAT_TYPE_CHOICES)
price = models.DecimalField(decimal_places=2, max_digits=10)
currency = models.ForeignKey(Currency)
upgrade_deadline = models.DateTimeField()
credit_provider = models.CharField(max_length=255)
credit_hours = models.IntegerField()
history = HistoricalRecords()
class Meta(object):
unique_together = (
('course_run', 'type', 'currency', 'credit_provider')
)
class CourseOrganization(TimeStampedModel):
""" CourseOrganization model. """
OWNER = 'owner'
SPONSOR = 'sponsor'
RELATION_TYPE_CHOICES = (
(OWNER, _('Owner')),
(SPONSOR, _('Sponsor')),
)
course = models.ForeignKey(Course)
organization = models.ForeignKey(Organization)
relation_type = models.CharField(max_length=63, choices=RELATION_TYPE_CHOICES)
class Meta(object):
index_together = (
('course', 'relation_type'),
)
unique_together = (
('course', 'relation_type', 'relation_type'),
)
course_discovery/apps/course_metadata/tests/factories.py
import factory
from factory.fuzzy import FuzzyText
from course_discovery.apps.course_metadata.models import Course
from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Person
class CourseFactory(factory.Factory):
class Meta(object):
class CourseFactory(factory.DjangoModelFactory):
key = FuzzyText(prefix='course-id/')
title = FuzzyText(prefix="Test çօմɾʂҽ ")
short_description = FuzzyText(prefix="Test çօմɾʂҽ short description")
full_description = FuzzyText(prefix="Test çօմɾʂҽ FULL description")
class Meta:
model = Course
exclude = ('name',)
id = FuzzyText(prefix='course-id/', suffix='/fake')
name = FuzzyText(prefix="էҽʂէ çօմɾʂҽ ")
@factory.lazy_attribute
def body(self):
return {
'id': self.id,
'name': self.name
}
@classmethod
def _create(cls, model_class, *args, **kwargs):
obj = model_class(*args, **kwargs)
obj.save()
return obj
class CourseRunFactory(factory.DjangoModelFactory):
key = FuzzyText(prefix='course-run-id/', suffix='/fake')
course = factory.SubFactory(CourseFactory)
title_override = None
short_description_override = None
full_description_override = None
class Meta:
model = CourseRun
class OrganizationFactory(factory.DjangoModelFactory):
key = FuzzyText(prefix='Org.fake/')
name = FuzzyText()
class Meta:
model = Organization
class PersonFactory(factory.DjangoModelFactory):
key = FuzzyText(prefix='Person.fake/')
name = FuzzyText()
title = FuzzyText()
bio = FuzzyText()
class Meta:
model = Person
course_discovery/apps/course_metadata/tests/test_models.py
import json
from urllib.parse import urlparse, parse_qs
import ddt
from django.test import TestCase
import responses
from django.test import TestCase, override_settings
from course_discovery.apps.course_metadata.models import AbstractNamedModel, AbstractMediaModel
from course_discovery.apps.course_metadata.tests import factories
from course_discovery.apps.core.tests.mixins import ElasticsearchTestMixin
from course_discovery.apps.course_metadata.exceptions import CourseNotFoundError
from course_discovery.apps.course_metadata.models import Course
from course_discovery.apps.course_metadata.tests.factories import CourseFactory
ACCESS_TOKEN = 'secret'
COURSES_API_URL = 'https://lms.example.com/api/courses/v1'
ECOMMERCE_API_URL = 'https://ecommerce.example.com/api/v2'
JSON = 'application/json'
@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL, COURSES_API_URL=COURSES_API_URL)
class CourseTests(ElasticsearchTestMixin, TestCase):
def assert_course_attrs(self, course, attrs):
"""
Validate the attributes of a given Course.
Args:
course (Course)
attrs (dict)
"""
for attr, value in attrs.items():
self.assertEqual(getattr(course, attr), value)
@responses.activate
def mock_refresh_all(self):
"""
Mock the external APIs and refresh all course data.
Returns:
[dict]: List of dictionaries representing course content bodies.
"""
course_bodies = [
{
'id': 'a/b/c',
'url': 'https://ecommerce.example.com/api/v2/courses/a/b/c/',
'name': 'aaaaa',
'verification_deadline': '2022-01-01T01:00:00Z',
'type': 'verified',
'last_edited': '2015-08-19T15:47:24Z'
},
{
'id': 'aaa/bbb/ccc',
'url': 'https://ecommerce.example.com/api/v2/courses/aaa/bbb/ccc/',
'name': 'Introduction to Biology - The Secret of Life',
'verification_deadline': None,
'type': 'audit',
'last_edited': '2015-08-06T19:11:19Z'
}
]
def ecommerce_api_callback(url, data):
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
if page < count:
next = '{}?page={}'.format(url, page)
body = {
'count': count,
'next': next,
'previous': None,
'results': [data[page - 1]]
}
return 200, {}, json.dumps(body)
return request_callback
def courses_api_callback(url, data):
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
if page < count:
next = '{}?page={}'.format(url, page)
body = {
'pagination': {
'count': count,
'next': next,
'previous': None,
},
'results': [data[page - 1]]
}
return 200, {}, json.dumps(body)
return request_callback
url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
responses.add_callback(responses.GET, url, callback=ecommerce_api_callback(url, course_bodies),
content_type=JSON)
url = '{host}/courses/'.format(host=COURSES_API_URL)
responses.add_callback(responses.GET, url, callback=courses_api_callback(url, course_bodies), content_type=JSON)
# Refresh all course data
Course.refresh_all(ACCESS_TOKEN)
self.refresh_index()
return course_bodies
def test_init(self):
""" Verify the constructor requires a non-empty string for the ID. """
msg = 'Course ID cannot be empty or None.'
with self.assertRaisesRegex(ValueError, msg):
Course(None)
with self.assertRaisesRegex(ValueError, msg):
Course('')
def test_eq(self):
""" Verify the __eq__ method returns True if two Course objects have the same `id`. """
course = CourseFactory()
# Both objects must be of type Course
self.assertNotEqual(course, 1)
# A Course should be equal to itself
self.assertEqual(course, course)
# Two Courses are equal if their id attributes match
self.assertEqual(course, Course(id=course.id, body=course.body))
class CourseTests(TestCase):
""" Tests for the `Course` model. """
def test_str(self):
""" Verify the __str__ method returns a string representation of the Course. """
course = CourseFactory()
expected = 'Course {id}: {name}'.format(id=course.id, name=course.name)
self.assertEqual(str(course), expected)
def test_all(self):
""" Verify the method returns a list of all courses. """
course_bodies = self.mock_refresh_all()
courses = []
for body in course_bodies:
courses.append(Course.get(body['id']))
""" Verify casting an instance to a string returns a string containing the key and title. """
course = factories.CourseFactory()
self.assertEqual(str(course), '{key}: {title}'.format(key=course.key, title=course.title))
expected = {
'limit': 10,
'offset': 0,
'total': 2,
'results': courses,
}
self.assertDictEqual(Course.all(), expected)
@ddt.ddt
class CourseRunTests(TestCase):
""" Tests for the `CourseRun` model. """
def test_all_with_limit_and_offset(self):
""" Verify the method supports limit-offset pagination. """
limit = 1
courses = [CourseFactory(id='1'), CourseFactory(id='2')]
self.refresh_index()
def setUp(self):
super(CourseRunTests, self).setUp()
self.course_run = factories.CourseRunFactory()
for offset, course in enumerate(courses):
expected = {
'limit': limit,
'offset': offset,
'total': len(courses),
'results': [course],
}
self.assertDictEqual(Course.all(limit=limit, offset=offset), expected)
def test_get(self):
""" Verify the method returns a single course. """
course = CourseFactory()
retrieved = Course.get(course.id)
self.assertEqual(course, retrieved)
def test_str(self):
""" Verify casting an instance to a string returns a string containing the key and title. """
course_run = self.course_run
# pylint: disable=no-member
self.assertEqual(str(course_run), '{key}: {title}'.format(key=course_run.key, title=course_run.title))
@ddt.data('title', 'short_description', 'full_description')
def test_override_fields(self, field_name):
""" Verify the `CourseRun`'s override field overrides the related `Course`'s field. """
override_field_name = "{}_override".format(field_name)
self.assertIsNone(getattr(self.course_run, override_field_name))
self.assertEqual(getattr(self.course_run, field_name), getattr(self.course_run.course, field_name))
# Setting the property to a non-empty value should set the override field,
# and trigger the field property getter to use the override.
override_text = 'A Better World'
setattr(self.course_run, field_name, override_text)
self.assertEqual(getattr(self.course_run, override_field_name), override_text)
self.assertEqual(getattr(self.course_run, field_name), override_text)
# Setting the title property to an empty value should set the title_override field to None,
# and trigger the title property getter to use the title of the parent course.
setattr(self.course_run, field_name, None)
self.assertIsNone(getattr(self.course_run, override_field_name))
self.assertEqual(getattr(self.course_run, field_name), getattr(self.course_run.course, field_name))
class OrganizationTests(TestCase):
""" Tests for the `Organization` model. """
def test_get_with_missing_course(self):
"""
Verify the method raises a CourseNotFoundError if the specified course does not exist in the data store.
"""
# Note (CCB): This consistently fails on Travis with the error below. Trying index refresh as a last-ditch
# effort to resolve.
#
# elasticsearch.exceptions.TransportError: TransportError(503,
# 'NoShardAvailableActionException[[course_discovery_test][1] null]; nested:
# IllegalIndexShardStateException[[course_discovery_test][1] CurrentState[POST_RECOVERY] operations only
# allowed when started/relocated]; ')
#
self.refresh_index()
course_id = 'fake.course'
expected_msg_regexp = r'Course \[{}\] was not found in the data store.'.format(course_id)
with self.assertRaisesRegex(CourseNotFoundError, expected_msg_regexp):
Course.get(course_id)
def test_str(self):
""" Verify casting an instance to a string returns a string containing the key and name. """
organization = factories.OrganizationFactory()
self.assertEqual(str(organization), '{key}: {name}'.format(key=organization.key, name=organization.name))
def test_search(self):
""" Verify the method returns query results from the data store. """
prefix = 'test'
query = {
'query': {
'bool': {
'must': [
{
'wildcard': {
'course.name': prefix + '*'
}
}
]
}
}
}
courses = []
for i in range(3):
courses.append(CourseFactory.create(name=prefix + str(i)))
CourseFactory.create()
courses.sort(key=lambda course: course.id.lower())
self.refresh_index()
class PersonTests(TestCase):
""" Tests for the `Person` model. """
expected = {
'limit': 10,
'offset': 0,
'total': len(courses),
'results': courses,
}
self.assertEqual(Course.search(query), expected)
def test_str(self):
""" Verify casting an instance to a string returns a string containing the key and name. """
person = factories.PersonFactory()
self.assertEqual(str(person), '{key}: {name}'.format(key=person.key, name=person.name))
@responses.activate
def test_refresh(self):
""" Verify the method refreshes data for a single course. """
course_id = 'SesameStreetX/Cookies/1T2016'
name = 'C is for Cookie'
body = {
'id': course_id,
'name': name
}
# Mock the call to the E-Commerce API
url = '{host}/courses/{course_id}/'.format(host=ECOMMERCE_API_URL, course_id=course_id)
responses.add(responses.GET, url, body=json.dumps(body), content_type=JSON)
class AbstractNamedModelTests(TestCase):
""" Tests for AbstractNamedModel. """
# Refresh the course, and ensure the attributes are correct.
course = Course.refresh(course_id, ACCESS_TOKEN)
attrs = {
'id': course_id,
'body': body,
'name': name,
}
self.assert_course_attrs(course, attrs)
def test_str(self):
""" Verify casting an instance to a string returns a string containing the name. """
# Ensure the data is persisted to the data store
course = Course.get(course_id)
self.assert_course_attrs(course, attrs)
class TestAbstractNamedModel(AbstractNamedModel):
pass
def test_refresh_all(self):
""" Verify the method refreshes data for all courses. """
course_bodies = self.mock_refresh_all()
self.refresh_index()
name = 'abc'
instance = TestAbstractNamedModel(name=name)
self.assertEqual(str(instance), name)
# Ensure the data is persisted to the data store
for body in course_bodies:
course_id = body['id']
attrs = {
'id': course_id,
'body': body,
'name': body['name'],
}
course = Course.get(course_id)
self.assert_course_attrs(course, attrs)
def test_name(self):
""" Verify the method returns the course name. """
name = 'ABC Course'
course = Course('a/b/c', {'name': name})
self.assertEqual(course.name, name)
class AbstractMediaModelTests(TestCase):
""" Tests for AbstractMediaModel. """
def test_save(self):
""" Verify the method creates and/or updates new courses. """
course_id = 'TestX/Saving/4T2015'
body = {
'id': course_id,
'name': 'Save Me!'
}
def test_str(self):
""" Verify casting an instance to a string returns a string containing the name. """
self.assertFalse(self.es.exists(index=self.index, doc_type=Course.doc_type, id=course_id))
Course(course_id, body).save()
self.refresh_index()
class TestAbstractMediaModel(AbstractMediaModel):
pass
self.assertTrue(self.es.exists(index=self.index, doc_type=Course.doc_type, id=course_id))
course = Course.get(course_id)
self.assertEqual(course.id, course_id)
self.assertEqual(course.body, body)
src = 'http://example.com/image.jpg'
instance = TestAbstractMediaModel(src=src)
self.assertEqual(str(instance), src)
course_discovery/apps/course_metadata/tests/test_refresh_all_courses.py deleted 100644 → 0
""" Tests for Refresh All Courses management command. """
from django.core.management import call_command
from django.test import TestCase
from django.test.utils import override_settings
from edx_rest_api_client.client import EdxRestApiClient
from mock import patch
from course_discovery.apps.course_metadata.models import Course
@override_settings(
SOCIAL_AUTH_EDX_OIDC_URL_ROOT="http://auth-url.com/oauth2",
SOCIAL_AUTH_EDX_OIDC_KEY="client_id",
SOCIAL_AUTH_EDX_OIDC_SECRET="client_secret"
)
class RefreshAllCoursesCommandTests(TestCase):
""" Tests for refresh_all_courses management command. """
cmd = 'refresh_all_courses'
def test_call_with_access_token(self):
""" Verify the management command calls Course.refresh_all() with access token. """
access_token = 'secret'
with patch.object(Course, 'refresh_all') as mock_refresh:
call_command(self.cmd, access_token=access_token)
mock_refresh.assert_called_once_with(access_token=access_token)
def test_call_with_client_credentials(self):
""" Verify the management command calls Course.refresh_all() with client credentials. """
access_token = 'secret'
with patch.object(EdxRestApiClient, 'get_oauth_access_token') as mock_access_token:
mock_access_token.return_value = (access_token, None)
with patch.object(Course, 'refresh_all') as mock_refresh:
call_command(self.cmd)
mock_refresh.assert_called_once_with(access_token=access_token)
def test_call_with_client_credentials_error(self):
""" Verify the command requires an access token to complete. """
with patch.object(EdxRestApiClient, 'get_oauth_access_token') as mock_access_token:
mock_access_token.side_effect = Exception()
with self.assertRaises(Exception):
call_command(self.cmd)
course_discovery/settings/base.py
......@@ -33,6 +33,8 @@ THIRD_PARTY_APPS = (
'rest_framework_swagger',
'social.apps.django_app.default',
'waffle',
'sortedm2m',
'simple_history',
)
PROJECT_APPS = (
......@@ -57,6 +59,7 @@ MIDDLEWARE_CLASSES = (
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'social.apps.django_app.middleware.SocialAuthExceptionMiddleware',
'waffle.middleware.WaffleMiddleware',
'simple_history.middleware.HistoryRequestMiddleware',
)
ROOT_URLCONF = 'course_discovery.urls'
......
course_discovery/settings/utils.py
from os import environ, path
import sys
from logging.handlers import SysLogHandler
from os import environ
from django.core.exceptions import ImproperlyConfigured
......@@ -12,4 +10,3 @@ def get_env_setting(setting):
except KeyError:
error_msg = "Set the [{}] env variable!".format(setting)
raise ImproperlyConfigured(error_msg)
requirements/base.txt
django==1.8.7
django-extensions==1.5.9
django-simple-history==1.8.1
django-sortedm2m==1.1.1
django-waffle==0.11
djangorestframework==3.3.1
djangorestframework-jwt==1.7.2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment