Skip to content

C
course-discovery
Commit b7e4f051 by Clinton Blackburn
Options

Merge pull request #49 from edx/feature/data-model-redux 

Updated Data Model
parents a9ffdd1e cbb8a529
Showing with 1294 additions and 864 deletions
course_discovery/apps/api/pagination.py deleted 100644 → 0
from rest_framework.pagination import LimitOffsetPagination
class ElasticsearchLimitOffsetPagination(LimitOffsetPagination):
def paginate_queryset(self, queryset, request, view=None):
"""
Convert a paginated Elasticsearch response to a response suitable for DRF.
Args:
queryset (dict): Elasticsearch response
request (Request): HTTP request
Returns:
List of data.
"""
# pylint: disable=attribute-defined-outside-init
self.limit = self.get_limit(request)
self.offset = self.get_offset(request)
self.count = queryset['total']
self.request = request
if self.count > self.limit and self.template is not None:
self.display_page_controls = True
return queryset['results']
course_discovery/apps/api/serializers.py
...@@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy as _ ...@@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers from rest_framework import serializers
from course_discovery.apps.catalogs.models import Catalog from course_discovery.apps.catalogs.models import Catalog
from course_discovery.apps.course_metadata.models import Course
class CatalogSerializer(serializers.ModelSerializer): class CatalogSerializer(serializers.ModelSerializer):
...@@ -12,10 +13,13 @@ class CatalogSerializer(serializers.ModelSerializer): ...@@ -12,10 +13,13 @@ class CatalogSerializer(serializers.ModelSerializer):
fields = ('id', 'name', 'query', 'url',) fields = ('id', 'name', 'query', 'url',)
class CourseSerializer(serializers.Serializer): # pylint: disable=abstract-method class CourseSerializer(serializers.ModelSerializer):
id = serializers.CharField(help_text=_('Course ID')) key = serializers.CharField()
name = serializers.CharField(help_text=_('Course name')) title = serializers.CharField()
url = serializers.HyperlinkedIdentityField(view_name='api:v1:course-detail', lookup_field='id')
class Meta(object):
model = Course
fields = ('key', 'title',)
class ContainedCoursesSerializer(serializers.Serializer): # pylint: disable=abstract-method class ContainedCoursesSerializer(serializers.Serializer): # pylint: disable=abstract-method
......
course_discovery/apps/api/tests/test_serializers.py
...@@ -25,14 +25,13 @@ class CatalogSerializerTests(TestCase): ...@@ -25,14 +25,13 @@ class CatalogSerializerTests(TestCase):
class CourseSerializerTests(TestCase): class CourseSerializerTests(TestCase):
def test_data(self): def test_data(self):
course = CourseFactory() course = CourseFactory()
path = reverse('api:v1:course-detail', kwargs={'id': course.id}) path = reverse('api:v1:course-detail', kwargs={'key': course.key})
request = RequestFactory().get(path) request = RequestFactory().get(path)
serializer = CourseSerializer(course, context={'request': request}) serializer = CourseSerializer(course, context={'request': request})
expected = { expected = {
'id': course.id, 'key': course.key,
'name': course.name, 'title': course.title,
'url': request.build_absolute_uri(),
} }
self.assertDictEqual(serializer.data, expected) self.assertDictEqual(serializer.data, expected)
......
course_discovery/apps/api/v1/tests/test_views.py
...@@ -70,21 +70,8 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi ...@@ -70,21 +70,8 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
super(CatalogViewSetTests, self).setUp() super(CatalogViewSetTests, self).setUp()
self.user = UserFactory(is_staff=True, is_superuser=True) self.user = UserFactory(is_staff=True, is_superuser=True)
self.client.login(username=self.user.username, password=USER_PASSWORD) self.client.login(username=self.user.username, password=USER_PASSWORD)
query = { self.catalog = CatalogFactory(query='title:abc*')
'query': { self.course = CourseFactory(key='a/b/c', title='ABC Test Course')
'bool': {
'must': [
{
'wildcard': {
'course.name': 'abc*'
}
}
]
}
}
}
self.catalog = CatalogFactory(query=json.dumps(query))
self.course = CourseFactory(id='a/b/c', name='ABC Test Course')
self.refresh_index() self.refresh_index()
def generate_jwt_token_header(self, user): def generate_jwt_token_header(self, user):
...@@ -164,13 +151,13 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi ...@@ -164,13 +151,13 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
def test_contains(self): def test_contains(self):
""" Verify the endpoint returns a filtered list of courses contained in the catalog. """ """ Verify the endpoint returns a filtered list of courses contained in the catalog. """
course_id = self.course.id course_key = self.course.key
qs = urllib.parse.urlencode({'course_id': course_id}) qs = urllib.parse.urlencode({'course_id': course_key})
url = '{}?{}'.format(reverse('api:v1:catalog-contains', kwargs={'id': self.catalog.id}), qs) url = '{}?{}'.format(reverse('api:v1:catalog-contains', kwargs={'id': self.catalog.id}), qs)
response = self.client.get(url) response = self.client.get(url)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertEqual(response.data, {'courses': {course_id: True}}) self.assertEqual(response.data, {'courses': {course_key: True}})
def test_get(self): def test_get(self):
""" Verify the endpoint returns the details for a single catalog. """ """ Verify the endpoint returns the details for a single catalog. """
...@@ -242,10 +229,9 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin ...@@ -242,10 +229,9 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
def test_list(self, format): def test_list(self, format):
""" Verify the endpoint returns a list of all courses. """ """ Verify the endpoint returns a list of all courses. """
courses = CourseFactory.create_batch(10) courses = CourseFactory.create_batch(10)
courses.sort(key=lambda course: course.id.lower()) courses.sort(key=lambda course: course.key.lower())
url = reverse('api:v1:course-list') url = reverse('api:v1:course-list')
limit = 3 limit = 3
self.refresh_index()
response = self.client.get(url, {'format': format, 'limit': limit}) response = self.client.get(url, {'format': format, 'limit': limit})
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
...@@ -253,38 +239,6 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin ...@@ -253,38 +239,6 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
response.render() response.render()
def test_list_query(self):
""" Verify the endpoint returns a filtered list of courses. """
# Create courses that should NOT match our query
CourseFactory.create_batch(3)
# Create courses that SHOULD match our query
name = 'query test'
courses = [CourseFactory(name=name), CourseFactory(name=name)]
courses.sort(key=lambda course: course.id.lower())
self.refresh_index()
query = {
"query": {
"bool": {
"must": [
{
"term": {
"course.name": name
}
}
]
}
}
}
qs = urllib.parse.urlencode({'q': json.dumps(query)})
url = '{}?{}'.format(reverse('api:v1:course-list'), qs)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data['count'], len(courses))
self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
def test_retrieve(self): def test_retrieve(self):
""" Verify the endpoint returns a single course. """ """ Verify the endpoint returns a single course. """
self.assert_retrieve_success() self.assert_retrieve_success()
...@@ -292,7 +246,7 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin ...@@ -292,7 +246,7 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
def assert_retrieve_success(self, **headers): def assert_retrieve_success(self, **headers):
""" Asserts the endpoint returns details for a single course. """ """ Asserts the endpoint returns details for a single course. """
course = CourseFactory() course = CourseFactory()
url = reverse('api:v1:course-detail', kwargs={'id': course.id}) url = reverse('api:v1:course-detail', kwargs={'key': course.key})
response = self.client.get(url, format='json', **headers) response = self.client.get(url, format='json', **headers)
self.assertEqual(response.status_code, 200) self.assertEqual(response.status_code, 200)
self.assertEqual(response.data, self.serialize_course(course)) self.assertEqual(response.data, self.serialize_course(course))
......
course_discovery/apps/api/v1/views.py
import json
import logging import logging
from django.db.models.functions import Lower
from rest_framework import viewsets from rest_framework import viewsets
from rest_framework.decorators import detail_route from rest_framework.decorators import detail_route
from rest_framework.permissions import IsAuthenticated from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response from rest_framework.response import Response
from course_discovery.apps.api.pagination import ElasticsearchLimitOffsetPagination
from course_discovery.apps.api.serializers import CatalogSerializer, CourseSerializer, ContainedCoursesSerializer from course_discovery.apps.api.serializers import CatalogSerializer, CourseSerializer, ContainedCoursesSerializer
from course_discovery.apps.catalogs.models import Catalog from course_discovery.apps.catalogs.models import Catalog
from course_discovery.apps.course_metadata.constants import COURSE_ID_REGEX from course_discovery.apps.course_metadata.constants import COURSE_ID_REGEX
...@@ -91,49 +90,15 @@ class CatalogViewSet(viewsets.ModelViewSet): ...@@ -91,49 +90,15 @@ class CatalogViewSet(viewsets.ModelViewSet):
class CourseViewSet(viewsets.ReadOnlyModelViewSet): class CourseViewSet(viewsets.ReadOnlyModelViewSet):
""" Course resource. """ """ Course resource. """
lookup_field = 'id' lookup_field = 'key'
lookup_value_regex = COURSE_ID_REGEX lookup_value_regex = COURSE_ID_REGEX
permission_classes = (IsAuthenticated,) permission_classes = (IsAuthenticated,)
serializer_class = CourseSerializer serializer_class = CourseSerializer
pagination_class = ElasticsearchLimitOffsetPagination queryset = Course.objects.all().order_by(Lower('key'))
def get_object(self):
""" Return a single course. """
return Course.get(self.kwargs[self.lookup_url_kwarg or self.lookup_field])
def get_queryset(self):
# Note (CCB): This is solely here to appease DRF. It is not actually used.
return []
def get_data(self, limit, offset):
""" Return all courses. """
query = self.request.GET.get('q', None)
if query:
query = json.loads(query)
return Course.search(query, limit=limit, offset=offset)
else:
return Course.all(limit=limit, offset=offset)
def list(self, request, *args, **kwargs): # pylint: disable=unused-argument
"""
List all courses.
---
parameters:
- name: q
description: Query to filter the courses
required: false
type: string
paramType: query
multiple: false
"""
limit = self.paginator.get_limit(self.request)
offset = self.paginator.get_offset(self.request)
data = self.get_data(limit, offset)
page = self.paginate_queryset(data) def list(self, request, *args, **kwargs):
serializer = self.get_serializer(page, many=True) """ List all courses. """
return self.get_paginated_response(serializer.data) return super(CourseViewSet, self).list(request, *args, **kwargs)
def retrieve(self, request, *args, **kwargs): def retrieve(self, request, *args, **kwargs):
""" Retrieve details for a course. """ """ Retrieve details for a course. """
......
course_discovery/apps/catalogs/models.py
import json
from django.db import models from django.db import models
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from django_extensions.db.models import TimeStampedModel from django_extensions.db.models import TimeStampedModel
from haystack.query import SearchQuerySet
from course_discovery.apps.course_metadata.models import Course from course_discovery.apps.course_metadata.models import Course
...@@ -14,9 +13,14 @@ class Catalog(TimeStampedModel): ...@@ -14,9 +13,14 @@ class Catalog(TimeStampedModel):
def __str__(self): def __str__(self):
return 'Catalog #{id}: {name}'.format(id=self.id, name=self.name) # pylint: disable=no-member return 'Catalog #{id}: {name}'.format(id=self.id, name=self.name) # pylint: disable=no-member
@property def _get_query_results(self):
def query_as_dict(self): """
return json.loads(self.query) Returns the results of this Catalog's query.
Returns:
SearchQuerySet
"""
return SearchQuerySet().models(Course).raw_search(self.query)
def courses(self): def courses(self):
""" Returns the list of courses contained within this catalog. """ Returns the list of courses contained within this catalog.
...@@ -24,8 +28,8 @@ class Catalog(TimeStampedModel): ...@@ -24,8 +28,8 @@ class Catalog(TimeStampedModel):
Returns: Returns:
Course[] Course[]
""" """
results = self._get_query_results().load_all()
return Course.search(self.query_as_dict)['results'] return [result.object for result in results]
def contains(self, course_ids): # pylint: disable=unused-argument def contains(self, course_ids): # pylint: disable=unused-argument
""" Determines if the given courses are contained in this catalog. """ Determines if the given courses are contained in this catalog.
...@@ -37,26 +41,9 @@ class Catalog(TimeStampedModel): ...@@ -37,26 +41,9 @@ class Catalog(TimeStampedModel):
dict: Mapping of course IDs to booleans indicating if course is dict: Mapping of course IDs to booleans indicating if course is
contained in this catalog. contained in this catalog.
""" """
query = self.query_as_dict['query']
# Create a filtered query that includes that uses the catalog's query against a
# collection of courses filtered using the passed in course IDs.
filtered_query = {
"query": {
"filtered": {
"query": query,
"filter": {
"ids": {
"values": course_ids
}
}
}
}
}
contains = {course_id: False for course_id in course_ids} contains = {course_id: False for course_id in course_ids}
courses = Course.search(filtered_query)['results'] results = self._get_query_results().filter(key__in=course_ids)
for course in courses: for result in results:
contains[course.id] = True contains[result.get_stored_fields()['key']] = True
return contains return contains
course_discovery/apps/catalogs/tests/test_models.py
import json
from django.test import TestCase from django.test import TestCase
from course_discovery.apps.catalogs.tests import factories from course_discovery.apps.catalogs.tests import factories
...@@ -12,21 +10,8 @@ class CatalogTests(ElasticsearchTestMixin, TestCase): ...@@ -12,21 +10,8 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
def setUp(self): def setUp(self):
super(CatalogTests, self).setUp() super(CatalogTests, self).setUp()
query = { self.catalog = factories.CatalogFactory(query='title:abc*')
'query': { self.course = CourseFactory(key='a/b/c', title='ABCs of Ͳҽʂէìղց')
'bool': {
'must': [
{
'wildcard': {
'course.name': 'abc*'
}
}
]
}
}
}
self.catalog = factories.CatalogFactory(query=json.dumps(query))
self.course = CourseFactory(id='a/b/c', name='ABCs of Ͳҽʂէìղց')
self.refresh_index() self.refresh_index()
def test_unicode(self): def test_unicode(self):
...@@ -44,5 +29,8 @@ class CatalogTests(ElasticsearchTestMixin, TestCase): ...@@ -44,5 +29,8 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
def test_contains(self): def test_contains(self):
""" Verify the method returns a mapping of course IDs to booleans. """ """ Verify the method returns a mapping of course IDs to booleans. """
other_id = 'd/e/f' uncontained_course = CourseFactory(key='d/e/f', title='ABDEF')
self.assertDictEqual(self.catalog.contains([self.course.id, other_id]), {self.course.id: True, other_id: False}) self.assertDictEqual(
self.catalog.contains([self.course.key, uncontained_course.key]),
{self.course.key: True, uncontained_course.key: False}
)
course_discovery/apps/core/admin.py
...@@ -5,26 +5,30 @@ from django.contrib.auth.admin import UserAdmin ...@@ -5,26 +5,30 @@ from django.contrib.auth.admin import UserAdmin
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from course_discovery.apps.core.forms import UserThrottleRateForm from course_discovery.apps.core.forms import UserThrottleRateForm
from course_discovery.apps.core.models import User, UserThrottleRate from course_discovery.apps.core.models import User, UserThrottleRate, Currency
@admin.register(User)
class CustomUserAdmin(UserAdmin): class CustomUserAdmin(UserAdmin):
""" Admin configuration for the custom User model. """ """ Admin configuration for the custom User model. """
list_display = ('username', 'email', 'full_name', 'first_name', 'last_name', 'is_staff') list_display = ('username', 'email', 'full_name', 'first_name', 'last_name', 'is_staff')
fieldsets = ( fieldsets = (
(None, {'fields': ('username', 'password')}), (None, {'fields': ('username', 'password')}),
(_('Personal info'), {'fields': ('full_name', 'first_name', 'last_name', 'email')}), (_('Personal info'), {'fields': ('full_name', 'first_name', 'last_name', 'email')}),
(_('Permissions'), {'fields': ('is_active', 'is_staff', 'is_superuser', (_('Permissions'), {'fields': ('is_active', 'is_staff', 'is_superuser', 'groups', 'user_permissions')}),
'groups', 'user_permissions')}),
(_('Important dates'), {'fields': ('last_login', 'date_joined')}), (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
) )
@admin.register(UserThrottleRate)
class UserThrottleRateAdmin(admin.ModelAdmin): class UserThrottleRateAdmin(admin.ModelAdmin):
""" Admin configuration for the UserThrottleRate model. """ """ Admin configuration for the UserThrottleRate model. """
form = UserThrottleRateForm form = UserThrottleRateForm
raw_id_fields = ('user',) raw_id_fields = ('user',)
admin.site.register(User, CustomUserAdmin) @admin.register(Currency)
admin.site.register(UserThrottleRate, UserThrottleRateAdmin) class CurrencyAdmin(admin.ModelAdmin):
list_display = ('code', 'name',)
ordering = ('code', 'name',)
search_fields = ('code', 'name',)
course_discovery/apps/course_metadata/management/commands/install_es_indexes.py → course_discovery/apps/core/management/commands/install_es_indexes.py
...@@ -4,7 +4,7 @@ from django.conf import settings ...@@ -4,7 +4,7 @@ from django.conf import settings
from django.core.management import BaseCommand from django.core.management import BaseCommand
from elasticsearch import Elasticsearch from elasticsearch import Elasticsearch
from course_discovery.apps.course_metadata.utils import ElasticsearchUtils from course_discovery.apps.core.utils import ElasticsearchUtils
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -13,8 +13,8 @@ class Command(BaseCommand): ...@@ -13,8 +13,8 @@ class Command(BaseCommand):
help = 'Install any required Elasticsearch indexes' help = 'Install any required Elasticsearch indexes'
def handle(self, *args, **options): def handle(self, *args, **options):
host = settings.ELASTICSEARCH['host'] host = settings.HAYSTACK_CONNECTIONS['default']['URL']
alias = settings.ELASTICSEARCH['index'] alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host) logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host)
es = Elasticsearch(host) es = Elasticsearch(host)
......
course_discovery/apps/core/migrations/0004_currency.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0003_auto_20160315_1910'),
]
operations = [
migrations.CreateModel(
name='Currency',
fields=[
('code', models.CharField(unique=True, primary_key=True, serialize=False, max_length=6)),
('name', models.CharField(max_length=255)),
],
options={
'verbose_name_plural': 'Currencies',
},
),
]
course_discovery/apps/core/models.py
...@@ -37,3 +37,15 @@ class UserThrottleRate(models.Model): ...@@ -37,3 +37,15 @@ class UserThrottleRate(models.Model):
'The rate of requests to limit this user to. The format is specified by Django' 'The rate of requests to limit this user to. The format is specified by Django'
' Rest Framework (see http://www.django-rest-framework.org/api-guide/throttling/).') ' Rest Framework (see http://www.django-rest-framework.org/api-guide/throttling/).')
) )
class Currency(models.Model):
""" Table of currencies as defined by ISO-4217. """
code = models.CharField(max_length=6, primary_key=True, unique=True)
name = models.CharField(max_length=255)
def __str__(self):
return '{code} - {name}'.format(code=self.code, name=self.name)
class Meta(object):
verbose_name_plural = 'Currencies'
course_discovery/apps/core/tests/mixins.py
...@@ -3,7 +3,7 @@ import logging ...@@ -3,7 +3,7 @@ import logging
from django.conf import settings from django.conf import settings
from elasticsearch import Elasticsearch from elasticsearch import Elasticsearch
from course_discovery.apps.course_metadata.utils import ElasticsearchUtils from course_discovery.apps.core.utils import ElasticsearchUtils
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -12,8 +12,8 @@ class ElasticsearchTestMixin(object): ...@@ -12,8 +12,8 @@ class ElasticsearchTestMixin(object):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
super(ElasticsearchTestMixin, cls).setUpClass() super(ElasticsearchTestMixin, cls).setUpClass()
host = settings.ELASTICSEARCH['host'] host = settings.HAYSTACK_CONNECTIONS['default']['URL']
cls.index = settings.ELASTICSEARCH['index'] cls.index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
cls.es = Elasticsearch(host) cls.es = Elasticsearch(host)
def setUp(self): def setUp(self):
......
course_discovery/apps/course_metadata/tests/test_install_es_indexes.py → course_discovery/apps/core/tests/test_install_es_indexes.py
...@@ -10,7 +10,7 @@ LOGGER_NAME = 'courses.management.commands.install_es_indexes' ...@@ -10,7 +10,7 @@ LOGGER_NAME = 'courses.management.commands.install_es_indexes'
class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase): class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase):
def test_create_index(self): def test_create_index(self):
""" Verify the app sets the alias and creates a new index. """ """ Verify the app sets the alias and creates a new index. """
index = settings.ELASTICSEARCH['index'] index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
# Delete the index # Delete the index
self.es.indices.delete(index=index, ignore=404) # pylint: disable=unexpected-keyword-arg self.es.indices.delete(index=index, ignore=404) # pylint: disable=unexpected-keyword-arg
...@@ -23,7 +23,7 @@ class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase): ...@@ -23,7 +23,7 @@ class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase):
def test_alias_exists(self): def test_alias_exists(self):
""" Verify the app does not setup a new Elasticsearch index if the alias is already set. """ """ Verify the app does not setup a new Elasticsearch index if the alias is already set. """
index = settings.ELASTICSEARCH['index'] index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
# Verify the index exists # Verify the index exists
self.assertTrue(self.es.indices.exists(index=index)) self.assertTrue(self.es.indices.exists(index=index))
......
course_discovery/apps/core/tests/test_models.py
...@@ -4,7 +4,7 @@ from django.test import TestCase ...@@ -4,7 +4,7 @@ from django.test import TestCase
from django_dynamic_fixture import G from django_dynamic_fixture import G
from social.apps.django_app.default.models import UserSocialAuth from social.apps.django_app.default.models import UserSocialAuth
from course_discovery.apps.core.models import User from course_discovery.apps.core.models import User, Currency
# pylint: disable=no-member # pylint: disable=no-member
...@@ -38,3 +38,15 @@ class UserTests(TestCase): ...@@ -38,3 +38,15 @@ class UserTests(TestCase):
user = G(User, full_name=full_name, first_name=first_name, last_name=last_name) user = G(User, full_name=full_name, first_name=first_name, last_name=last_name)
self.assertEqual(user.get_full_name(), full_name) self.assertEqual(user.get_full_name(), full_name)
class CurrencyTests(TestCase):
""" Tests for the Currency class. """
def test_str(self):
""" Verify casting an instance to a string returns a string containing the ID and name of the currency. """
code = 'USD',
name = 'U.S. Dollar'
instance = Currency(code=code, name=name)
self.assertEqual(str(instance), '{code} - {name}'.format(code=code, name=name))
course_discovery/apps/core/tests/test_throttles.py
...@@ -15,7 +15,7 @@ class RateLimitingTest(APITestCase): ...@@ -15,7 +15,7 @@ class RateLimitingTest(APITestCase):
def setUp(self): def setUp(self):
super(RateLimitingTest, self).setUp() super(RateLimitingTest, self).setUp()
self.url = reverse('api:v1:course-list') self.url = reverse('api:v1:catalog-list')
self.user = UserFactory() self.user = UserFactory()
self.client.login(username=self.user.username, password=USER_PASSWORD) self.client.login(username=self.user.username, password=USER_PASSWORD)
......
course_discovery/apps/course_metadata/utils.py → course_discovery/apps/core/utils.py
import datetime import datetime
import logging import logging
from course_discovery.apps.course_metadata.config import COURSES_INDEX_CONFIG
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -18,7 +16,7 @@ class ElasticsearchUtils(object): ...@@ -18,7 +16,7 @@ class ElasticsearchUtils(object):
# Create an index with a unique (timestamped) name # Create an index with a unique (timestamped) name
timestamp = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S") timestamp = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
index = '{alias}_{timestamp}'.format(alias=alias, timestamp=timestamp) index = '{alias}_{timestamp}'.format(alias=alias, timestamp=timestamp)
es.indices.create(index=index, body=COURSES_INDEX_CONFIG) es.indices.create(index=index)
logger.info('...index [%s] created.', index) logger.info('...index [%s] created.', index)
# Point the alias to the new index # Point the alias to the new index
......
course_discovery/apps/course_metadata/admin.py 0 → 100644
from django.contrib import admin
from course_discovery.apps.course_metadata.models import (
Seat, Image, Video, LevelType, Subject, Prerequisite, ExpectedLearningItem, Course, CourseRun, Organization, Person,
CourseOrganization, SyllabusItem
)
class CourseOrganizationInline(admin.TabularInline):
model = CourseOrganization
extra = 1
class SeatInline(admin.TabularInline):
model = Seat
extra = 1
@admin.register(Course)
class CourseAdmin(admin.ModelAdmin):
inlines = (CourseOrganizationInline,)
@admin.register(CourseRun)
class CourseRunAdmin(admin.ModelAdmin):
inlines = (SeatInline,)
# Register all models using basic ModelAdmin classes
models = (Image, Video, LevelType, Subject, Prerequisite, ExpectedLearningItem, Organization, Person, SyllabusItem)
for model in models:
admin.site.register(model)
course_discovery/apps/course_metadata/apps.py
...@@ -2,5 +2,5 @@ from django.apps import AppConfig ...@@ -2,5 +2,5 @@ from django.apps import AppConfig
class CourseMetadataConfig(AppConfig): class CourseMetadataConfig(AppConfig):
name = 'course_metadata' name = 'course_discovery.apps.course_metadata'
verbose_name = 'Course Metadata' verbose_name = 'Course Metadata'
course_discovery/apps/course_metadata/config.py deleted 100644 → 0
COURSES_INDEX_CONFIG = {
'settings': {
'analysis': {
'analyzer': {
'lowercase_keyword': {
'tokenizer': 'keyword',
'filter': ['lowercase']
}
}
}
},
'mappings': {
'course': {
'properties': {
'id': {
'type': 'string',
'analyzer': 'lowercase_keyword'
},
'name': {
'type': 'string',
'analyzer': 'lowercase_keyword'
}
}
}
}
}
course_discovery/apps/course_metadata/constants.py
COURSE_ID_REGEX = r'[^/+]+(/|\+)[^/+]+(/|\+)[^/]+' COURSE_ID_REGEX = r'[^/+]+(/|\+)[^/+]+'
COURSE_ID_PATTERN = r'(?P<id>{})'.format(COURSE_ID_REGEX) COURSE_RUN_ID_REGEX = r'[^/+]+(/|\+)[^/+]+(/|\+)[^/]+'
course_discovery/apps/course_metadata/exceptions.py deleted 100644 → 0
class CourseNotFoundError(Exception):
""" The specified course was not found in the data store. """
pass
course_discovery/apps/course_metadata/management/commands/refresh_all_courses.py deleted 100644 → 0
import logging
from django.conf import settings
from django.core.management import BaseCommand
from edx_rest_api_client.client import EdxRestApiClient
from course_discovery.apps.course_metadata.models import Course
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'Refresh course data from external sources.'
def add_arguments(self, parser):
parser.add_argument(
'--access_token',
action='store',
dest='access_token',
default=None,
help='OAuth2 access token used to authenticate API calls.'
)
def handle(self, *args, **options):
access_token = options.get('access_token')
if not access_token:
logger.info('No access token provided. Retrieving access token using client_credential flow...')
try:
access_token, __ = EdxRestApiClient.get_oauth_access_token(
'{root}/access_token'.format(root=settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT),
settings.SOCIAL_AUTH_EDX_OIDC_KEY,
settings.SOCIAL_AUTH_EDX_OIDC_SECRET
)
except Exception:
logger.exception('No access token provided or acquired through client_credential flow.')
raise
Course.refresh_all(access_token=access_token)
course_discovery/apps/course_metadata/migrations/0001_initial.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import sortedm2m.fields
import django.db.models.deletion
from django.conf import settings
import django_extensions.db.fields
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core', '0004_currency'),
('ietf_language_tags', '0002_language_tag_data_migration'),
]
operations = [
migrations.CreateModel(
name='AbstractMediaModel',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('src', models.URLField(max_length=255, unique=True)),
('description', models.CharField(max_length=255, blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='Course',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True, db_index=True)),
('title', models.CharField(max_length=255, default=None, blank=True, null=True)),
('short_description', models.CharField(max_length=255, default=None, blank=True, null=True)),
('full_description', models.TextField(default=None, blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='CourseOrganization',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('relation_type', models.CharField(choices=[('owner', 'Owner'), ('sponsor', 'Sponsor')], max_length=63)),
('course', models.ForeignKey(to='course_metadata.Course')),
],
),
migrations.CreateModel(
name='CourseRun',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True)),
('title_override', models.CharField(max_length=255, default=None, help_text="Title specific for this run of a course. Leave this value blank to default to the parent course's title.", blank=True, null=True)),
('start', models.DateTimeField(blank=True, null=True)),
('end', models.DateTimeField(blank=True, null=True)),
('enrollment_start', models.DateTimeField(blank=True, null=True)),
('enrollment_end', models.DateTimeField(blank=True, null=True)),
('announcement', models.DateTimeField(blank=True, null=True)),
('short_description_override', models.CharField(max_length=255, default=None, help_text="Short description specific for this run of a course. Leave this value blank to default to the parent course's short_description attribute.", blank=True, null=True)),
('full_description_override', models.TextField(default=None, help_text="Full description specific for this run of a course. Leave this value blank to default to the parent course's full_description attribute.", blank=True, null=True)),
('min_effort', models.PositiveSmallIntegerField(help_text='Estimated minimum number of hours per week needed to complete a course run.', blank=True, null=True)),
('max_effort', models.PositiveSmallIntegerField(help_text='Estimated maximum number of hours per week needed to complete a course run.', blank=True, null=True)),
('pacing_type', models.CharField(choices=[('self_paced', 'Self-paced'), ('instructor_paced', 'Instructor-paced')], max_length=255, db_index=True, blank=True, null=True)),
('course', models.ForeignKey(to='course_metadata.Course')),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='ExpectedLearningItem',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('value', models.CharField(max_length=255)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='HistoricalCourse',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('title', models.CharField(max_length=255, default=None, blank=True, null=True)),
('short_description', models.CharField(max_length=255, default=None, blank=True, null=True)),
('full_description', models.TextField(default=None, blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical course',
},
),
migrations.CreateModel(
name='HistoricalCourseRun',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('title_override', models.CharField(max_length=255, default=None, help_text="Title specific for this run of a course. Leave this value blank to default to the parent course's title.", blank=True, null=True)),
('start', models.DateTimeField(blank=True, null=True)),
('end', models.DateTimeField(blank=True, null=True)),
('enrollment_start', models.DateTimeField(blank=True, null=True)),
('enrollment_end', models.DateTimeField(blank=True, null=True)),
('announcement', models.DateTimeField(blank=True, null=True)),
('short_description_override', models.CharField(max_length=255, default=None, help_text="Short description specific for this run of a course. Leave this value blank to default to the parent course's short_description attribute.", blank=True, null=True)),
('full_description_override', models.TextField(default=None, help_text="Full description specific for this run of a course. Leave this value blank to default to the parent course's full_description attribute.", blank=True, null=True)),
('min_effort', models.PositiveSmallIntegerField(help_text='Estimated minimum number of hours per week needed to complete a course run.', blank=True, null=True)),
('max_effort', models.PositiveSmallIntegerField(help_text='Estimated maximum number of hours per week needed to complete a course run.', blank=True, null=True)),
('pacing_type', models.CharField(choices=[('self_paced', 'Self-paced'), ('instructor_paced', 'Instructor-paced')], max_length=255, db_index=True, blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('course', models.ForeignKey(db_constraint=False, to='course_metadata.Course', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
('language', models.ForeignKey(db_constraint=False, to='ietf_language_tags.LanguageTag', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical course run',
},
),
migrations.CreateModel(
name='HistoricalOrganization',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('description', models.TextField(blank=True, null=True)),
('homepage_url', models.URLField(max_length=255, blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical organization',
},
),
migrations.CreateModel(
name='HistoricalPerson',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, db_index=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('title', models.CharField(max_length=255, blank=True, null=True)),
('bio', models.TextField(blank=True, null=True)),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical person',
},
),
migrations.CreateModel(
name='HistoricalSeat',
fields=[
('id', models.IntegerField(db_index=True, blank=True, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('type', models.CharField(choices=[('honor', 'Honor'), ('audit', 'Audit'), ('verified', 'Verified'), ('professional', 'Professional'), ('credit', 'Credit')], max_length=63)),
('price', models.DecimalField(decimal_places=2, max_digits=10)),
('upgrade_deadline', models.DateTimeField()),
('credit_provider', models.CharField(max_length=255)),
('credit_hours', models.IntegerField()),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField()),
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
('course_run', models.ForeignKey(db_constraint=False, to='course_metadata.CourseRun', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
('currency', models.ForeignKey(db_constraint=False, to='core.Currency', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+')),
('history_user', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+')),
],
options={
'get_latest_by': 'history_date',
'ordering': ('-history_date', '-history_id'),
'verbose_name': 'historical seat',
},
),
migrations.CreateModel(
name='LevelType',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('name', models.CharField(max_length=255, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Organization',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('description', models.TextField(blank=True, null=True)),
('homepage_url', models.URLField(max_length=255, blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='Person',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('key', models.CharField(max_length=255, unique=True)),
('name', models.CharField(max_length=255, blank=True, null=True)),
('title', models.CharField(max_length=255, blank=True, null=True)),
('bio', models.TextField(blank=True, null=True)),
('organizations', models.ManyToManyField(to='course_metadata.Organization', blank=True)),
],
options={
'verbose_name_plural': 'People',
},
),
migrations.CreateModel(
name='Prerequisite',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('name', models.CharField(max_length=255, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Seat',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('type', models.CharField(choices=[('honor', 'Honor'), ('audit', 'Audit'), ('verified', 'Verified'), ('professional', 'Professional'), ('credit', 'Credit')], max_length=63)),
('price', models.DecimalField(decimal_places=2, max_digits=10)),
('upgrade_deadline', models.DateTimeField()),
('credit_provider', models.CharField(max_length=255)),
('credit_hours', models.IntegerField()),
('course_run', models.ForeignKey(to='course_metadata.CourseRun', related_name='seats')),
('currency', models.ForeignKey(to='core.Currency')),
],
),
migrations.CreateModel(
name='Subject',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('name', models.CharField(max_length=255, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='SyllabusItem',
fields=[
('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
('value', models.CharField(max_length=255)),
('parent', models.ForeignKey(to='course_metadata.SyllabusItem', null=True, blank=True, related_name='children')),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
),
migrations.CreateModel(
name='Image',
fields=[
('abstractmediamodel_ptr', models.OneToOneField(serialize=False, parent_link=True, to='course_metadata.AbstractMediaModel', primary_key=True, auto_created=True)),
('height', models.IntegerField(blank=True, null=True)),
('width', models.IntegerField(blank=True, null=True)),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
bases=('course_metadata.abstractmediamodel',),
),
migrations.CreateModel(
name='Video',
fields=[
('abstractmediamodel_ptr', models.OneToOneField(serialize=False, parent_link=True, to='course_metadata.AbstractMediaModel', primary_key=True, auto_created=True)),
('image', models.ForeignKey(to='course_metadata.Image')),
],
options={
'abstract': False,
'get_latest_by': 'modified',
'ordering': ('-modified', '-created'),
},
bases=('course_metadata.abstractmediamodel',),
),
migrations.AddField(
model_name='historicalcourserun',
name='syllabus',
field=models.ForeignKey(db_constraint=False, to='course_metadata.SyllabusItem', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourse',
name='level_type',
field=models.ForeignKey(db_constraint=False, to='course_metadata.LevelType', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='courserun',
name='instructors',
field=sortedm2m.fields.SortedManyToManyField(to='course_metadata.Person', help_text=None, blank=True, related_name='courses_instructed'),
),
migrations.AddField(
model_name='courserun',
name='language',
field=models.ForeignKey(to='ietf_language_tags.LanguageTag', null=True, blank=True),
),
migrations.AddField(
model_name='courserun',
name='staff',
field=sortedm2m.fields.SortedManyToManyField(to='course_metadata.Person', help_text=None, blank=True, related_name='courses_staffed'),
),
migrations.AddField(
model_name='courserun',
name='syllabus',
field=models.ForeignKey(default=None, null=True, to='course_metadata.SyllabusItem', blank=True),
),
migrations.AddField(
model_name='courserun',
name='transcript_languages',
field=models.ManyToManyField(to='ietf_language_tags.LanguageTag', blank=True, related_name='transcript_courses'),
),
migrations.AddField(
model_name='courseorganization',
name='organization',
field=models.ForeignKey(to='course_metadata.Organization'),
),
migrations.AddField(
model_name='course',
name='expected_learning_items',
field=sortedm2m.fields.SortedManyToManyField(to='course_metadata.ExpectedLearningItem', help_text=None, blank=True),
),
migrations.AddField(
model_name='course',
name='level_type',
field=models.ForeignKey(default=None, null=True, to='course_metadata.LevelType', blank=True),
),
migrations.AddField(
model_name='course',
name='organizations',
field=models.ManyToManyField(to='course_metadata.Organization', through='course_metadata.CourseOrganization', blank=True),
),
migrations.AddField(
model_name='course',
name='prerequisites',
field=models.ManyToManyField(to='course_metadata.Prerequisite', blank=True),
),
migrations.AddField(
model_name='course',
name='subjects',
field=models.ManyToManyField(to='course_metadata.Subject', blank=True),
),
migrations.AlterUniqueTogether(
name='seat',
unique_together=set([('course_run', 'type', 'currency', 'credit_provider')]),
),
migrations.AddField(
model_name='person',
name='profile_image',
field=models.ForeignKey(to='course_metadata.Image', null=True, blank=True),
),
migrations.AddField(
model_name='organization',
name='logo_image',
field=models.ForeignKey(to='course_metadata.Image', null=True, blank=True),
),
migrations.AddField(
model_name='historicalperson',
name='profile_image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalorganization',
name='logo_image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourserun',
name='image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourserun',
name='video',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Video', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourse',
name='image',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Image', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='historicalcourse',
name='video',
field=models.ForeignKey(db_constraint=False, to='course_metadata.Video', null=True, on_delete=django.db.models.deletion.DO_NOTHING, blank=True, related_name='+'),
),
migrations.AddField(
model_name='courserun',
name='image',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Image', blank=True),
),
migrations.AddField(
model_name='courserun',
name='video',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Video', blank=True),
),
migrations.AlterUniqueTogether(
name='courseorganization',
unique_together=set([('course', 'relation_type', 'relation_type')]),
),
migrations.AlterIndexTogether(
name='courseorganization',
index_together=set([('course', 'relation_type')]),
),
migrations.AddField(
model_name='course',
name='image',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Image', blank=True),
),
migrations.AddField(
model_name='course',
name='video',
field=models.ForeignKey(default=None, null=True, to='course_metadata.Video', blank=True),
),
]
course_discovery/apps/course_metadata/models.py
import logging import logging
from django.conf import settings from django.db import models
from edx_rest_api_client.client import EdxRestApiClient from django.utils.translation import ugettext_lazy as _
from elasticsearch import Elasticsearch, NotFoundError from django_extensions.db.models import TimeStampedModel
from simple_history.models import HistoricalRecords
from sortedm2m.fields import SortedManyToManyField
from course_discovery.apps.course_metadata.exceptions import CourseNotFoundError from course_discovery.apps.core.models import Currency
from course_discovery.apps.ietf_language_tags.models import LanguageTag
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Course(object): class AbstractNamedModel(TimeStampedModel):
""" """ Abstract base class for models with only a name field. """
Course model. name = models.CharField(max_length=255, unique=True)
This model is backed by Elasticsearch.
"""
# Elasticsearch document type for courses.
doc_type = 'course'
# Elasticsearch index where course data is stored
_index = settings.ELASTICSEARCH['index']
@classmethod
def _es_client(cls):
""" Elasticsearch client. """
return Elasticsearch(settings.ELASTICSEARCH['host'])
@classmethod
def _hit_to_course(cls, hit):
return Course(hit['_source']['id'], hit['_source'])
@classmethod
def all(cls, limit=10, offset=0):
"""
Return a list of all courses.
Args:
limit (int): Maximum number of results to return
offset (int): Starting index from which to return results
Returns:
dict: Representation of data suitable for pagination
Examples:
{
'limit': 10,
'offset': 0,
'total': 2,
'results': [`Course`, `Course`],
}
"""
query = {
'query': {
'match_all': {}
}
}
return cls.search(query, limit=limit, offset=offset)
@classmethod
def get(cls, id): # pylint: disable=redefined-builtin
"""
Retrieve a single course.
Args:
id (str): Course ID
Returns:
Course: The course corresponding to the given ID.
Raises:
CourseNotFoundError: if the course is not found.
"""
try:
response = cls._es_client().get(index=cls._index, doc_type=cls.doc_type, id=id)
return cls._hit_to_course(response)
except NotFoundError:
raise CourseNotFoundError('Course [{}] was not found in the data store.'.format(id))
@classmethod
def search(cls, query, limit=10, offset=0):
"""
Search the data store for courses.
Args:
query (dict): Elasticsearch query used to find courses.
limit (int): Maximum number of results to return
offset (int): Index of first result to return
Returns:
dict: Representation of data suitable for pagination
Examples:
{
'limit': 10,
'offset': 0,
'total': 2,
'results': [`Course`, `Course`],
}
"""
query.setdefault('from', offset)
query.setdefault('size', limit)
query.setdefault('sort', {'id': 'asc'})
logger.debug('Querying [%s]: %s', cls._index, query)
response = cls._es_client().search(index=cls._index, doc_type=cls.doc_type, body=query)
hits = response['hits']
total = hits['total']
logger.info('Course search returned [%d] courses.', total)
return {
'limit': limit,
'offset': offset,
'total': total,
'results': [cls._hit_to_course(hit) for hit in hits['hits']]
}
@classmethod
def refresh(cls, course_id, access_token):
"""
Refresh the course data from the raw data sources.
Args:
course_id (str): Course ID
access_token (str): OAuth access token
Returns:
Course
"""
client = EdxRestApiClient(settings.ECOMMERCE_API_URL, oauth_access_token=access_token)
body = client.courses(course_id).get(include_products=True)
course = Course(course_id, body)
course.save()
return course
@classmethod
def refresh_all(cls, access_token):
"""
Refresh all course data.
Args:
access_token (str): OAuth access token
Returns:
None
"""
cls.refresh_all_ecommerce_data(access_token)
cls.refresh_all_course_api_data(access_token)
@classmethod
def refresh_all_ecommerce_data(cls, access_token):
ecommerce_api_url = settings.ECOMMERCE_API_URL
client = EdxRestApiClient(ecommerce_api_url, oauth_access_token=access_token)
count = None
page = 1
logger.info('Refreshing ecommerce data from %s....', ecommerce_api_url)
while page:
response = client.courses().get(include_products=True, page=page, page_size=50)
count = response['count']
results = response['results']
logger.info('Retrieved %d courses...', len(results))
if response['next']:
page += 1
else:
page = None
for body in results:
Course(body['id']).update(body)
logger.info('Retrieved %d courses from %s.', count, ecommerce_api_url)
@classmethod
def refresh_all_course_api_data(cls, access_token):
course_api_url = settings.COURSES_API_URL
client = EdxRestApiClient(course_api_url, oauth_access_token=access_token)
count = None
page = 1
logger.info('Refreshing course api data from %s....', course_api_url)
while page:
# TODO Update API to not require username?
response = client.courses().get(page=page, page_size=50, username='ecommerce_worker')
count = response['pagination']['count']
results = response['results']
logger.info('Retrieved %d courses...', len(results))
if response['pagination']['next']:
page += 1
else:
page = None
for body in results:
Course(body['id']).update(body)
logger.info('Retrieved %d courses from %s.', count, course_api_url)
def __init__(self, id, body=None): # pylint: disable=redefined-builtin
if not id:
raise ValueError('Course ID cannot be empty or None.')
self.id = id
self.body = body or {}
def __eq__(self, other):
"""
Determine if this Course equals another.
Args:
other (Course): object with which to compare
Returns: True iff. the two Course objects have the same `id` value; otherwise, False.
""" def __str__(self):
return self.id is not None \ return self.name
and isinstance(other, Course) \
and self.id == getattr(other, 'id', None) \ class Meta(object):
and self.body == getattr(other, 'body', None) abstract = True
def __repr__(self):
return 'Course {id}: {name}'.format(id=self.id, name=self.name) class AbstractMediaModel(TimeStampedModel):
""" Abstract base class for media-related (e.g. image, video) models. """
src = models.URLField(max_length=255, unique=True)
description = models.CharField(max_length=255, null=True, blank=True)
def __str__(self):
return self.src
class Image(AbstractMediaModel):
""" Image model. """
height = models.IntegerField(null=True, blank=True)
width = models.IntegerField(null=True, blank=True)
class Video(AbstractMediaModel):
""" Video model. """
image = models.ForeignKey(Image)
class LevelType(AbstractNamedModel):
""" LevelType model. """
pass
class Subject(AbstractNamedModel):
""" Subject model. """
pass
class Prerequisite(AbstractNamedModel):
""" Prerequisite model. """
pass
class ExpectedLearningItem(TimeStampedModel):
""" ExpectedLearningItem model. """
value = models.CharField(max_length=255)
class SyllabusItem(TimeStampedModel):
""" SyllabusItem model. """
parent = models.ForeignKey('self', blank=True, null=True, related_name='children')
value = models.CharField(max_length=255)
class Organization(TimeStampedModel):
""" Organization model. """
key = models.CharField(max_length=255, unique=True)
name = models.CharField(max_length=255, null=True, blank=True)
description = models.TextField(null=True, blank=True)
homepage_url = models.URLField(max_length=255, null=True, blank=True)
logo_image = models.ForeignKey(Image, null=True, blank=True)
history = HistoricalRecords()
def __str__(self):
return '{key}: {name}'.format(key=self.key, name=self.name)
class Person(TimeStampedModel):
""" Person model. """
key = models.CharField(max_length=255, unique=True)
name = models.CharField(max_length=255, null=True, blank=True)
title = models.CharField(max_length=255, null=True, blank=True)
bio = models.TextField(null=True, blank=True)
profile_image = models.ForeignKey(Image, null=True, blank=True)
organizations = models.ManyToManyField(Organization, blank=True)
history = HistoricalRecords()
def __str__(self):
return '{key}: {name}'.format(key=self.key, name=self.name)
class Meta(object):
verbose_name_plural = 'People'
class Course(TimeStampedModel):
""" Course model. """
key = models.CharField(max_length=255, db_index=True, unique=True)
title = models.CharField(max_length=255, default=None, null=True, blank=True)
short_description = models.CharField(max_length=255, default=None, null=True, blank=True)
full_description = models.TextField(default=None, null=True, blank=True)
organizations = models.ManyToManyField('Organization', through='CourseOrganization', blank=True)
subjects = models.ManyToManyField(Subject, blank=True)
prerequisites = models.ManyToManyField(Prerequisite, blank=True)
level_type = models.ForeignKey(LevelType, default=None, null=True, blank=True)
expected_learning_items = SortedManyToManyField(ExpectedLearningItem, blank=True)
image = models.ForeignKey(Image, default=None, null=True, blank=True)
video = models.ForeignKey(Video, default=None, null=True, blank=True)
history = HistoricalRecords()
def __str__(self):
return '{key}: {title}'.format(key=self.key, title=self.title)
class CourseRun(TimeStampedModel):
""" CourseRun model. """
SELF_PACED = 'self_paced'
INSTRUCTOR_PACED = 'instructor_paced'
PACING_CHOICES = (
# Translators: Self-paced refers to course runs that operate on the student's schedule.
(SELF_PACED, _('Self-paced')),
# Translators: Instructor-paced refers to course runs that operate on a schedule set by the instructor,
# similar to a normal university course.
(INSTRUCTOR_PACED, _('Instructor-paced')),
)
course = models.ForeignKey(Course)
key = models.CharField(max_length=255, unique=True)
title_override = models.CharField(
max_length=255, default=None, null=True, blank=True,
help_text=_(
"Title specific for this run of a course. Leave this value blank to default to the parent course's title."))
start = models.DateTimeField(null=True, blank=True)
end = models.DateTimeField(null=True, blank=True)
enrollment_start = models.DateTimeField(null=True, blank=True)
enrollment_end = models.DateTimeField(null=True, blank=True)
announcement = models.DateTimeField(null=True, blank=True)
short_description_override = models.CharField(
max_length=255, default=None, null=True, blank=True,
help_text=_(
"Short description specific for this run of a course. Leave this value blank to default to "
"the parent course's short_description attribute."))
full_description_override = models.TextField(
default=None, null=True, blank=True,
help_text=_(
"Full description specific for this run of a course. Leave this value blank to default to "
"the parent course's full_description attribute."))
instructors = SortedManyToManyField(Person, blank=True, related_name='courses_instructed')
staff = SortedManyToManyField(Person, blank=True, related_name='courses_staffed')
min_effort = models.PositiveSmallIntegerField(
null=True, blank=True,
help_text=_('Estimated minimum number of hours per week needed to complete a course run.'))
max_effort = models.PositiveSmallIntegerField(
null=True, blank=True,
help_text=_('Estimated maximum number of hours per week needed to complete a course run.'))
language = models.ForeignKey(LanguageTag, null=True, blank=True)
transcript_languages = models.ManyToManyField(LanguageTag, blank=True, related_name='transcript_courses')
pacing_type = models.CharField(max_length=255, choices=PACING_CHOICES, db_index=True, null=True, blank=True)
syllabus = models.ForeignKey(SyllabusItem, default=None, null=True, blank=True)
image = models.ForeignKey(Image, default=None, null=True, blank=True)
video = models.ForeignKey(Video, default=None, null=True, blank=True)
history = HistoricalRecords()
@property
def title(self):
return self.title_override or self.course.title
@title.setter
def title(self, value):
# Treat empty strings as NULL
value = value or None
self.title_override = value
@property
def short_description(self):
return self.short_description_override or self.course.short_description
@short_description.setter
def short_description(self, value):
# Treat empty strings as NULL
value = value or None
self.short_description_override = value
@property @property
def name(self): def full_description(self):
return self.body.get('name') return self.full_description_override or self.course.full_description
def save(self): @full_description.setter
""" Save the course to the data store. """ def full_description(self, value):
logger.info('Indexing course %s...', self.id) # Treat empty strings as NULL
self._es_client().index(index=self._index, doc_type=self.doc_type, id=self.id, body=self.body) value = value or None
logger.info('Finished indexing course %s.', self.id) self.full_description_override = value
def update(self, body): def __str__(self):
""" Updates (merges) the data in the index with the provided data. return '{key}: {title}'.format(key=self.key, title=self.title)
Args:
body (dict): Data to be merged into the index. class Seat(TimeStampedModel):
""" Seat model. """
Returns: HONOR = 'honor'
None AUDIT = 'audit'
""" VERIFIED = 'verified'
body = { PROFESSIONAL = 'professional'
'doc': body, CREDIT = 'credit'
'doc_as_upsert': True,
} SEAT_TYPE_CHOICES = (
logger.info('Updating course %s...', self.id) (HONOR, _('Honor')),
self._es_client().update(index=self._index, doc_type=self.doc_type, id=self.id, body=body) (AUDIT, _('Audit')),
logger.info('Finished updating course %s.', self.id) (VERIFIED, _('Verified')),
(PROFESSIONAL, _('Professional')),
(CREDIT, _('Credit')),
)
course_run = models.ForeignKey(CourseRun, related_name='seats')
type = models.CharField(max_length=63, choices=SEAT_TYPE_CHOICES)
price = models.DecimalField(decimal_places=2, max_digits=10)
currency = models.ForeignKey(Currency)
upgrade_deadline = models.DateTimeField()
credit_provider = models.CharField(max_length=255)
credit_hours = models.IntegerField()
history = HistoricalRecords()
class Meta(object):
unique_together = (
('course_run', 'type', 'currency', 'credit_provider')
)
class CourseOrganization(TimeStampedModel):
""" CourseOrganization model. """
OWNER = 'owner'
SPONSOR = 'sponsor'
RELATION_TYPE_CHOICES = (
(OWNER, _('Owner')),
(SPONSOR, _('Sponsor')),
)
course = models.ForeignKey(Course)
organization = models.ForeignKey(Organization)
relation_type = models.CharField(max_length=63, choices=RELATION_TYPE_CHOICES)
class Meta(object):
index_together = (
('course', 'relation_type'),
)
unique_together = (
('course', 'relation_type', 'relation_type'),
)
course_discovery/apps/course_metadata/search_indexes.py 0 → 100644
from haystack import indexes
from course_discovery.apps.course_metadata.models import Course
class CourseIndex(indexes.SearchIndex, indexes.Indexable):
text = indexes.CharField(document=True, use_template=True)
key = indexes.CharField(model_attr='key', stored=True)
title = indexes.CharField(model_attr='title')
organizations = indexes.MultiValueField()
short_description = indexes.CharField(model_attr='short_description', null=True)
full_description = indexes.CharField(model_attr='full_description', null=True)
level_type = indexes.CharField(model_attr='level_type__name', null=True)
def prepare_organizations(self, obj):
return [organization.name for organization in obj.organizations.all()]
def get_model(self):
return Course
def index_queryset(self, using=None):
return self.get_model().objects.all()
def get_updated_field(self): # pragma: no cover
return 'modified'
course_discovery/apps/course_metadata/tests/factories.py
import factory import factory
from factory.fuzzy import FuzzyText from factory.fuzzy import FuzzyText
from course_discovery.apps.course_metadata.models import Course from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Person
class CourseFactory(factory.Factory): class CourseFactory(factory.DjangoModelFactory):
class Meta(object): key = FuzzyText(prefix='course-id/')
title = FuzzyText(prefix="Test çօմɾʂҽ ")
short_description = FuzzyText(prefix="Test çօմɾʂҽ short description")
full_description = FuzzyText(prefix="Test çօմɾʂҽ FULL description")
class Meta:
model = Course model = Course
exclude = ('name',)
id = FuzzyText(prefix='course-id/', suffix='/fake') class CourseRunFactory(factory.DjangoModelFactory):
name = FuzzyText(prefix="էҽʂէ çօմɾʂҽ ") key = FuzzyText(prefix='course-run-id/', suffix='/fake')
course = factory.SubFactory(CourseFactory)
@factory.lazy_attribute title_override = None
def body(self): short_description_override = None
return { full_description_override = None
'id': self.id,
'name': self.name class Meta:
} model = CourseRun
@classmethod
def _create(cls, model_class, *args, **kwargs): class OrganizationFactory(factory.DjangoModelFactory):
obj = model_class(*args, **kwargs) key = FuzzyText(prefix='Org.fake/')
obj.save() name = FuzzyText()
return obj
class Meta:
model = Organization
class PersonFactory(factory.DjangoModelFactory):
key = FuzzyText(prefix='Person.fake/')
name = FuzzyText()
title = FuzzyText()
bio = FuzzyText()
class Meta:
model = Person
course_discovery/apps/course_metadata/tests/test_models.py
import json import ddt
from urllib.parse import urlparse, parse_qs from django.test import TestCase
import responses from course_discovery.apps.course_metadata.models import AbstractNamedModel, AbstractMediaModel
from django.test import TestCase, override_settings from course_discovery.apps.course_metadata.tests import factories
from course_discovery.apps.core.tests.mixins import ElasticsearchTestMixin
from course_discovery.apps.course_metadata.exceptions import CourseNotFoundError
from course_discovery.apps.course_metadata.models import Course
from course_discovery.apps.course_metadata.tests.factories import CourseFactory
ACCESS_TOKEN = 'secret' class CourseTests(TestCase):
COURSES_API_URL = 'https://lms.example.com/api/courses/v1' """ Tests for the `Course` model. """
ECOMMERCE_API_URL = 'https://ecommerce.example.com/api/v2'
JSON = 'application/json'
@override_settings(ECOMMERCE_API_URL=ECOMMERCE_API_URL, COURSES_API_URL=COURSES_API_URL)
class CourseTests(ElasticsearchTestMixin, TestCase):
def assert_course_attrs(self, course, attrs):
"""
Validate the attributes of a given Course.
Args:
course (Course)
attrs (dict)
"""
for attr, value in attrs.items():
self.assertEqual(getattr(course, attr), value)
@responses.activate
def mock_refresh_all(self):
"""
Mock the external APIs and refresh all course data.
Returns:
[dict]: List of dictionaries representing course content bodies.
"""
course_bodies = [
{
'id': 'a/b/c',
'url': 'https://ecommerce.example.com/api/v2/courses/a/b/c/',
'name': 'aaaaa',
'verification_deadline': '2022-01-01T01:00:00Z',
'type': 'verified',
'last_edited': '2015-08-19T15:47:24Z'
},
{
'id': 'aaa/bbb/ccc',
'url': 'https://ecommerce.example.com/api/v2/courses/aaa/bbb/ccc/',
'name': 'Introduction to Biology - The Secret of Life',
'verification_deadline': None,
'type': 'audit',
'last_edited': '2015-08-06T19:11:19Z'
}
]
def ecommerce_api_callback(url, data):
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
if page < count:
next = '{}?page={}'.format(url, page)
body = {
'count': count,
'next': next,
'previous': None,
'results': [data[page - 1]]
}
return 200, {}, json.dumps(body)
return request_callback
def courses_api_callback(url, data):
def request_callback(request):
# pylint: disable=redefined-builtin
next = None
count = len(course_bodies)
# Use the querystring to determine which page should be returned. Default to page 1.
# Note that the values of the dict returned by `parse_qs` are lists, hence the `[1]` default value.
qs = parse_qs(urlparse(request.path_url).query)
page = int(qs.get('page', [1])[0])
if page < count:
next = '{}?page={}'.format(url, page)
body = {
'pagination': {
'count': count,
'next': next,
'previous': None,
},
'results': [data[page - 1]]
}
return 200, {}, json.dumps(body)
return request_callback
url = '{host}/courses/'.format(host=ECOMMERCE_API_URL)
responses.add_callback(responses.GET, url, callback=ecommerce_api_callback(url, course_bodies),
content_type=JSON)
url = '{host}/courses/'.format(host=COURSES_API_URL)
responses.add_callback(responses.GET, url, callback=courses_api_callback(url, course_bodies), content_type=JSON)
# Refresh all course data
Course.refresh_all(ACCESS_TOKEN)
self.refresh_index()
return course_bodies
def test_init(self):
""" Verify the constructor requires a non-empty string for the ID. """
msg = 'Course ID cannot be empty or None.'
with self.assertRaisesRegex(ValueError, msg):
Course(None)
with self.assertRaisesRegex(ValueError, msg):
Course('')
def test_eq(self):
""" Verify the __eq__ method returns True if two Course objects have the same `id`. """
course = CourseFactory()
# Both objects must be of type Course
self.assertNotEqual(course, 1)
# A Course should be equal to itself
self.assertEqual(course, course)
# Two Courses are equal if their id attributes match
self.assertEqual(course, Course(id=course.id, body=course.body))
def test_str(self): def test_str(self):
""" Verify the __str__ method returns a string representation of the Course. """ """ Verify casting an instance to a string returns a string containing the key and title. """
course = CourseFactory() course = factories.CourseFactory()
expected = 'Course {id}: {name}'.format(id=course.id, name=course.name) self.assertEqual(str(course), '{key}: {title}'.format(key=course.key, title=course.title))
self.assertEqual(str(course), expected)
def test_all(self):
""" Verify the method returns a list of all courses. """
course_bodies = self.mock_refresh_all()
courses = []
for body in course_bodies:
courses.append(Course.get(body['id']))
expected = {
'limit': 10,
'offset': 0,
'total': 2,
'results': courses,
}
self.assertDictEqual(Course.all(), expected) @ddt.ddt
class CourseRunTests(TestCase):
""" Tests for the `CourseRun` model. """
def test_all_with_limit_and_offset(self): def setUp(self):
""" Verify the method supports limit-offset pagination. """ super(CourseRunTests, self).setUp()
limit = 1 self.course_run = factories.CourseRunFactory()
courses = [CourseFactory(id='1'), CourseFactory(id='2')]
self.refresh_index()
for offset, course in enumerate(courses): def test_str(self):
expected = { """ Verify casting an instance to a string returns a string containing the key and title. """
'limit': limit, course_run = self.course_run
'offset': offset, # pylint: disable=no-member
'total': len(courses), self.assertEqual(str(course_run), '{key}: {title}'.format(key=course_run.key, title=course_run.title))
'results': [course],
} @ddt.data('title', 'short_description', 'full_description')
self.assertDictEqual(Course.all(limit=limit, offset=offset), expected) def test_override_fields(self, field_name):
""" Verify the `CourseRun`'s override field overrides the related `Course`'s field. """
def test_get(self): override_field_name = "{}_override".format(field_name)
""" Verify the method returns a single course. """ self.assertIsNone(getattr(self.course_run, override_field_name))
course = CourseFactory() self.assertEqual(getattr(self.course_run, field_name), getattr(self.course_run.course, field_name))
retrieved = Course.get(course.id)
self.assertEqual(course, retrieved) # Setting the property to a non-empty value should set the override field,
# and trigger the field property getter to use the override.
override_text = 'A Better World'
setattr(self.course_run, field_name, override_text)
self.assertEqual(getattr(self.course_run, override_field_name), override_text)
self.assertEqual(getattr(self.course_run, field_name), override_text)
# Setting the title property to an empty value should set the title_override field to None,
# and trigger the title property getter to use the title of the parent course.
setattr(self.course_run, field_name, None)
self.assertIsNone(getattr(self.course_run, override_field_name))
self.assertEqual(getattr(self.course_run, field_name), getattr(self.course_run.course, field_name))
class OrganizationTests(TestCase):
""" Tests for the `Organization` model. """
def test_get_with_missing_course(self): def test_str(self):
""" """ Verify casting an instance to a string returns a string containing the key and name. """
Verify the method raises a CourseNotFoundError if the specified course does not exist in the data store. organization = factories.OrganizationFactory()
""" self.assertEqual(str(organization), '{key}: {name}'.format(key=organization.key, name=organization.name))
# Note (CCB): This consistently fails on Travis with the error below. Trying index refresh as a last-ditch
# effort to resolve.
#
# elasticsearch.exceptions.TransportError: TransportError(503,
# 'NoShardAvailableActionException[[course_discovery_test][1] null]; nested:
# IllegalIndexShardStateException[[course_discovery_test][1] CurrentState[POST_RECOVERY] operations only
# allowed when started/relocated]; ')
#
self.refresh_index()
course_id = 'fake.course'
expected_msg_regexp = r'Course \[{}\] was not found in the data store.'.format(course_id)
with self.assertRaisesRegex(CourseNotFoundError, expected_msg_regexp):
Course.get(course_id)
def test_search(self):
""" Verify the method returns query results from the data store. """
prefix = 'test'
query = {
'query': {
'bool': {
'must': [
{
'wildcard': {
'course.name': prefix + '*'
}
}
]
}
}
}
courses = []
for i in range(3):
courses.append(CourseFactory.create(name=prefix + str(i)))
CourseFactory.create()
courses.sort(key=lambda course: course.id.lower()) class PersonTests(TestCase):
self.refresh_index() """ Tests for the `Person` model. """
expected = { def test_str(self):
'limit': 10, """ Verify casting an instance to a string returns a string containing the key and name. """
'offset': 0, person = factories.PersonFactory()
'total': len(courses), self.assertEqual(str(person), '{key}: {name}'.format(key=person.key, name=person.name))
'results': courses,
}
self.assertEqual(Course.search(query), expected)
@responses.activate
def test_refresh(self):
""" Verify the method refreshes data for a single course. """
course_id = 'SesameStreetX/Cookies/1T2016'
name = 'C is for Cookie'
body = {
'id': course_id,
'name': name
}
# Mock the call to the E-Commerce API class AbstractNamedModelTests(TestCase):
url = '{host}/courses/{course_id}/'.format(host=ECOMMERCE_API_URL, course_id=course_id) """ Tests for AbstractNamedModel. """
responses.add(responses.GET, url, body=json.dumps(body), content_type=JSON)
# Refresh the course, and ensure the attributes are correct. def test_str(self):
course = Course.refresh(course_id, ACCESS_TOKEN) """ Verify casting an instance to a string returns a string containing the name. """
attrs = {
'id': course_id,
'body': body,
'name': name,
}
self.assert_course_attrs(course, attrs)
# Ensure the data is persisted to the data store class TestAbstractNamedModel(AbstractNamedModel):
course = Course.get(course_id) pass
self.assert_course_attrs(course, attrs)
def test_refresh_all(self): name = 'abc'
""" Verify the method refreshes data for all courses. """ instance = TestAbstractNamedModel(name=name)
course_bodies = self.mock_refresh_all() self.assertEqual(str(instance), name)
self.refresh_index()
# Ensure the data is persisted to the data store
for body in course_bodies:
course_id = body['id']
attrs = {
'id': course_id,
'body': body,
'name': body['name'],
}
course = Course.get(course_id)
self.assert_course_attrs(course, attrs)
def test_name(self): class AbstractMediaModelTests(TestCase):
""" Verify the method returns the course name. """ """ Tests for AbstractMediaModel. """
name = 'ABC Course'
course = Course('a/b/c', {'name': name})
self.assertEqual(course.name, name)
def test_save(self): def test_str(self):
""" Verify the method creates and/or updates new courses. """ """ Verify casting an instance to a string returns a string containing the name. """
course_id = 'TestX/Saving/4T2015'
body = {
'id': course_id,
'name': 'Save Me!'
}
self.assertFalse(self.es.exists(index=self.index, doc_type=Course.doc_type, id=course_id)) class TestAbstractMediaModel(AbstractMediaModel):
Course(course_id, body).save() pass
self.refresh_index()
self.assertTrue(self.es.exists(index=self.index, doc_type=Course.doc_type, id=course_id)) src = 'http://example.com/image.jpg'
course = Course.get(course_id) instance = TestAbstractMediaModel(src=src)
self.assertEqual(course.id, course_id) self.assertEqual(str(instance), src)
self.assertEqual(course.body, body)
course_discovery/apps/course_metadata/tests/test_refresh_all_courses.py deleted 100644 → 0
""" Tests for Refresh All Courses management command. """
from django.core.management import call_command
from django.test import TestCase
from django.test.utils import override_settings
from edx_rest_api_client.client import EdxRestApiClient
from mock import patch
from course_discovery.apps.course_metadata.models import Course
@override_settings(
SOCIAL_AUTH_EDX_OIDC_URL_ROOT="http://auth-url.com/oauth2",
SOCIAL_AUTH_EDX_OIDC_KEY="client_id",
SOCIAL_AUTH_EDX_OIDC_SECRET="client_secret"
)
class RefreshAllCoursesCommandTests(TestCase):
""" Tests for refresh_all_courses management command. """
cmd = 'refresh_all_courses'
def test_call_with_access_token(self):
""" Verify the management command calls Course.refresh_all() with access token. """
access_token = 'secret'
with patch.object(Course, 'refresh_all') as mock_refresh:
call_command(self.cmd, access_token=access_token)
mock_refresh.assert_called_once_with(access_token=access_token)
def test_call_with_client_credentials(self):
""" Verify the management command calls Course.refresh_all() with client credentials. """
access_token = 'secret'
with patch.object(EdxRestApiClient, 'get_oauth_access_token') as mock_access_token:
mock_access_token.return_value = (access_token, None)
with patch.object(Course, 'refresh_all') as mock_refresh:
call_command(self.cmd)
mock_refresh.assert_called_once_with(access_token=access_token)
def test_call_with_client_credentials_error(self):
""" Verify the command requires an access token to complete. """
with patch.object(EdxRestApiClient, 'get_oauth_access_token') as mock_access_token:
mock_access_token.side_effect = Exception()
with self.assertRaises(Exception):
call_command(self.cmd)
course_discovery/apps/ietf_language_tags/__init__.py 0 → 100644
default_app_config = 'course_discovery.apps.ietf_language_tags.apps.IETFLanguageTagsConfig'
course_discovery/apps/ietf_language_tags/admin.py 0 → 100644
""" Admin configuration. """
from django.contrib import admin
from course_discovery.apps.ietf_language_tags.models import LanguageTag
@admin.register(LanguageTag)
class LanguageTagAdmin(admin.ModelAdmin):
list_display = ('code', 'name',)
ordering = ('code', 'name',)
search_fields = ('code', 'name',)
course_discovery/apps/ietf_language_tags/apps.py 0 → 100644
from django.apps import AppConfig
class IETFLanguageTagsConfig(AppConfig):
name = 'course_discovery.apps.ietf_language_tags'
verbose_name = 'IETF Language Tags'
course_discovery/apps/ietf_language_tags/migrations/0001_initial.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
]
operations = [
migrations.CreateModel(
name='LanguageTag',
fields=[
('code', models.CharField(primary_key=True, max_length=50, serialize=False)),
('name', models.CharField(max_length=255)),
],
),
]
course_discovery/apps/ietf_language_tags/migrations/0002_language_tag_data_migration.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
# Set of language names, language tags. Source: http://ss64.com/locale.html
LANGTAGS = (
("Afrikaans", "af"),
("Albanian", "sq"),
("Arabic – Algeria", "ar-dz"),
("Arabic – Bahrain", "ar-bh"),
("Arabic – Egypt", "ar-eg"),
("Arabic – Iraq", "ar-iq"),
("Arabic – Jordan", "ar-jo"),
("Arabic – Kuwait", "ar-kw"),
("Arabic – Lebanon", "ar-lb"),
("Arabic – Libya", "ar-ly"),
("Arabic – Morocco", "ar-ma"),
("Arabic – Oman", "ar-om"),
("Arabic – Qatar", "ar-qa"),
("Arabic – Saudi Arabia", "ar-sa"),
("Arabic – Syria", "ar-sy"),
("Arabic – Tunisia", "ar-tn"),
("Arabic – United Arab Emirates", "ar-ae"),
("Arabic – Yemen", "ar-ye"),
("Armenian", "hy"),
("Azeri – Latin", "az-az"),
("Basque (Basque)", "eu"),
("Belarusian", "be"),
("Bulgarian", "bg"),
("Catalan", "ca"),
("Chinese – China", "zh-cn"),
("Chinese – Hong Kong SAR", "zh-hk"),
("Chinese – Macau SAR", "zh-mo"),
("Chinese – Singapore", "zh-sg"),
("Chinese – Taiwan", "zh-tw"),
("Croatian", "hr"),
("Czech", "cs"),
("Danish", "da"),
("Dutch – Belgium", "nl-be"),
("Dutch – Netherlands", "nl-nl"),
("English – Australia", "en-au"),
("English – Belize", "en-bz"),
("English – Canada", "en-ca"),
("English – Caribbean", "en-cb"),
("English – India", "en-in"),
("English – Ireland", "en-ie"),
("English – Jamaica", "en-jm"),
("English – Malaysia", "en-my"),
("English – New Zealand", "en-nz"),
("English – Phillippines", "en-ph"),
("English – Singapore", "en-sg"),
("English – Southern Africa", "en-za"),
("English – Trinidad", "en-tt"),
("English – Great Britain", "en-gb"),
("English – United States", "en-us"),
("English – Zimbabwe", "en-zw"),
("Estonian", "et"),
("Farsi", "fa"),
("Finnish", "fi"),
("Faroese", "fo"),
("French – France", "fr-fr"),
("French – Belgium", "fr-be"),
("French – Canada", "fr-ca"),
("French – Luxembourg", "fr-lu"),
("French – Switzerland", "fr-ch"),
("Irish – Ireland", "gd-ie"),
("Scottish Gaelic – United Kingdom", "gd"),
("German – Germany", "de-de"),
("German – Austria", "de-at"),
("German – Liechtenstein", "de-li"),
("German – Luxembourg", "de-lu"),
("German – Switzerland", "de-ch"),
("Greek", "el"),
("Hebrew", "he"),
("Hindi", "hi"),
("Hungarian", "hu"),
("Icelandic", "is"),
("Indonesian", "id"),
("Italian – Italy", "it-it"),
("Italian – Switzerland", "it-ch"),
("Japanese", "ja"),
("Korean", "ko"),
("Latvian", "lv"),
("Lithuanian", "lt"),
("F.Y.R.O. Macedonia", "mk"),
("Malay – Malaysia", "ms-my"),
("Malay – Brunei", "ms-bn"),
("Maltese", "mt"),
("Marathi", "mr"),
("Norwegian – Bokmål", "nb-no"),
("Norwegian – Nynorsk", "nn-no"),
("Polish", "pl"),
("Portuguese – Portugal", "pt-pt"),
("Portuguese – Brazil", "pt-br"),
("Raeto-Romance", "rm"),
("Romanian – Romania", "ro"),
("Romanian – Republic of Moldova", "ro-mo"),
("Russian", "ru"),
("Russian – Republic of Moldova", "ru-mo"),
("Sanskrit", "sa"),
("Serbian – Latin", "sr-sp"),
("Setsuana", "tn"),
("Slovenian", "sl"),
("Slovak", "sk"),
("Sorbian", "sb"),
("Spanish – Spain (Modern)", "es-es"),
("Spanish – Argentina", "es-ar"),
("Spanish – Bolivia", "es-bo"),
("Spanish – Chile", "es-cl"),
("Spanish – Colombia", "es-co"),
("Spanish – Costa Rica", "es-cr"),
("Spanish – Dominican Republic", "es-do"),
("Spanish – Ecuador", "es-ec"),
("Spanish – Guatemala", "es-gt"),
("Spanish – Honduras", "es-hn"),
("Spanish – Mexico", "es-mx"),
("Spanish – Nicaragua", "es-ni"),
("Spanish – Panama", "es-pa"),
("Spanish – Peru", "es-pe"),
("Spanish – Puerto Rico", "es-pr"),
("Spanish – Paraguay", "es-py"),
("Spanish – El Salvador", "es-sv"),
("Spanish – Uruguay", "es-uy"),
("Spanish – Venezuela", "es-ve"),
("Southern Sotho", "st"),
("Swahili", "sw"),
("Swedish – Sweden", "sv-se"),
("Swedish – Finland", "sv-fi"),
("Tamil", "ta"),
("Tatar", "tt"),
("Thai", "th"),
("Turkish", "tr"),
("Tsonga", "ts"),
("Ukrainian", "uk"),
("Urdu", "ur"),
("Uzbek – Latin", "uz-uz"),
("Vietnamese", "vi"),
("Xhosa", "xh"),
("Yiddish", "yi"),
("Zulu", "zu"),
)
def add_language_tags(apps, schema_editor):
LanguageTag = apps.get_model('ietf_language_tags', 'LanguageTag')
for name, code in LANGTAGS:
LanguageTag.objects.update_or_create(code=code, defaults={ 'name': name })
def drop_language_tags(apps, schema_editor):
LanguageTag = apps.get_model('ietf_language_tags', 'LanguageTag')
codes = [code for __, code in LANGTAGS]
LanguageTag.objects.filter(code__in=codes).delete()
class Migration(migrations.Migration):
dependencies = [
('ietf_language_tags', '0001_initial'),
]
operations = [
migrations.RunPython(add_language_tags, drop_language_tags)
]
course_discovery/apps/ietf_language_tags/models.py 0 → 100644
""" IETF language tag models. """
from django.db import models
class LanguageTag(models.Model):
""" Table of language tags as defined by BCP 47. https://tools.ietf.org/html/bcp47 """
code = models.CharField(max_length=50, primary_key=True)
name = models.CharField(max_length=255)
def __str__(self):
return '{code} - {name}'.format(code=self.code, name=self.name)
course_discovery/apps/ietf_language_tags/tests/test_models.py 0 → 100644
""" Tests for models. """
from django.test import TestCase
from course_discovery.apps.ietf_language_tags.models import LanguageTag
class LanguageTagTests(TestCase):
""" Tests for the LanguageTag class. """
def test_str(self):
""" Verify casting a LanguageTag to a string returns a string containing the code and name of the model. """
code = 'te-st',
name = 'Test LanguageTag'
tag = LanguageTag(code=code, name=name)
self.assertEqual(str(tag), '{code} - {name}'.format(code=code, name=name))
course_discovery/settings/base.py
...@@ -33,10 +33,14 @@ THIRD_PARTY_APPS = ( ...@@ -33,10 +33,14 @@ THIRD_PARTY_APPS = (
'rest_framework_swagger', 'rest_framework_swagger',
'social.apps.django_app.default', 'social.apps.django_app.default',
'waffle', 'waffle',
'sortedm2m',
'simple_history',
'haystack',
) )
PROJECT_APPS = ( PROJECT_APPS = (
'course_discovery.apps.core', 'course_discovery.apps.core',
'course_discovery.apps.ietf_language_tags',
'course_discovery.apps.api', 'course_discovery.apps.api',
'course_discovery.apps.catalogs', 'course_discovery.apps.catalogs',
'course_discovery.apps.course_metadata', 'course_discovery.apps.course_metadata',
...@@ -56,6 +60,7 @@ MIDDLEWARE_CLASSES = ( ...@@ -56,6 +60,7 @@ MIDDLEWARE_CLASSES = (
'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware',
'social.apps.django_app.middleware.SocialAuthExceptionMiddleware', 'social.apps.django_app.middleware.SocialAuthExceptionMiddleware',
'waffle.middleware.WaffleMiddleware', 'waffle.middleware.WaffleMiddleware',
'simple_history.middleware.HistoryRequestMiddleware',
) )
ROOT_URLCONF = 'course_discovery.urls' ROOT_URLCONF = 'course_discovery.urls'
...@@ -272,11 +277,19 @@ SWAGGER_SETTINGS = { ...@@ -272,11 +277,19 @@ SWAGGER_SETTINGS = {
'doc_expansion': 'list', 'doc_expansion': 'list',
} }
ELASTICSEARCH = { ELASTICSEARCH_URL = 'http://127.0.0.1:9200/'
'host': 'localhost:9200', ELASTICSEARCH_INDEX_NAME = 'catalog'
'index': 'course_discovery',
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': ELASTICSEARCH_URL,
'INDEX_NAME': ELASTICSEARCH_INDEX_NAME,
},
} }
HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
# TODO Replace with None and document. # TODO Replace with None and document.
ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/' ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/' COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'
......
course_discovery/settings/devstack.py
...@@ -20,6 +20,15 @@ if os.environ.get('ENABLE_DJANGO_TOOLBAR', False): ...@@ -20,6 +20,15 @@ if os.environ.get('ENABLE_DJANGO_TOOLBAR', False):
INTERNAL_IPS = ('127.0.0.1',) INTERNAL_IPS = ('127.0.0.1',)
# END TOOLBAR CONFIGURATION # END TOOLBAR CONFIGURATION
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': 'http://es:9200/',
'INDEX_NAME': 'catalog',
},
}
##################################################################### #####################################################################
# Lastly, see if the developer has any local overrides. # Lastly, see if the developer has any local overrides.
if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')): if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):
......
course_discovery/settings/test.py
import os
from course_discovery.settings.base import * from course_discovery.settings.base import *
# TEST SETTINGS # TEST SETTINGS
...@@ -30,9 +29,12 @@ DATABASES = { ...@@ -30,9 +29,12 @@ DATABASES = {
} }
# END IN-MEMORY TEST DATABASE # END IN-MEMORY TEST DATABASE
ELASTICSEARCH = { HAYSTACK_CONNECTIONS = {
'host': os.environ.get('TEST_ELASTICSEARCH_HOST', 'localhost'), 'default': {
'index': 'course_discovery_test', 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': os.environ.get('TEST_ELASTICSEARCH_URL', 'http://127.0.0.1:9200/'),
'INDEX_NAME': 'catalog_test',
},
} }
JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key' JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'
......
course_discovery/settings/utils.py
from os import environ, path from os import environ
import sys
from logging.handlers import SysLogHandler
from django.core.exceptions import ImproperlyConfigured from django.core.exceptions import ImproperlyConfigured
...@@ -12,4 +10,3 @@ def get_env_setting(setting): ...@@ -12,4 +10,3 @@ def get_env_setting(setting):
except KeyError: except KeyError:
error_msg = "Set the [{}] env variable!".format(setting) error_msg = "Set the [{}] env variable!".format(setting)
raise ImproperlyConfigured(error_msg) raise ImproperlyConfigured(error_msg)
course_discovery/templates/search/indexes/course_metadata/course_text.txt 0 → 100644
{{ object.key }}
{{ object.title }}
{{ object.organizations.all|default:'' }}
{{ object.short_description|default:'' }}
{{ object.full_description|default:'' }}
{{ object.level_type|default:'' }}
docker-compose.yml
...@@ -35,7 +35,7 @@ course-discovery: ...@@ -35,7 +35,7 @@ course-discovery:
- .:/edx/app/course_discovery/discovery - .:/edx/app/course_discovery/discovery
command: /edx/app/discovery/devstack.sh start command: /edx/app/discovery/devstack.sh start
environment: environment:
TEST_ELASTICSEARCH_HOST: "es" TEST_ELASTICSEARCH_URL: "http://es:9200"
ports: ports:
- "18381:18381" - "18381:18381"
- "8381:8381" - "8381:8381"
......
requirements/base.txt
django==1.8.7 django==1.8.7
django-extensions==1.5.9 django-extensions==1.5.9
django-haystack==2.4.1
django-simple-history==1.8.1
django-sortedm2m==1.1.1
django-waffle==0.11 django-waffle==0.11
djangorestframework==3.3.1 djangorestframework==3.3.1
djangorestframework-jwt==1.7.2 djangorestframework-jwt==1.7.2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment