Merge pull request #49 from edx/feature/data-model-redux

Updated Data Model

course_discovery/apps/api/pagination.py

-from rest_framework.pagination import LimitOffsetPagination
-
-
-class ElasticsearchLimitOffsetPagination(LimitOffsetPagination):
-    def paginate_queryset(self, queryset, request, view=None):
-        """
-        Convert a paginated Elasticsearch response to a response suitable for DRF.
-
-        Args:
-            queryset (dict): Elasticsearch response
-            request (Request): HTTP request
-
-        Returns:
-            List of data.
-        """
-        # pylint: disable=attribute-defined-outside-init
-        self.limit = self.get_limit(request)
-        self.offset = self.get_offset(request)
-        self.count = queryset['total']
-        self.request = request
-
-        if self.count > self.limit and self.template is not None:
-            self.display_page_controls = True
-
-        return queryset['results']

course_discovery/apps/api/serializers.py

@@ -2,6 +2,7 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
 from course_discovery.apps.catalogs.models import Catalog
+from course_discovery.apps.course_metadata.models import Course
 
 
 class CatalogSerializer(serializers.ModelSerializer):
@@ -12,10 +13,13 @@ class CatalogSerializer(serializers.ModelSerializer):
         fields = ('id', 'name', 'query', 'url',)
 
 
-class CourseSerializer(serializers.Serializer):  # pylint: disable=abstract-method
-    id = serializers.CharField(help_text=_('Course ID'))
-    name = serializers.CharField(help_text=_('Course name'))
-    url = serializers.HyperlinkedIdentityField(view_name='api:v1:course-detail', lookup_field='id')
+class CourseSerializer(serializers.ModelSerializer):
+    key = serializers.CharField()
+    title = serializers.CharField()
+
+    class Meta(object):
+        model = Course
+        fields = ('key', 'title',)
 
 
 class ContainedCoursesSerializer(serializers.Serializer):  # pylint: disable=abstract-method

course_discovery/apps/api/tests/test_serializers.py

@@ -25,14 +25,13 @@ class CatalogSerializerTests(TestCase):
 class CourseSerializerTests(TestCase):
     def test_data(self):
         course = CourseFactory()
-        path = reverse('api:v1:course-detail', kwargs={'id': course.id})
+        path = reverse('api:v1:course-detail', kwargs={'key': course.key})
         request = RequestFactory().get(path)
         serializer = CourseSerializer(course, context={'request': request})
 
         expected = {
-            'id': course.id,
-            'name': course.name,
-            'url': request.build_absolute_uri(),
+            'key': course.key,
+            'title': course.title,
         }
         self.assertDictEqual(serializer.data, expected)
 

course_discovery/apps/api/v1/tests/test_views.py

@@ -70,21 +70,8 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
         super(CatalogViewSetTests, self).setUp()
         self.user = UserFactory(is_staff=True, is_superuser=True)
         self.client.login(username=self.user.username, password=USER_PASSWORD)
-        query = {
-            'query': {
-                'bool': {
-                    'must': [
-                        {
-                            'wildcard': {
-                                'course.name': 'abc*'
-                            }
-                        }
-                    ]
-                }
-            }
-        }
-        self.catalog = CatalogFactory(query=json.dumps(query))
-        self.course = CourseFactory(id='a/b/c', name='ABC Test Course')
+        self.catalog = CatalogFactory(query='title:abc*')
+        self.course = CourseFactory(key='a/b/c', title='ABC Test Course')
         self.refresh_index()
 
     def generate_jwt_token_header(self, user):
@@ -164,13 +151,13 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
 
     def test_contains(self):
         """ Verify the endpoint returns a filtered list of courses contained in the catalog. """
-        course_id = self.course.id
-        qs = urllib.parse.urlencode({'course_id': course_id})
+        course_key = self.course.key
+        qs = urllib.parse.urlencode({'course_id': course_key})
         url = '{}?{}'.format(reverse('api:v1:catalog-contains', kwargs={'id': self.catalog.id}), qs)
 
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data, {'courses': {course_id: True}})
+        self.assertEqual(response.data, {'courses': {course_key: True}})
 
     def test_get(self):
         """ Verify the endpoint returns the details for a single catalog. """
@@ -242,10 +229,9 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
     def test_list(self, format):
         """ Verify the endpoint returns a list of all courses. """
         courses = CourseFactory.create_batch(10)
-        courses.sort(key=lambda course: course.id.lower())
+        courses.sort(key=lambda course: course.key.lower())
         url = reverse('api:v1:course-list')
         limit = 3
-        self.refresh_index()
 
         response = self.client.get(url, {'format': format, 'limit': limit})
         self.assertEqual(response.status_code, 200)
@@ -253,38 +239,6 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
 
         response.render()
 
-    def test_list_query(self):
-        """ Verify the endpoint returns a filtered list of courses. """
-        # Create courses that should NOT match our query
-        CourseFactory.create_batch(3)
-
-        # Create courses that SHOULD match our query
-        name = 'query test'
-        courses = [CourseFactory(name=name), CourseFactory(name=name)]
-        courses.sort(key=lambda course: course.id.lower())
-        self.refresh_index()
-
-        query = {
-            "query": {
-                "bool": {
-                    "must": [
-                        {
-                            "term": {
-                                "course.name": name
-                            }
-                        }
-                    ]
-                }
-            }
-        }
-        qs = urllib.parse.urlencode({'q': json.dumps(query)})
-        url = '{}?{}'.format(reverse('api:v1:course-list'), qs)
-
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data['count'], len(courses))
-        self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
-
     def test_retrieve(self):
         """ Verify the endpoint returns a single course. """
         self.assert_retrieve_success()
@@ -292,7 +246,7 @@ class CourseViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixin
     def assert_retrieve_success(self, **headers):
         """ Asserts the endpoint returns details for a single course. """
         course = CourseFactory()
-        url = reverse('api:v1:course-detail', kwargs={'id': course.id})
+        url = reverse('api:v1:course-detail', kwargs={'key': course.key})
         response = self.client.get(url, format='json', **headers)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data, self.serialize_course(course))

course_discovery/apps/api/v1/views.py

-import json
 import logging
 
+from django.db.models.functions import Lower
 from rest_framework import viewsets
 from rest_framework.decorators import detail_route
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 
-from course_discovery.apps.api.pagination import ElasticsearchLimitOffsetPagination
 from course_discovery.apps.api.serializers import CatalogSerializer, CourseSerializer, ContainedCoursesSerializer
 from course_discovery.apps.catalogs.models import Catalog
 from course_discovery.apps.course_metadata.constants import COURSE_ID_REGEX
@@ -91,49 +90,15 @@ class CatalogViewSet(viewsets.ModelViewSet):
 
 class CourseViewSet(viewsets.ReadOnlyModelViewSet):
     """ Course resource. """
-    lookup_field = 'id'
+    lookup_field = 'key'
     lookup_value_regex = COURSE_ID_REGEX
     permission_classes = (IsAuthenticated,)
     serializer_class = CourseSerializer
-    pagination_class = ElasticsearchLimitOffsetPagination
-
-    def get_object(self):
-        """ Return a single course. """
-        return Course.get(self.kwargs[self.lookup_url_kwarg or self.lookup_field])
-
-    def get_queryset(self):
-        # Note (CCB): This is solely here to appease DRF. It is not actually used.
-        return []
-
-    def get_data(self, limit, offset):
-        """ Return all courses. """
-        query = self.request.GET.get('q', None)
-
-        if query:
-            query = json.loads(query)
-            return Course.search(query, limit=limit, offset=offset)
-        else:
-            return Course.all(limit=limit, offset=offset)
-
-    def list(self, request, *args, **kwargs):  # pylint: disable=unused-argument
-        """
-        List all courses.
-        ---
-        parameters:
-            - name: q
-              description: Query to filter the courses
-              required: false
-              type: string
-              paramType: query
-              multiple: false
-        """
-        limit = self.paginator.get_limit(self.request)
-        offset = self.paginator.get_offset(self.request)
-        data = self.get_data(limit, offset)
+    queryset = Course.objects.all().order_by(Lower('key'))
 
-        page = self.paginate_queryset(data)
-        serializer = self.get_serializer(page, many=True)
-        return self.get_paginated_response(serializer.data)
+    def list(self, request, *args, **kwargs):
+        """ List all courses. """
+        return super(CourseViewSet, self).list(request, *args, **kwargs)
 
     def retrieve(self, request, *args, **kwargs):
         """ Retrieve details for a course. """

course_discovery/apps/catalogs/models.py

-import json
-
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from haystack.query import SearchQuerySet
 
 from course_discovery.apps.course_metadata.models import Course
 
@@ -14,9 +13,14 @@ class Catalog(TimeStampedModel):
     def __str__(self):
         return 'Catalog #{id}: {name}'.format(id=self.id, name=self.name)  # pylint: disable=no-member
 
-    @property
-    def query_as_dict(self):
-        return json.loads(self.query)
+    def _get_query_results(self):
+        """
+        Returns the results of this Catalog's query.
+
+        Returns:
+            SearchQuerySet
+        """
+        return SearchQuerySet().models(Course).raw_search(self.query)
 
     def courses(self):
         """ Returns the list of courses contained within this catalog.
@@ -24,8 +28,8 @@ class Catalog(TimeStampedModel):
         Returns:
             Course[]
         """
-
-        return Course.search(self.query_as_dict)['results']
+        results = self._get_query_results().load_all()
+        return [result.object for result in results]
 
     def contains(self, course_ids):  # pylint: disable=unused-argument
         """ Determines if the given courses are contained in this catalog.
@@ -37,26 +41,9 @@ class Catalog(TimeStampedModel):
             dict: Mapping of course IDs to booleans indicating if course is
                   contained in this catalog.
         """
-        query = self.query_as_dict['query']
-
-        # Create a filtered query that includes that uses the catalog's query against a
-        # collection of courses filtered using the passed in course IDs.
-        filtered_query = {
-            "query": {
-                "filtered": {
-                    "query": query,
-                    "filter": {
-                        "ids": {
-                            "values": course_ids
-                        }
-                    }
-                }
-            }
-        }
-
         contains = {course_id: False for course_id in course_ids}
-        courses = Course.search(filtered_query)['results']
-        for course in courses:
-            contains[course.id] = True
+        results = self._get_query_results().filter(key__in=course_ids)
+        for result in results:
+            contains[result.get_stored_fields()['key']] = True
 
         return contains

course_discovery/apps/catalogs/tests/test_models.py

-import json
-
 from django.test import TestCase
 
 from course_discovery.apps.catalogs.tests import factories
@@ -12,21 +10,8 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
 
     def setUp(self):
         super(CatalogTests, self).setUp()
-        query = {
-            'query': {
-                'bool': {
-                    'must': [
-                        {
-                            'wildcard': {
-                                'course.name': 'abc*'
-                            }
-                        }
-                    ]
-                }
-            }
-        }
-        self.catalog = factories.CatalogFactory(query=json.dumps(query))
-        self.course = CourseFactory(id='a/b/c', name='ABCs of Ͳҽʂէìղց')
+        self.catalog = factories.CatalogFactory(query='title:abc*')
+        self.course = CourseFactory(key='a/b/c', title='ABCs of Ͳҽʂէìղց')
         self.refresh_index()
 
     def test_unicode(self):
@@ -44,5 +29,8 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
 
     def test_contains(self):
         """ Verify the method returns a mapping of course IDs to booleans. """
-        other_id = 'd/e/f'
-        self.assertDictEqual(self.catalog.contains([self.course.id, other_id]), {self.course.id: True, other_id: False})
+        uncontained_course = CourseFactory(key='d/e/f', title='ABDEF')
+        self.assertDictEqual(
+            self.catalog.contains([self.course.key, uncontained_course.key]),
+            {self.course.key: True, uncontained_course.key: False}
+        )

course_discovery/apps/core/admin.py

@@ -5,26 +5,30 @@ from django.contrib.auth.admin import UserAdmin
 from django.utils.translation import ugettext_lazy as _
 
 from course_discovery.apps.core.forms import UserThrottleRateForm
-from course_discovery.apps.core.models import User, UserThrottleRate
+from course_discovery.apps.core.models import User, UserThrottleRate, Currency
 
 
+@admin.register(User)
 class CustomUserAdmin(UserAdmin):
     """ Admin configuration for the custom User model. """
     list_display = ('username', 'email', 'full_name', 'first_name', 'last_name', 'is_staff')
     fieldsets = (
         (None, {'fields': ('username', 'password')}),
         (_('Personal info'), {'fields': ('full_name', 'first_name', 'last_name', 'email')}),
-        (_('Permissions'), {'fields': ('is_active', 'is_staff', 'is_superuser',
-                                       'groups', 'user_permissions')}),
+        (_('Permissions'), {'fields': ('is_active', 'is_staff', 'is_superuser', 'groups', 'user_permissions')}),
         (_('Important dates'), {'fields': ('last_login', 'date_joined')}),
     )
 
 
+@admin.register(UserThrottleRate)
 class UserThrottleRateAdmin(admin.ModelAdmin):
     """ Admin configuration for the UserThrottleRate model. """
     form = UserThrottleRateForm
     raw_id_fields = ('user',)
 
 
-admin.site.register(User, CustomUserAdmin)
-admin.site.register(UserThrottleRate, UserThrottleRateAdmin)
+@admin.register(Currency)
+class CurrencyAdmin(admin.ModelAdmin):
+    list_display = ('code', 'name',)
+    ordering = ('code', 'name',)
+    search_fields = ('code', 'name',)

course_discovery/apps/course_metadata/management/commands/install_es_indexes.py → course_discovery/apps/core/management/commands/install_es_indexes.py

@@ -4,7 +4,7 @@ from django.conf import settings
 from django.core.management import BaseCommand
 from elasticsearch import Elasticsearch
 
-from course_discovery.apps.course_metadata.utils import ElasticsearchUtils
+from course_discovery.apps.core.utils import ElasticsearchUtils
 
 logger = logging.getLogger(__name__)
 
@@ -13,8 +13,8 @@ class Command(BaseCommand):
     help = 'Install any required Elasticsearch indexes'
 
     def handle(self, *args, **options):
-        host = settings.ELASTICSEARCH['host']
-        alias = settings.ELASTICSEARCH['index']
+        host = settings.HAYSTACK_CONNECTIONS['default']['URL']
+        alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
 
         logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host)
         es = Elasticsearch(host)

course_discovery/apps/core/migrations/0004_currency.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0003_auto_20160315_1910'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Currency',
+            fields=[
+                ('code', models.CharField(unique=True, primary_key=True, serialize=False, max_length=6)),
+                ('name', models.CharField(max_length=255)),
+            ],
+            options={
+                'verbose_name_plural': 'Currencies',
+            },
+        ),
+    ]

course_discovery/apps/core/models.py

@@ -37,3 +37,15 @@ class UserThrottleRate(models.Model):
             'The rate of requests to limit this user to. The format is specified by Django'
             ' Rest Framework (see http://www.django-rest-framework.org/api-guide/throttling/).')
     )
+
+
+class Currency(models.Model):
+    """ Table of currencies as defined by ISO-4217. """
+    code = models.CharField(max_length=6, primary_key=True, unique=True)
+    name = models.CharField(max_length=255)
+
+    def __str__(self):
+        return '{code} - {name}'.format(code=self.code, name=self.name)
+
+    class Meta(object):
+        verbose_name_plural = 'Currencies'

course_discovery/apps/core/tests/mixins.py

@@ -3,7 +3,7 @@ import logging
 from django.conf import settings
 from elasticsearch import Elasticsearch
 
-from course_discovery.apps.course_metadata.utils import ElasticsearchUtils
+from course_discovery.apps.core.utils import ElasticsearchUtils
 
 logger = logging.getLogger(__name__)
 
@@ -12,8 +12,8 @@ class ElasticsearchTestMixin(object):
     @classmethod
     def setUpClass(cls):
         super(ElasticsearchTestMixin, cls).setUpClass()
-        host = settings.ELASTICSEARCH['host']
-        cls.index = settings.ELASTICSEARCH['index']
+        host = settings.HAYSTACK_CONNECTIONS['default']['URL']
+        cls.index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
         cls.es = Elasticsearch(host)
 
     def setUp(self):

course_discovery/apps/course_metadata/tests/test_install_es_indexes.py → course_discovery/apps/core/tests/test_install_es_indexes.py

@@ -10,7 +10,7 @@ LOGGER_NAME = 'courses.management.commands.install_es_indexes'
 class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase):
     def test_create_index(self):
         """ Verify the app sets the alias and creates a new index. """
-        index = settings.ELASTICSEARCH['index']
+        index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
 
         # Delete the index
         self.es.indices.delete(index=index, ignore=404)  # pylint: disable=unexpected-keyword-arg
@@ -23,7 +23,7 @@ class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase):
 
     def test_alias_exists(self):
         """ Verify the app does not setup a new Elasticsearch index if the alias is already set. """
-        index = settings.ELASTICSEARCH['index']
+        index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
 
         # Verify the index exists
         self.assertTrue(self.es.indices.exists(index=index))

course_discovery/apps/core/tests/test_models.py

@@ -4,7 +4,7 @@ from django.test import TestCase
 from django_dynamic_fixture import G
 from social.apps.django_app.default.models import UserSocialAuth
 
-from course_discovery.apps.core.models import User
+from course_discovery.apps.core.models import User, Currency
 
 
 # pylint: disable=no-member
@@ -38,3 +38,15 @@ class UserTests(TestCase):
 
         user = G(User, full_name=full_name, first_name=first_name, last_name=last_name)
         self.assertEqual(user.get_full_name(), full_name)
+
+
+class CurrencyTests(TestCase):
+    """ Tests for the Currency class. """
+
+    def test_str(self):
+        """ Verify casting an instance to a string returns a string containing the ID and name of the currency. """
+
+        code = 'USD',
+        name = 'U.S. Dollar'
+        instance = Currency(code=code, name=name)
+        self.assertEqual(str(instance), '{code} - {name}'.format(code=code, name=name))

course_discovery/apps/core/tests/test_throttles.py

@@ -15,7 +15,7 @@ class RateLimitingTest(APITestCase):
 
     def setUp(self):
         super(RateLimitingTest, self).setUp()
-        self.url = reverse('api:v1:course-list')
+        self.url = reverse('api:v1:catalog-list')
         self.user = UserFactory()
         self.client.login(username=self.user.username, password=USER_PASSWORD)
 

course_discovery/apps/course_metadata/utils.py → course_discovery/apps/core/utils.py

 import datetime
 import logging
 
-from course_discovery.apps.course_metadata.config import COURSES_INDEX_CONFIG
-
 logger = logging.getLogger(__name__)
 
 
@@ -18,7 +16,7 @@ class ElasticsearchUtils(object):
             # Create an index with a unique (timestamped) name
             timestamp = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
             index = '{alias}_{timestamp}'.format(alias=alias, timestamp=timestamp)
-            es.indices.create(index=index, body=COURSES_INDEX_CONFIG)
+            es.indices.create(index=index)
             logger.info('...index [%s] created.', index)
 
             # Point the alias to the new index

course_discovery/apps/course_metadata/admin.py

+from django.contrib import admin
+
+from course_discovery.apps.course_metadata.models import (
+    Seat, Image, Video, LevelType, Subject, Prerequisite, ExpectedLearningItem, Course, CourseRun, Organization, Person,
+    CourseOrganization, SyllabusItem
+)
+
+
+class CourseOrganizationInline(admin.TabularInline):
+    model = CourseOrganization
+    extra = 1
+
+
+class SeatInline(admin.TabularInline):
+    model = Seat
+    extra = 1
+
+
+@admin.register(Course)
+class CourseAdmin(admin.ModelAdmin):
+    inlines = (CourseOrganizationInline,)
+
+
+@admin.register(CourseRun)
+class CourseRunAdmin(admin.ModelAdmin):
+    inlines = (SeatInline,)
+
+
+# Register all models using basic ModelAdmin classes
+models = (Image, Video, LevelType, Subject, Prerequisite, ExpectedLearningItem, Organization, Person, SyllabusItem)
+
+for model in models:
+    admin.site.register(model)

course_discovery/apps/course_metadata/apps.py

@@ -2,5 +2,5 @@ from django.apps import AppConfig
 
 
 class CourseMetadataConfig(AppConfig):
-    name = 'course_metadata'
+    name = 'course_discovery.apps.course_metadata'
     verbose_name = 'Course Metadata'

course_discovery/apps/course_metadata/config.py

-COURSES_INDEX_CONFIG = {
-    'settings': {
-        'analysis': {
-            'analyzer': {
-                'lowercase_keyword': {
-                    'tokenizer': 'keyword',
-                    'filter': ['lowercase']
-                }
-            }
-        }
-    },
-    'mappings': {
-        'course': {
-            'properties': {
-                'id': {
-                    'type': 'string',
-                    'analyzer': 'lowercase_keyword'
-                },
-                'name': {
-                    'type': 'string',
-                    'analyzer': 'lowercase_keyword'
-                }
-            }
-        }
-    }
-}

course_discovery/apps/course_metadata/constants.py

-COURSE_ID_REGEX = r'[^/+]+(/|\+)[^/+]+(/|\+)[^/]+'
-COURSE_ID_PATTERN = r'(?P<id>{})'.format(COURSE_ID_REGEX)
+COURSE_ID_REGEX = r'[^/+]+(/|\+)[^/+]+'
+COURSE_RUN_ID_REGEX = r'[^/+]+(/|\+)[^/+]+(/|\+)[^/]+'

course_discovery/apps/course_metadata/exceptions.py

-class CourseNotFoundError(Exception):
-    """ The specified course was not found in the data store. """
-    pass

course_discovery/apps/course_metadata/management/commands/refresh_all_courses.py

-import logging
-
-from django.conf import settings
-from django.core.management import BaseCommand
-from edx_rest_api_client.client import EdxRestApiClient
-
-from course_discovery.apps.course_metadata.models import Course
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    help = 'Refresh course data from external sources.'
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            '--access_token',
-            action='store',
-            dest='access_token',
-            default=None,
-            help='OAuth2 access token used to authenticate API calls.'
-        )
-
-    def handle(self, *args, **options):
-        access_token = options.get('access_token')
-
-        if not access_token:
-            logger.info('No access token provided. Retrieving access token using client_credential flow...')
-
-            try:
-                access_token, __ = EdxRestApiClient.get_oauth_access_token(
-                    '{root}/access_token'.format(root=settings.SOCIAL_AUTH_EDX_OIDC_URL_ROOT),
-                    settings.SOCIAL_AUTH_EDX_OIDC_KEY,
-                    settings.SOCIAL_AUTH_EDX_OIDC_SECRET
-                )
-            except Exception:
-                logger.exception('No access token provided or acquired through client_credential flow.')
-                raise
-
-        Course.refresh_all(access_token=access_token)

course_discovery/apps/course_metadata/search_indexes.py

+from haystack import indexes
+
+from course_discovery.apps.course_metadata.models import Course
+
+
+class CourseIndex(indexes.SearchIndex, indexes.Indexable):
+    text = indexes.CharField(document=True, use_template=True)
+    key = indexes.CharField(model_attr='key', stored=True)
+    title = indexes.CharField(model_attr='title')
+    organizations = indexes.MultiValueField()
+    short_description = indexes.CharField(model_attr='short_description', null=True)
+    full_description = indexes.CharField(model_attr='full_description', null=True)
+    level_type = indexes.CharField(model_attr='level_type__name', null=True)
+
+    def prepare_organizations(self, obj):
+        return [organization.name for organization in obj.organizations.all()]
+
+    def get_model(self):
+        return Course
+
+    def index_queryset(self, using=None):
+        return self.get_model().objects.all()
+
+    def get_updated_field(self):  # pragma: no cover
+        return 'modified'

course_discovery/apps/course_metadata/tests/factories.py

 import factory
 from factory.fuzzy import FuzzyText
 
-from course_discovery.apps.course_metadata.models import Course
+from course_discovery.apps.course_metadata.models import Course, CourseRun, Organization, Person
 
 
-class CourseFactory(factory.Factory):
-    class Meta(object):
+class CourseFactory(factory.DjangoModelFactory):
+    key = FuzzyText(prefix='course-id/')
+    title = FuzzyText(prefix="Test çօմɾʂҽ ")
+    short_description = FuzzyText(prefix="Test çօմɾʂҽ short description")
+    full_description = FuzzyText(prefix="Test çօմɾʂҽ FULL description")
+
+    class Meta:
         model = Course
-        exclude = ('name',)
-
-    id = FuzzyText(prefix='course-id/', suffix='/fake')
-    name = FuzzyText(prefix="էҽʂէ çօմɾʂҽ ")
-
-    @factory.lazy_attribute
-    def body(self):
-        return {
-            'id': self.id,
-            'name': self.name
-        }
-
-    @classmethod
-    def _create(cls, model_class, *args, **kwargs):
-        obj = model_class(*args, **kwargs)
-        obj.save()
-        return obj
+
+
+class CourseRunFactory(factory.DjangoModelFactory):
+    key = FuzzyText(prefix='course-run-id/', suffix='/fake')
+    course = factory.SubFactory(CourseFactory)
+    title_override = None
+    short_description_override = None
+    full_description_override = None
+
+    class Meta:
+        model = CourseRun
+
+
+class OrganizationFactory(factory.DjangoModelFactory):
+    key = FuzzyText(prefix='Org.fake/')
+    name = FuzzyText()
+
+    class Meta:
+        model = Organization
+
+
+class PersonFactory(factory.DjangoModelFactory):
+    key = FuzzyText(prefix='Person.fake/')
+    name = FuzzyText()
+    title = FuzzyText()
+    bio = FuzzyText()
+
+    class Meta:
+        model = Person

course_discovery/apps/course_metadata/tests/test_refresh_all_courses.py

-""" Tests for Refresh All Courses management command. """
-
-from django.core.management import call_command
-from django.test import TestCase
-from django.test.utils import override_settings
-from edx_rest_api_client.client import EdxRestApiClient
-from mock import patch
-
-from course_discovery.apps.course_metadata.models import Course
-
-
-@override_settings(
-    SOCIAL_AUTH_EDX_OIDC_URL_ROOT="http://auth-url.com/oauth2",
-    SOCIAL_AUTH_EDX_OIDC_KEY="client_id",
-    SOCIAL_AUTH_EDX_OIDC_SECRET="client_secret"
-)
-class RefreshAllCoursesCommandTests(TestCase):
-    """ Tests for refresh_all_courses management command. """
-    cmd = 'refresh_all_courses'
-
-    def test_call_with_access_token(self):
-        """ Verify the management command calls Course.refresh_all() with access token. """
-        access_token = 'secret'
-
-        with patch.object(Course, 'refresh_all') as mock_refresh:
-            call_command(self.cmd, access_token=access_token)
-            mock_refresh.assert_called_once_with(access_token=access_token)
-
-    def test_call_with_client_credentials(self):
-        """ Verify the management command calls Course.refresh_all() with client credentials. """
-        access_token = 'secret'
-
-        with patch.object(EdxRestApiClient, 'get_oauth_access_token') as mock_access_token:
-            mock_access_token.return_value = (access_token, None)
-            with patch.object(Course, 'refresh_all') as mock_refresh:
-                call_command(self.cmd)
-                mock_refresh.assert_called_once_with(access_token=access_token)
-
-    def test_call_with_client_credentials_error(self):
-        """ Verify the command requires an access token to complete. """
-        with patch.object(EdxRestApiClient, 'get_oauth_access_token') as mock_access_token:
-            mock_access_token.side_effect = Exception()
-            with self.assertRaises(Exception):
-                call_command(self.cmd)

course_discovery/apps/ietf_language_tags/__init__.py

+default_app_config = 'course_discovery.apps.ietf_language_tags.apps.IETFLanguageTagsConfig'

course_discovery/apps/ietf_language_tags/admin.py

+""" Admin configuration. """
+
+from django.contrib import admin
+
+from course_discovery.apps.ietf_language_tags.models import LanguageTag
+
+
+@admin.register(LanguageTag)
+class LanguageTagAdmin(admin.ModelAdmin):
+    list_display = ('code', 'name',)
+    ordering = ('code', 'name',)
+    search_fields = ('code', 'name',)

course_discovery/apps/ietf_language_tags/apps.py

+from django.apps import AppConfig
+
+
+class IETFLanguageTagsConfig(AppConfig):
+    name = 'course_discovery.apps.ietf_language_tags'
+    verbose_name = 'IETF Language Tags'

course_discovery/apps/ietf_language_tags/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LanguageTag',
+            fields=[
+                ('code', models.CharField(primary_key=True, max_length=50, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

course_discovery/apps/ietf_language_tags/migrations/0002_language_tag_data_migration.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+# Set of language names, language tags.   Source: http://ss64.com/locale.html
+LANGTAGS = (
+    ("Afrikaans", "af"),
+    ("Albanian", "sq"),
+    ("Arabic – Algeria", "ar-dz"),
+    ("Arabic – Bahrain", "ar-bh"),
+    ("Arabic – Egypt", "ar-eg"),
+    ("Arabic – Iraq", "ar-iq"),
+    ("Arabic – Jordan", "ar-jo"),
+    ("Arabic – Kuwait", "ar-kw"),
+    ("Arabic – Lebanon", "ar-lb"),
+    ("Arabic – Libya", "ar-ly"),
+    ("Arabic – Morocco", "ar-ma"),
+    ("Arabic – Oman", "ar-om"),
+    ("Arabic – Qatar", "ar-qa"),
+    ("Arabic – Saudi Arabia", "ar-sa"),
+    ("Arabic – Syria", "ar-sy"),
+    ("Arabic – Tunisia", "ar-tn"),
+    ("Arabic – United Arab Emirates", "ar-ae"),
+    ("Arabic – Yemen", "ar-ye"),
+    ("Armenian", "hy"),
+    ("Azeri – Latin", "az-az"),
+    ("Basque (Basque)", "eu"),
+    ("Belarusian", "be"),
+    ("Bulgarian", "bg"),
+    ("Catalan", "ca"),
+    ("Chinese – China", "zh-cn"),
+    ("Chinese – Hong Kong SAR", "zh-hk"),
+    ("Chinese – Macau SAR", "zh-mo"),
+    ("Chinese – Singapore", "zh-sg"),
+    ("Chinese – Taiwan", "zh-tw"),
+    ("Croatian", "hr"),
+    ("Czech", "cs"),
+    ("Danish", "da"),
+    ("Dutch – Belgium", "nl-be"),
+    ("Dutch – Netherlands", "nl-nl"),
+    ("English – Australia", "en-au"),
+    ("English – Belize", "en-bz"),
+    ("English – Canada", "en-ca"),
+    ("English – Caribbean", "en-cb"),
+    ("English – India", "en-in"),
+    ("English – Ireland", "en-ie"),
+    ("English – Jamaica", "en-jm"),
+    ("English – Malaysia", "en-my"),
+    ("English – New Zealand", "en-nz"),
+    ("English – Phillippines", "en-ph"),
+    ("English – Singapore", "en-sg"),
+    ("English – Southern Africa", "en-za"),
+    ("English – Trinidad", "en-tt"),
+    ("English – Great Britain", "en-gb"),
+    ("English – United States", "en-us"),
+    ("English – Zimbabwe", "en-zw"),
+    ("Estonian", "et"),
+    ("Farsi", "fa"),
+    ("Finnish", "fi"),
+    ("Faroese", "fo"),
+    ("French – France", "fr-fr"),
+    ("French – Belgium", "fr-be"),
+    ("French – Canada", "fr-ca"),
+    ("French – Luxembourg", "fr-lu"),
+    ("French – Switzerland", "fr-ch"),
+    ("Irish – Ireland", "gd-ie"),
+    ("Scottish Gaelic – United Kingdom", "gd"),
+    ("German – Germany", "de-de"),
+    ("German – Austria", "de-at"),
+    ("German – Liechtenstein", "de-li"),
+    ("German – Luxembourg", "de-lu"),
+    ("German – Switzerland", "de-ch"),
+    ("Greek", "el"),
+    ("Hebrew", "he"),
+    ("Hindi", "hi"),
+    ("Hungarian", "hu"),
+    ("Icelandic", "is"),
+    ("Indonesian", "id"),
+    ("Italian – Italy", "it-it"),
+    ("Italian – Switzerland", "it-ch"),
+    ("Japanese", "ja"),
+    ("Korean", "ko"),
+    ("Latvian", "lv"),
+    ("Lithuanian", "lt"),
+    ("F.Y.R.O. Macedonia", "mk"),
+    ("Malay – Malaysia", "ms-my"),
+    ("Malay – Brunei", "ms-bn"),
+    ("Maltese", "mt"),
+    ("Marathi", "mr"),
+    ("Norwegian – Bokmål", "nb-no"),
+    ("Norwegian – Nynorsk", "nn-no"),
+    ("Polish", "pl"),
+    ("Portuguese – Portugal", "pt-pt"),
+    ("Portuguese – Brazil", "pt-br"),
+    ("Raeto-Romance", "rm"),
+    ("Romanian – Romania", "ro"),
+    ("Romanian – Republic of Moldova", "ro-mo"),
+    ("Russian", "ru"),
+    ("Russian – Republic of Moldova", "ru-mo"),
+    ("Sanskrit", "sa"),
+    ("Serbian – Latin", "sr-sp"),
+    ("Setsuana", "tn"),
+    ("Slovenian", "sl"),
+    ("Slovak", "sk"),
+    ("Sorbian", "sb"),
+    ("Spanish – Spain (Modern)", "es-es"),
+    ("Spanish – Argentina", "es-ar"),
+    ("Spanish – Bolivia", "es-bo"),
+    ("Spanish – Chile", "es-cl"),
+    ("Spanish – Colombia", "es-co"),
+    ("Spanish – Costa Rica", "es-cr"),
+    ("Spanish – Dominican Republic", "es-do"),
+    ("Spanish – Ecuador", "es-ec"),
+    ("Spanish – Guatemala", "es-gt"),
+    ("Spanish – Honduras", "es-hn"),
+    ("Spanish – Mexico", "es-mx"),
+    ("Spanish – Nicaragua", "es-ni"),
+    ("Spanish – Panama", "es-pa"),
+    ("Spanish – Peru", "es-pe"),
+    ("Spanish – Puerto Rico", "es-pr"),
+    ("Spanish – Paraguay", "es-py"),
+    ("Spanish – El Salvador", "es-sv"),
+    ("Spanish – Uruguay", "es-uy"),
+    ("Spanish – Venezuela", "es-ve"),
+    ("Southern Sotho", "st"),
+    ("Swahili", "sw"),
+    ("Swedish – Sweden", "sv-se"),
+    ("Swedish – Finland", "sv-fi"),
+    ("Tamil", "ta"),
+    ("Tatar", "tt"),
+    ("Thai", "th"),
+    ("Turkish", "tr"),
+    ("Tsonga", "ts"),
+    ("Ukrainian", "uk"),
+    ("Urdu", "ur"),
+    ("Uzbek – Latin", "uz-uz"),
+    ("Vietnamese", "vi"),
+    ("Xhosa", "xh"),
+    ("Yiddish", "yi"),
+    ("Zulu", "zu"),
+)
+
+
+def add_language_tags(apps, schema_editor):
+    LanguageTag = apps.get_model('ietf_language_tags', 'LanguageTag')
+
+    for name, code in LANGTAGS:
+        LanguageTag.objects.update_or_create(code=code, defaults={ 'name': name })
+
+
+def drop_language_tags(apps, schema_editor):
+    LanguageTag = apps.get_model('ietf_language_tags', 'LanguageTag')
+
+    codes = [code for __, code in LANGTAGS]
+    LanguageTag.objects.filter(code__in=codes).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ietf_language_tags', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_language_tags, drop_language_tags)
+    ]

course_discovery/apps/ietf_language_tags/models.py

+""" IETF language tag models. """
+
+from django.db import models
+
+
+class LanguageTag(models.Model):
+    """ Table of language tags as defined by BCP 47. https://tools.ietf.org/html/bcp47 """
+    code = models.CharField(max_length=50, primary_key=True)
+    name = models.CharField(max_length=255)
+
+    def __str__(self):
+        return '{code} - {name}'.format(code=self.code, name=self.name)

course_discovery/apps/ietf_language_tags/tests/test_models.py

+""" Tests for models. """
+
+from django.test import TestCase
+
+from course_discovery.apps.ietf_language_tags.models import LanguageTag
+
+
+class LanguageTagTests(TestCase):
+    """ Tests for the LanguageTag class. """
+
+    def test_str(self):
+        """ Verify casting a LanguageTag to a string returns a string containing the code and name of the model. """
+
+        code = 'te-st',
+        name = 'Test LanguageTag'
+        tag = LanguageTag(code=code, name=name)
+        self.assertEqual(str(tag), '{code} - {name}'.format(code=code, name=name))

course_discovery/settings/base.py

@@ -33,10 +33,14 @@ THIRD_PARTY_APPS = (
     'rest_framework_swagger',
     'social.apps.django_app.default',
     'waffle',
+    'sortedm2m',
+    'simple_history',
+    'haystack',
 )
 
 PROJECT_APPS = (
     'course_discovery.apps.core',
+    'course_discovery.apps.ietf_language_tags',
     'course_discovery.apps.api',
     'course_discovery.apps.catalogs',
     'course_discovery.apps.course_metadata',
@@ -56,6 +60,7 @@ MIDDLEWARE_CLASSES = (
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'social.apps.django_app.middleware.SocialAuthExceptionMiddleware',
     'waffle.middleware.WaffleMiddleware',
+    'simple_history.middleware.HistoryRequestMiddleware',
 )
 
 ROOT_URLCONF = 'course_discovery.urls'
@@ -272,11 +277,19 @@ SWAGGER_SETTINGS = {
     'doc_expansion': 'list',
 }
 
-ELASTICSEARCH = {
-    'host': 'localhost:9200',
-    'index': 'course_discovery',
+ELASTICSEARCH_URL = 'http://127.0.0.1:9200/'
+ELASTICSEARCH_INDEX_NAME = 'catalog'
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': ELASTICSEARCH_URL,
+        'INDEX_NAME': ELASTICSEARCH_INDEX_NAME,
+    },
 }
 
+HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
+
 # TODO Replace with None and document.
 ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
 COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'

course_discovery/settings/devstack.py

@@ -20,6 +20,15 @@ if os.environ.get('ENABLE_DJANGO_TOOLBAR', False):
 INTERNAL_IPS = ('127.0.0.1',)
 # END TOOLBAR CONFIGURATION
 
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': 'http://es:9200/',
+        'INDEX_NAME': 'catalog',
+    },
+}
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):

course_discovery/settings/test.py

-import os
 from course_discovery.settings.base import *
 
 # TEST SETTINGS
@@ -30,9 +29,12 @@ DATABASES = {
 }
 # END IN-MEMORY TEST DATABASE
 
-ELASTICSEARCH = {
-    'host': os.environ.get('TEST_ELASTICSEARCH_HOST', 'localhost'),
-    'index': 'course_discovery_test',
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': os.environ.get('TEST_ELASTICSEARCH_URL', 'http://127.0.0.1:9200/'),
+        'INDEX_NAME': 'catalog_test',
+    },
 }
 
 JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'

course_discovery/settings/utils.py

-from os import environ, path
-import sys
-from logging.handlers import SysLogHandler
+from os import environ
 
 from django.core.exceptions import ImproperlyConfigured
 
@@ -12,4 +10,3 @@ def get_env_setting(setting):
     except KeyError:
         error_msg = "Set the [{}] env variable!".format(setting)
         raise ImproperlyConfigured(error_msg)
-

course_discovery/templates/search/indexes/course_metadata/course_text.txt

+{{ object.key }}
+{{ object.title }}
+{{ object.organizations.all|default:'' }}
+{{ object.short_description|default:'' }}
+{{ object.full_description|default:'' }}
+{{ object.level_type|default:'' }}

docker-compose.yml

@@ -35,7 +35,7 @@ course-discovery:
     - .:/edx/app/course_discovery/discovery
   command: /edx/app/discovery/devstack.sh start
   environment:
-    TEST_ELASTICSEARCH_HOST: "es"
+    TEST_ELASTICSEARCH_URL: "http://es:9200"
   ports:
     - "18381:18381"
     - "8381:8381"

requirements/base.txt

 django==1.8.7
 django-extensions==1.5.9
+django-haystack==2.4.1
+django-simple-history==1.8.1
+django-sortedm2m==1.1.1
 django-waffle==0.11
 djangorestframework==3.3.1
 djangorestframework-jwt==1.7.2