Merge pull request #129 from edx/clintonb/search-api

Search: courses and course runs

course_discovery/apps/api/serializers.py

+# pylint: disable=abstract-method
+
+import datetime
 from urllib.parse import urlencode
 
 from django.contrib.auth import get_user_model
 from django.utils.translation import ugettext_lazy as _
+from drf_haystack.serializers import HaystackSerializer, HaystackFacetSerializer
 from rest_framework import serializers
 
 from course_discovery.apps.catalogs.models import Catalog
 from course_discovery.apps.course_metadata.models import (
     Course, CourseRun, Image, Organization, Person, Prerequisite, Seat, Subject, Video
 )
+from course_discovery.apps.course_metadata.search_indexes import CourseIndex, CourseRunIndex
 
 User = get_user_model()
 
+COMMON_IGNORED_FIELDS = ('text',)
+COMMON_SEARCH_FIELD_ALIASES = {
+    'q': 'text',
+}
+COURSE_RUN_FACET_FIELD_OPTIONS = {
+    'level_type': {},
+    'organizations': {},
+    'prerequisites': {},
+    'subjects': {},
+    'language': {},
+    'transcript_languages': {},
+    'pacing_type': {},
+    'start': {
+        "start_date": datetime.datetime.now() - datetime.timedelta(days=365),
+        "end_date": datetime.datetime.now(),
+        "gap_by": "month",
+        "gap_amount": 1,
+    },
+    'content_type': {},
+}
+COURSE_RUN_SEARCH_FIELDS = (
+    'key', 'title', 'short_description', 'full_description', 'start', 'end', 'enrollment_start', 'enrollment_end',
+    'pacing_type', 'language', 'transcript_languages', 'marketing_url', 'text',
+)
+
 
 def get_marketing_url_for_user(user, marketing_url):
     """
@@ -47,12 +77,14 @@ class NamedModelSerializer(serializers.ModelSerializer):
 
 class SubjectSerializer(NamedModelSerializer):
     """Serializer for the ``Subject`` model."""
+
     class Meta(NamedModelSerializer.Meta):
         model = Subject
 
 
 class PrerequisiteSerializer(NamedModelSerializer):
     """Serializer for the ``Prerequisite`` model."""
+
     class Meta(NamedModelSerializer.Meta):
         model = Prerequisite
 
@@ -169,7 +201,7 @@ class CourseRunSerializer(TimestampModelSerializer):
         return get_marketing_url_for_user(self.context['request'].user, obj.marketing_url)
 
 
-class ContainedCourseRunsSerializer(serializers.Serializer):  # pylint: disable=abstract-method
+class ContainedCourseRunsSerializer(serializers.Serializer):
     """Serializer used to represent course runs contained by a catalog."""
     course_runs = serializers.DictField(
         child=serializers.BooleanField(),
@@ -207,7 +239,7 @@ class CourseSerializerExcludingClosedRuns(CourseSerializer):
     course_runs = CourseRunSerializer(many=True, source='active_course_runs')
 
 
-class ContainedCoursesSerializer(serializers.Serializer):  # pylint: disable=abstract-method
+class ContainedCoursesSerializer(serializers.Serializer):
     """Serializer used to represent courses contained by a catalog."""
     courses = serializers.DictField(
         child=serializers.BooleanField(),
@@ -330,3 +362,70 @@ class FlattenedCourseRunWithCourseSerializer(CourseRunSerializer):
 
     def get_course_key(self, obj):
         return obj.course.key
+
+
+class CourseSearchSerializer(HaystackSerializer):
+    content_type = serializers.CharField(source='model_name')
+
+    class Meta:
+        field_aliases = COMMON_SEARCH_FIELD_ALIASES
+        fields = ('key', 'title', 'short_description', 'full_description', 'text',)
+        ignore_fields = COMMON_IGNORED_FIELDS
+        index_classes = [CourseIndex]
+
+
+class CourseFacetSerializer(HaystackFacetSerializer):
+    serialize_objects = True
+
+    class Meta:
+        field_aliases = COMMON_SEARCH_FIELD_ALIASES
+        field_options = {
+            'level_type': {},
+            'organizations': {},
+            'prerequisites': {},
+            'subjects': {},
+        }
+        ignore_fields = COMMON_IGNORED_FIELDS
+
+
+class CourseRunSearchSerializer(HaystackSerializer):
+    content_type = serializers.CharField(source='model_name')
+
+    class Meta:
+        field_aliases = COMMON_SEARCH_FIELD_ALIASES
+        fields = COURSE_RUN_SEARCH_FIELDS
+        ignore_fields = COMMON_IGNORED_FIELDS
+        index_classes = [CourseRunIndex]
+
+
+class CourseRunFacetSerializer(HaystackFacetSerializer):
+    serialize_objects = True
+
+    class Meta:
+        field_aliases = COMMON_SEARCH_FIELD_ALIASES
+        field_options = COURSE_RUN_FACET_FIELD_OPTIONS
+        ignore_fields = COMMON_IGNORED_FIELDS
+
+
+class AggregateSearchSerializer(HaystackSerializer):
+    class Meta:
+        field_aliases = COMMON_SEARCH_FIELD_ALIASES
+        fields = COURSE_RUN_SEARCH_FIELDS
+        ignore_fields = COMMON_IGNORED_FIELDS
+        serializers = {
+            CourseRunIndex: CourseRunSearchSerializer,
+            CourseIndex: CourseSearchSerializer,
+        }
+
+
+class AggregateFacetSearchSerializer(HaystackFacetSerializer):
+    serialize_objects = True
+
+    class Meta:
+        field_aliases = COMMON_SEARCH_FIELD_ALIASES
+        field_options = COURSE_RUN_FACET_FIELD_OPTIONS
+        ignore_fields = COMMON_IGNORED_FIELDS
+        serializers = {
+            CourseRunIndex: CourseRunFacetSerializer,
+            CourseIndex: CourseFacetSerializer,
+        }

course_discovery/apps/api/v1/tests/test_views/test_management.py

@@ -5,13 +5,13 @@ from rest_framework.test import APITestCase
 from course_discovery.apps.core.tests.factories import UserFactory
 
 
-class RefreshCourseMetadataTests(APITestCase):
-    """ Tests for the refresh_course_metadata management endpoint. """
-    path = reverse('api:v1:management-refresh-course-metadata')
-    call_command_path = 'course_discovery.apps.api.v1.views.call_command'
+class ManagementCommandViewTestMixin(object):
+    call_command_path = None
+    command_name = None
+    path = None
 
     def setUp(self):
-        super(RefreshCourseMetadataTests, self).setUp()
+        super(ManagementCommandViewTestMixin, self).setUp()
         self.superuser = UserFactory(is_superuser=True)
         self.client.force_authenticate(self.superuser)  # pylint: disable=no-member
 
@@ -20,12 +20,8 @@ class RefreshCourseMetadataTests(APITestCase):
         response = self.client.post(self.path)
         self.assertEqual(response.status_code, 403)
 
-    def test_superuser_required(self):
-        """ Verify only superusers can access the endpoint. """
-        with mock.patch(self.call_command_path, return_value=None):
-            response = self.client.post(self.path)
-            self.assertEqual(response.status_code, 200)
-
+    def test_non_superusers_denied(self):
+        """ Verify access is denied to non-superusers. """
         # Anonymous user
         self.client.logout()
         self.assert_access_forbidden()
@@ -42,8 +38,8 @@ class RefreshCourseMetadataTests(APITestCase):
         self.assert_successful_response('abc123')
 
     def assert_successful_response(self, access_token=None):
-        """ Asserts the endpoint called the refresh_course_metadata management command with the correct arguments,
-        and the endpoint returns HTTP 200 with text/plain content type. """
+        """ Asserts the endpoint called the correct management command with the correct arguments, and the endpoint
+        returns HTTP 200 with text/plain content type. """
         data = {'access_token': access_token} if access_token else None
         with mock.patch(self.call_command_path, return_value=None) as mocked_call_command:
             response = self.client.post(self.path, data)
@@ -55,9 +51,26 @@ class RefreshCourseMetadataTests(APITestCase):
         expected = {
             'settings': 'course_discovery.settings.test'
         }
-        if access_token:
-            expected['access_token'] = access_token
 
         self.assertTrue(mocked_call_command.called)
-        self.assertEqual(args[0], 'refresh_course_metadata')
+        self.assertEqual(args[0], self.command_name)
         self.assertDictContainsSubset(expected, kwargs)
+
+
+class RefreshCourseMetadataTests(ManagementCommandViewTestMixin, APITestCase):
+    """ Tests for the refresh_course_metadata management endpoint. """
+    call_command_path = 'course_discovery.apps.api.v1.views.call_command'
+    command_name = 'refresh_course_metadata'
+    path = reverse('api:v1:management-refresh-course-metadata')
+
+    def test_success_response(self):
+        """ Verify a successful response calls the management command and returns the plain text output. """
+        super(RefreshCourseMetadataTests, self).test_success_response()
+        self.assert_successful_response(access_token='abc123')
+
+
+class UpdateIndexTests(ManagementCommandViewTestMixin, APITestCase):
+    """ Tests for the update_index management endpoint. """
+    call_command_path = 'course_discovery.apps.api.v1.views.call_command'
+    command_name = 'update_index'
+    path = reverse('api:v1:management-update-index')

course_discovery/apps/api/v1/tests/test_views/test_search.py

+import json
+import urllib.parse
+
+import ddt
+from django.core.urlresolvers import reverse
+from rest_framework.test import APITestCase
+
+from course_discovery.apps.core.tests.factories import UserFactory, USER_PASSWORD
+from course_discovery.apps.core.tests.mixins import ElasticsearchTestMixin
+from course_discovery.apps.course_metadata.tests.factories import CourseRunFactory
+
+
+@ddt.ddt
+class CourseRunSearchViewSetTests(ElasticsearchTestMixin, APITestCase):
+    """ Tests for CourseRunSearchViewSet. """
+    faceted_path = reverse('api:v1:search-course_runs-facets')
+    list_path = reverse('api:v1:search-course_runs-list')
+
+    def setUp(self):
+        super(CourseRunSearchViewSetTests, self).setUp()
+        self.user = UserFactory()
+        self.client.login(username=self.user.username, password=USER_PASSWORD)
+
+    def get_search_response(self, query=None, faceted=False):
+        qs = ''
+
+        if query:
+            qs = urllib.parse.urlencode({'q': query})
+
+        path = self.faceted_path if faceted else self.list_path
+        url = '{path}?{qs}'.format(path=path, qs=qs)
+        return self.client.get(url)
+
+    def serialize_date(self, d):
+        return d.strftime('%Y-%m-%dT%H:%M:%S') if d else None
+
+    def serialize_language(self, language):
+        return language.name
+
+    def serialize_course_run(self, course_run):
+        return {
+            'transcript_languages': [self.serialize_language(l) for l in course_run.transcript_languages.all()],
+            'short_description': course_run.short_description,
+            'start': self.serialize_date(course_run.start),
+            'end': self.serialize_date(course_run.end),
+            'enrollment_start': self.serialize_date(course_run.enrollment_start),
+            'enrollment_end': self.serialize_date(course_run.enrollment_end),
+            'key': course_run.key,
+            'marketing_url': course_run.marketing_url,
+            'pacing_type': course_run.pacing_type,
+            'language': self.serialize_language(course_run.language),
+            'full_description': course_run.full_description,
+            'title': course_run.title,
+            'content_type': 'courserun'
+        }
+
+    @ddt.data(True, False)
+    def test_authentication(self, faceted):
+        """ Verify the endpoint requires authentication. """
+        self.client.logout()
+        response = self.get_search_response(faceted=faceted)
+        self.assertEqual(response.status_code, 403)
+
+    def test_search(self):
+        """ Verify the view returns search results. """
+        self.assert_successful_search(faceted=False)
+
+    def test_faceted_search(self):
+        """ Verify the view returns results and facets. """
+        course_run, response_data = self.assert_successful_search(faceted=True)
+
+        # Validate the pacing facet
+        expected = {
+            'text': course_run.pacing_type,
+            'count': 1,
+        }
+        self.assertDictContainsSubset(expected, response_data['fields']['pacing_type'][0])
+
+    def assert_successful_search(self, faceted=False):
+        """ Asserts the search functionality returns results for a generated query. """
+
+        # Generate data that should be indexed and returned by the query
+        course_run = CourseRunFactory(course__title='Software Testing')
+        response = self.get_search_response('software', faceted=faceted)
+
+        self.assertEqual(response.status_code, 200)
+        response_data = json.loads(response.content.decode('utf-8'))
+
+        # Validate the search results
+        expected = {
+            'count': 1,
+            'results': [
+                self.serialize_course_run(course_run)
+            ]
+        }
+        actual = response_data['objects'] if faceted else response_data
+        self.assertDictContainsSubset(expected, actual)
+
+        return course_run, response_data

course_discovery/apps/api/v1/urls.py

 """ API v1 URLs. """
-from rest_framework import routers
 from django.conf.urls import include, url
+from rest_framework import routers
 
 from course_discovery.apps.api.v1 import views
 
-
 partners_router = routers.SimpleRouter()
 partners_router.register(r'affiliate_window/catalogs', views.AffiliateWindowViewSet, base_name='affiliate_window')
 partners_urls = partners_router.urls
@@ -17,5 +16,8 @@ router.register(r'catalogs', views.CatalogViewSet)
 router.register(r'courses', views.CourseViewSet, base_name='course')
 router.register(r'course_runs', views.CourseRunViewSet, base_name='course_run')
 router.register(r'management', views.ManagementViewSet, base_name='management')
+router.register(r'search/all', views.AggregateSearchViewSet, base_name='search-all')
+router.register(r'search/courses', views.CourseSearchViewSet, base_name='search-courses')
+router.register(r'search/course_runs', views.CourseRunSearchViewSet, base_name='search-course_runs')
 
 urlpatterns += router.urls

course_discovery/apps/course_metadata/models.py

@@ -267,6 +267,14 @@ class CourseRun(TimeStampedModel):
         value = value or None
         self.full_description_override = value
 
+    @property
+    def subjects(self):
+        return self.course.subjects
+
+    @property
+    def organizations(self):
+        return self.course.organizations
+
     @classmethod
     def search(cls, query):
         """ Queries the search index.

course_discovery/apps/course_metadata/search_indexes.py

@@ -4,18 +4,49 @@ from opaque_keys.edx.keys import CourseKey
 from course_discovery.apps.course_metadata.models import Course, CourseRun
 
 
-class CourseIndex(indexes.SearchIndex, indexes.Indexable):
+class BaseIndex(indexes.SearchIndex):
+    model = None
+
     text = indexes.CharField(document=True, use_template=True)
+    content_type = indexes.CharField(faceted=True)
+
+    def prepare_content_type(self, obj):  # pylint: disable=unused-argument
+        return self.model.__name__.lower()
+
+    def get_model(self):
+        return self.model
+
+    def get_updated_field(self):  # pragma: no cover
+        return 'modified'
+
+    def index_queryset(self, using=None):
+        return self.model.objects.all()
+
+
+class BaseCourseIndex(BaseIndex):
     key = indexes.CharField(model_attr='key', stored=True)
     title = indexes.CharField(model_attr='title')
     short_description = indexes.CharField(model_attr='short_description', null=True)
     full_description = indexes.CharField(model_attr='full_description', null=True)
-    level_type = indexes.CharField(model_attr='level_type__name', null=True)
+    subjects = indexes.MultiValueField(faceted=True)
+    organizations = indexes.MultiValueField(faceted=True)
+
+    def prepare_organizations(self, obj):
+        return ['{key}: {name}'.format(key=organization.key, name=organization.name) for organization in
+                obj.organizations.all()]
+
+    def prepare_subjects(self, obj):
+        return [subject.name for subject in obj.subjects.all()]
+
+
+class CourseIndex(BaseCourseIndex, indexes.Indexable):
+    model = Course
+
+    level_type = indexes.CharField(model_attr='level_type__name', null=True, faceted=True)
     course_runs = indexes.MultiValueField()
     expected_learning_items = indexes.MultiValueField()
-    organizations = indexes.MultiValueField()
-    prerequisites = indexes.MultiValueField()
-    subjects = indexes.MultiValueField()
+
+    prerequisites = indexes.MultiValueField(faceted=True)
 
     def prepare_course_runs(self, obj):
         return [course_run.key for course_run in obj.course_runs.all()]
@@ -23,46 +54,30 @@ class CourseIndex(indexes.SearchIndex, indexes.Indexable):
     def prepare_expected_learning_items(self, obj):
         return [item.value for item in obj.expected_learning_items.all()]
 
-    def prepare_organizations(self, obj):
-        return ['{key}: {name}'.format(key=organization.key, name=organization.name) for organization in
-                obj.organizations.all()]
-
     def prepare_prerequisites(self, obj):
         return [prerequisite.name for prerequisite in obj.prerequisites.all()]
 
-    def prepare_subjects(self, obj):
-        return [subject.name for subject in obj.subjects.all()]
 
-    def get_model(self):
-        return Course
+class CourseRunIndex(BaseCourseIndex, indexes.Indexable):
+    model = CourseRun
 
-    def index_queryset(self, using=None):
-        return self.get_model().objects.all()
-
-    def get_updated_field(self):  # pragma: no cover
-        return 'modified'
-
-
-class CourseRunIndex(indexes.SearchIndex, indexes.Indexable):
-    text = indexes.CharField(document=True, use_template=True)
     course_key = indexes.CharField(model_attr='course__key', stored=True)
-    key = indexes.CharField(model_attr='key', stored=True)
     org = indexes.CharField()
     number = indexes.CharField()
-    title = indexes.CharField(model_attr='title_override', null=True)
-    start = indexes.DateTimeField(model_attr='start', null=True)
+    start = indexes.DateTimeField(model_attr='start', null=True, faceted=True)
     end = indexes.DateTimeField(model_attr='end', null=True)
     enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True)
     enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True)
     announcement = indexes.DateTimeField(model_attr='announcement', null=True)
     min_effort = indexes.IntegerField(model_attr='min_effort', null=True)
     max_effort = indexes.IntegerField(model_attr='max_effort', null=True)
-    language = indexes.CharField(null=True)
-    transcript_languages = indexes.MultiValueField()
-    pacing_type = indexes.CharField(model_attr='pacing_type', null=True)
+    language = indexes.CharField(null=True, faceted=True)
+    transcript_languages = indexes.MultiValueField(faceted=True)
+    pacing_type = indexes.CharField(model_attr='pacing_type', null=True, faceted=True)
+    marketing_url = indexes.CharField(model_attr='marketing_url', null=True)
 
     def _prepare_language(self, language):
-        return '{code}: {name}'.format(code=language.code, name=language.name)
+        return language.name
 
     def prepare_language(self, obj):
         if obj.language:
@@ -79,9 +94,3 @@ class CourseRunIndex(indexes.SearchIndex, indexes.Indexable):
 
     def prepare_transcript_languages(self, obj):
         return [self._prepare_language(language) for language in obj.transcript_languages.all()]
-
-    def get_model(self):
-        return CourseRun
-
-    def get_updated_field(self):  # pragma: no cover
-        return 'modified'

course_discovery/apps/course_metadata/views.py

@@ -2,4 +2,8 @@ from django.views.generic import TemplateView
 
 
 class QueryPreviewView(TemplateView):
-    template_name = 'catalogs/preview.html'
+    template_name = 'demo/query_preview.html'
+
+
+class SearchDemoView(TemplateView):
+    template_name = 'demo/search.html'

course_discovery/templates/demo/search.html

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Search Demo</title>
+</head>
+<body>
+<h2>Search</h2>
+
+<form method="get" action=".">
+    <table>
+        {{ form.as_table }}
+        <tr>
+            <td>&nbsp;</td>
+            <td>
+                <input type="submit" value="Search">
+            </td>
+        </tr>
+    </table>
+
+    {% if query %}
+        <h3>Results</h3>
+
+        {% for result in page.object_list %}
+            <p>
+                <a href="{{ result.object.get_absolute_url }}">{{ result.object.title }}</a>
+            </p>
+        {% empty %}
+            <p>No results found.</p>
+        {% endfor %}
+
+        {% if page.has_previous or page.has_next %}
+            <div>
+                {% if page.has_previous %}
+                    <a href="?q={{ query }}&amp;page={{ page.previous_page_number }}">{% endif %}&laquo; Previous
+                {% if page.has_previous %}</a>{% endif %}
+                |
+                {% if page.has_next %}<a href="?q={{ query }}&amp;page={{ page.next_page_number }}">{% endif %}
+                Next &raquo;{% if page.has_next %}</a>{% endif %}
+            </div>
+        {% endif %}
+    {% else %}
+        {# Show some example queries to run, maybe query syntax, something else? #}
+    {% endif %}
+</form>
+</body>
+</html>
\ No newline at end of file

course_discovery/templates/search/indexes/course_metadata/courserun_text.txt

@@ -3,3 +3,7 @@
 {{ object.short_description|default:'' }}
 {{ object.full_description|default:'' }}
 {{ object.pacing_type|default:'' }}
+
+{% for language in object.transcript_languages.all %}
+    {{ language }}
+{% endfor %}

requirements/base.txt

@@ -11,6 +11,7 @@ djangorestframework-csv==1.4.1
 djangorestframework-jwt==1.8.0
 djangorestframework-xml==1.3.0
 django-rest-swagger[reST]==0.3.7
+drf-haystack==1.6.0rc1
 dry-rest-permissions==0.1.6
 edx-auth-backends==0.5.0
 edx-ccx-keys==0.2.0