Merge pull request #77 from edx/clintonb/coure-run-search

Added support for course run search

course_discovery/apps/api/serializers.py

@@ -130,7 +130,10 @@ class CatalogSerializer(serializers.ModelSerializer):
 
 class CourseRunSerializer(TimestampModelSerializer):
     course = serializers.SlugRelatedField(read_only=True, slug_field='key')
-    content_language = serializers.SlugRelatedField(read_only=True, slug_field='code', source='language')
+    content_language = serializers.SlugRelatedField(
+        read_only=True, slug_field='code', source='language',
+        help_text=_('Language in which the course is administered')
+    )
     transcript_languages = serializers.SlugRelatedField(many=True, read_only=True, slug_field='code')
     image = ImageSerializer()
     video = VideoSerializer()

course_discovery/apps/api/v1/tests/test_views/test_course_runs.py

@@ -34,3 +34,18 @@ class CourseRunViewSetTests(APITestCase):
             response.data['results'],
             CourseRunSerializer(CourseRun.objects.all().order_by(Lower('key')), many=True).data
         )
+
+    def test_list_query(self):
+        """ Verify the endpoint returns a filtered list of courses """
+        title = 'Some random course'
+        course_runs = CourseRunFactory.create_batch(3, title=title)
+        CourseRunFactory(title='non-matching name')
+        query = 'title:' + title
+        url = '{root}?q={query}'.format(root=reverse('api:v1:course_run-list'), query=query)
+
+        response = self.client.get(url)
+        actual_sorted = sorted(response.data['results'], key=lambda course_run: course_run['key'])
+        expected_sorted = sorted(
+            CourseRunSerializer(course_runs, many=True).data, key=lambda course_run: course_run['key']
+        )
+        self.assertListEqual(actual_sorted, expected_sorted)

course_discovery/apps/api/v1/views.py

@@ -19,6 +19,7 @@ from course_discovery.apps.api.serializers import (
 )
 from course_discovery.apps.api.renderers import AffiliateWindowXMLRenderer
 from course_discovery.apps.catalogs.models import Catalog
+from course_discovery.apps.core.utils import SearchQuerySetWrapper
 from course_discovery.apps.course_metadata.constants import COURSE_ID_REGEX, COURSE_RUN_ID_REGEX
 from course_discovery.apps.course_metadata.models import Course, CourseRun, Seat
 
@@ -124,10 +125,14 @@ class CourseViewSet(viewsets.ReadOnlyModelViewSet):
 
     def get_queryset(self):
         q = self.request.query_params.get('q', None)
-        queryset = Course.search(q) if q else super(CourseViewSet, self).get_queryset()
+
+        if q:
+            queryset = Course.search(q)
+        else:
+            queryset = super(CourseViewSet, self).get_queryset()
+
         return queryset.order_by(Lower('key'))
 
-    # The boilerplate methods are required to be recognized by swagger
     def list(self, request, *args, **kwargs):
         """ List all courses.
         ---
@@ -154,9 +159,24 @@ class CourseRunViewSet(viewsets.ReadOnlyModelViewSet):
     permission_classes = (IsAuthenticated,)
     serializer_class = CourseRunSerializer
 
-    # The boilerplate methods are required to be recognized by swagger
+    def get_queryset(self):
+        q = self.request.query_params.get('q', None)
+        if q:
+            return SearchQuerySetWrapper(CourseRun.search(q))
+        else:
+            return super(CourseRunViewSet, self).get_queryset()
+
     def list(self, request, *args, **kwargs):
-        """ List all course runs. """
+        """ List all courses runs.
+        ---
+        parameters:
+            - name: q
+              description: Elasticsearch querystring query
+              required: false
+              type: string
+              paramType: query
+              multiple: false
+        """
         return super(CourseRunViewSet, self).list(request, *args, **kwargs)
 
     def retrieve(self, request, *args, **kwargs):

course_discovery/apps/core/haystack_backends.py

+from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend, ElasticsearchSearchEngine
+
+
+# pylint: disable=abstract-method
+class SimplifiedElasticsearchSearchBackend(ElasticsearchSearchBackend):
+    def build_search_kwargs(self, *args, **kwargs):
+        """
+        Override default `build_search_kwargs` method to set simpler default search query settings.
+
+        source:
+          https://github.com/django-haystack/django-haystack/blob/master/haystack/backends/elasticsearch_backend.py#L254
+        Without this override the default is:
+          'query_string': {
+            'default_field': content_field,
+            'default_operator': DEFAULT_OPERATOR,
+            'query': query_string,
+            'analyze_wildcard': True,
+            'auto_generate_phrase_queries': True,
+            'fuzzy_min_sim': FUZZY_MIN_SIM,
+            'fuzzy_max_expansions': FUZZY_MAX_EXPANSIONS,
+          }
+        """
+        query_string = args[0]
+        search_kwargs = super(SimplifiedElasticsearchSearchBackend, self).build_search_kwargs(*args, **kwargs)
+
+        simple_query = {
+            'query': query_string,
+            'analyze_wildcard': True,
+            'auto_generate_phrase_queries': True,
+        }
+
+        if search_kwargs['query'].get('filtered', {}).get('query', {}).get('query_string'):
+            search_kwargs['query']['filtered']['query']['query_string'] = simple_query
+        elif search_kwargs['query'].get('query_string'):
+            search_kwargs['query']['query_string'] = simple_query
+
+        return search_kwargs
+
+
+class SimplifiedElasticsearchSearchEngine(ElasticsearchSearchEngine):
+    backend = SimplifiedElasticsearchSearchBackend

course_discovery/apps/core/tests/test_haystack_backends.py

+""" Haystack backend tests. """
+from mock import patch
+
+from django.test import TestCase
+from haystack.backends import BaseSearchBackend
+
+from course_discovery.apps.core.haystack_backends import SimplifiedElasticsearchSearchBackend
+
+
+class SimplifiedElasticsearchSearchEngineTests(TestCase):
+    """ Tests for core.context_processors.core """
+    def setUp(self):
+        super(SimplifiedElasticsearchSearchEngineTests, self).setUp()
+        self.all_query_string = "*:*"
+        self.specific_query_string = "tests:test query"
+        self.simple_query = {
+            'query': self.specific_query_string,
+            'analyze_wildcard': True,
+            'auto_generate_phrase_queries': True,
+        }
+        self.backend = SimplifiedElasticsearchSearchBackend(
+            'default',
+            URL='http://test-es.example.com',
+            INDEX_NAME='testing'
+        )
+
+    def test_build_search_kwargs_all_qs_with_filter(self):
+        with patch.object(BaseSearchBackend, 'build_models_list', return_value=['course_metadata.course']):
+            kwargs = self.backend.build_search_kwargs(self.all_query_string)
+
+        self.assertIsNone(kwargs['query'].get('query_string'))
+        self.assertIsNone(kwargs['query']['filtered']['query'].get('query_string'))
+
+    def test_build_search_kwargs_specific_qs_with_filter(self):
+        with patch.object(BaseSearchBackend, 'build_models_list', return_value=['course_metadata.course']):
+            kwargs = self.backend.build_search_kwargs(self.specific_query_string)
+
+        self.assertIsNone(kwargs['query'].get('query_string'))
+        self.assertDictEqual(kwargs['query']['filtered']['query'].get('query_string'), self.simple_query)
+
+    def test_build_search_kwargs_all_qs_no_filter(self):
+        with patch.object(BaseSearchBackend, 'build_models_list', return_value=[]):
+            kwargs = self.backend.build_search_kwargs(self.all_query_string)
+
+        self.assertIsNone(kwargs['query'].get('filtered'))
+        self.assertIsNone(kwargs['query'].get('query_string'))
+
+    def test_build_search_kwargs_specific_qs_no_filter(self):
+        with patch.object(BaseSearchBackend, 'build_models_list', return_value=[]):
+            kwargs = self.backend.build_search_kwargs(self.specific_query_string)
+
+        self.assertIsNone(kwargs['query'].get('filtered'))
+        self.assertDictEqual(kwargs['query'].get('query_string'), self.simple_query)

course_discovery/apps/core/tests/test_utils.py

 from django.db import models
 from django.test import TestCase
+from haystack.query import SearchQuerySet
 
-from course_discovery.apps.core.utils import get_all_related_field_names
+from course_discovery.apps.core.utils import get_all_related_field_names, SearchQuerySetWrapper
+from course_discovery.apps.course_metadata.models import CourseRun
+from course_discovery.apps.course_metadata.tests.factories import CourseRunFactory
 
 
 class UnrelatedModel(models.Model):
@@ -37,3 +40,24 @@ class ModelUtilTests(TestCase):
         """ Verify the method returns the names of all relational fields for a model. """
         self.assertEqual(get_all_related_field_names(UnrelatedModel), [])
         self.assertEqual(set(get_all_related_field_names(RelatedModel)), {'foreignrelatedmodel', 'm2mrelatedmodel'})
+
+
+class SearchQuerySetWrapperTests(TestCase):
+    def setUp(self):
+        super(SearchQuerySetWrapperTests, self).setUp()
+        title = 'Some random course'
+        query = 'title:' + title
+
+        CourseRunFactory.create_batch(3, title=title)
+        self.search_queryset = SearchQuerySet().models(CourseRun).raw_search(query).load_all()
+        self.course_runs = [e.object for e in self.search_queryset]
+        self.wrapper = SearchQuerySetWrapper(self.search_queryset)
+
+    def test_count(self):
+        self.assertEqual(self.search_queryset.count(), self.wrapper.count())
+
+    def test_iter(self):
+        self.assertEqual([e for e in self.course_runs], [e for e in self.wrapper])
+
+    def test_getitem(self):
+        self.assertEqual(self.course_runs[0], self.wrapper[0])

course_discovery/apps/core/utils.py

@@ -58,3 +58,25 @@ def delete_orphans(model):
     field_names = get_all_related_field_names(model)
     kwargs = {'{0}__isnull'.format(field_name): True for field_name in field_names}
     model.objects.filter(**kwargs).delete()
+
+
+class SearchQuerySetWrapper(object):
+    """
+    Decorates a SearchQuerySet object using a generator for efficient iteration
+    """
+    def __init__(self, qs):
+        self.qs = qs
+
+    def count(self):
+        return self.qs.count()
+
+    def __iter__(self):
+        for result in self.qs:
+            yield result.object
+
+    def __getitem__(self, key):
+        if isinstance(key, int) and (key >= 0 or key < self.count()):
+            # return the object at the specified position
+            return self.qs[key].object
+        # Pass the slice/range on to the delegate
+        return SearchQuerySetWrapper(self.qs[key])

course_discovery/apps/course_metadata/models.py

@@ -11,6 +11,7 @@ from sortedm2m.fields import SortedManyToManyField
 
 from course_discovery.apps.core.models import Currency
 from course_discovery.apps.course_metadata.query import CourseQuerySet
+from course_discovery.apps.course_metadata.utils import clean_query
 from course_discovery.apps.ietf_language_tags.models import LanguageTag
 
 logger = logging.getLogger(__name__)
@@ -163,8 +164,7 @@ class Course(TimeStampedModel):
         Returns:
             QuerySet
         """
-        # NOTE (CCB): Ensure the query is lowercase, since that is how we index our data.
-        query = query.lower()
+        query = clean_query(query)
         results = SearchQuerySet().models(cls).raw_search(query)
         ids = [result.pk for result in results]
         return cls.objects.filter(pk__in=ids)
@@ -255,6 +255,19 @@ class CourseRun(TimeStampedModel):
         value = value or None
         self.full_description_override = value
 
+    @classmethod
+    def search(cls, query):
+        """ Queries the search index.
+
+        Args:
+            query (str) -- Elasticsearch querystring (e.g. `title:intro*`)
+
+        Returns:
+            SearchQuerySet
+        """
+        query = clean_query(query)
+        return SearchQuerySet().models(cls).raw_search(query).load_all()
+
     def __str__(self):
         return '{key}: {title}'.format(key=self.key, title=self.title)
 

course_discovery/apps/course_metadata/search_indexes.py

 from haystack import indexes
+from opaque_keys.edx.keys import CourseKey
 
-from course_discovery.apps.course_metadata.models import Course
+from course_discovery.apps.course_metadata.models import Course, CourseRun
 
 
 class CourseIndex(indexes.SearchIndex, indexes.Indexable):
@@ -40,3 +41,47 @@ class CourseIndex(indexes.SearchIndex, indexes.Indexable):
 
     def get_updated_field(self):  # pragma: no cover
         return 'modified'
+
+
+class CourseRunIndex(indexes.SearchIndex, indexes.Indexable):
+    text = indexes.CharField(document=True, use_template=True)
+    course_key = indexes.CharField(model_attr='course__key', stored=True)
+    key = indexes.CharField(model_attr='key', stored=True)
+    org = indexes.CharField()
+    number = indexes.CharField()
+    title = indexes.CharField(model_attr='title_override', null=True)
+    start = indexes.DateTimeField(model_attr='start', null=True)
+    end = indexes.DateTimeField(model_attr='end', null=True)
+    enrollment_start = indexes.DateTimeField(model_attr='enrollment_start', null=True)
+    enrollment_end = indexes.DateTimeField(model_attr='enrollment_end', null=True)
+    announcement = indexes.DateTimeField(model_attr='announcement', null=True)
+    min_effort = indexes.IntegerField(model_attr='min_effort', null=True)
+    max_effort = indexes.IntegerField(model_attr='max_effort', null=True)
+    language = indexes.CharField(null=True)
+    transcript_languages = indexes.MultiValueField()
+    pacing_type = indexes.CharField(model_attr='pacing_type', null=True)
+
+    def _prepare_language(self, language):
+        return '{code}: {name}'.format(code=language.code, name=language.name)
+
+    def prepare_language(self, obj):
+        if obj.language:
+            return self._prepare_language(obj.language)
+        return None
+
+    def prepare_number(self, obj):
+        course_run_key = CourseKey.from_string(obj.key)
+        return course_run_key.course
+
+    def prepare_org(self, obj):
+        course_run_key = CourseKey.from_string(obj.key)
+        return course_run_key.org
+
+    def prepare_transcript_languages(self, obj):
+        return [self._prepare_language(language) for language in obj.transcript_languages.all()]
+
+    def get_model(self):
+        return CourseRun
+
+    def get_updated_field(self):  # pragma: no cover
+        return 'modified'

course_discovery/apps/course_metadata/tests/test_models.py

@@ -4,8 +4,9 @@ import ddt
 import pytz
 from django.test import TestCase
 
+from course_discovery.apps.core.utils import SearchQuerySetWrapper
 from course_discovery.apps.course_metadata.models import (
-    AbstractNamedModel, AbstractMediaModel, AbstractValueModel, CourseOrganization, Course
+    AbstractNamedModel, AbstractMediaModel, AbstractValueModel, CourseOrganization, Course, CourseRun
 )
 from course_discovery.apps.course_metadata.tests import factories
 
@@ -102,6 +103,15 @@ class CourseRunTests(TestCase):
         self.assertIsNone(getattr(self.course_run, override_field_name))
         self.assertEqual(getattr(self.course_run, field_name), getattr(self.course_run.course, field_name))
 
+    def test_search(self):
+        """ Verify the method returns a filtered queryset of course runs. """
+        title = 'Some random course run'
+        course_runs = factories.CourseRunFactory.create_batch(3, title=title)
+        query = 'title:' + title
+        actual_sorted = sorted(SearchQuerySetWrapper(CourseRun.search(query)), key=lambda course_run: course_run.key)
+        expected_sorted = sorted(course_runs, key=lambda course_run: course_run.key)
+        self.assertEqual(actual_sorted, expected_sorted)
+
 
 class OrganizationTests(TestCase):
     """ Tests for the `Organization` model. """

course_discovery/apps/course_metadata/utils.py

+RESERVED_ELASTICSEARCH_QUERY_OPERATORS = ('AND', 'OR', 'NOT', 'TO',)
+
+
+def clean_query(query):
+    """ Prepares a raw query for search.
+
+    Args:
+        query (str): query to clean.
+
+    Returns:
+        str: cleaned query
+    """
+    # Ensure the query is lowercase, since that is how we index our data.
+    query = query.lower()
+
+    # Ensure all operators are uppercase
+    for operator in RESERVED_ELASTICSEARCH_QUERY_OPERATORS:
+        old = ' {0} '.format(operator.lower())
+        new = ' {0} '.format(operator.upper())
+        query = query.replace(old, new)
+
+    return query

course_discovery/settings/base.py

@@ -303,7 +303,7 @@ ELASTICSEARCH_INDEX_NAME = 'catalog'
 
 HAYSTACK_CONNECTIONS = {
     'default': {
-        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'ENGINE': 'course_discovery.apps.core.haystack_backends.SimplifiedElasticsearchSearchEngine',
         'URL': ELASTICSEARCH_URL,
         'INDEX_NAME': ELASTICSEARCH_INDEX_NAME,
     },

course_discovery/static/js/catalog-preview.js

@@ -23,15 +23,7 @@ function getApiResponse(url) {
  * Form submission handler. Sends the query to the server and displays the list of courses.\
  */
 function onSubmit(e) {
-    var query = {
-            "query": {
-                "query_string": {
-                    "query": $query.val(),
-                    "analyze_wildcard": true
-                }
-            }
-        },
-        url = '/api/v1/courses/?q=' + encodeURIComponent(JSON.stringify(query));
+    var url = '/api/v1/course_runs/?q=' + encodeURIComponent($query.val());
 
     e.preventDefault();
 
@@ -56,16 +48,19 @@ function populateQueryWithExample(e) {
  */
 function populateFieldsTable() {
     var data = [
-        ['end', 'Course end date'],
+        ['announcement', 'Date the course is announced to the public'],
+        ['end', 'Course run end date'],
         ['enrollment_start', 'Enrollment start date'],
         ['enrollment_end', 'Enrollment end date'],
-        ['id', 'Course ID'],
-        ['name', 'Course name'],
+        ['key', 'Course run key'],
+        ['language', 'Language in which the course is administered'],
+        ['max_effort', 'Estimated maximum number of hours necessary to complete the course run'],
+        ['min_effort', 'Estimated minimum number of hours necessary to complete the course run'],
         ['number', 'Course number (e.g. 6.002x)'],
         ['org', 'Organization (e.g. MITx)'],
-        ['start', 'Course start date'],
-        ['type', 'Type of course (audit, credit, professional, verified)'],
-        ['verification_deadline', 'Final date to submit identity verification'],
+        ['pacing_type', 'Course run pacing. Options are either "instructor_paced" or "self_paced"'],
+        ['start', 'Course run start date'],
+        ['title', 'Course run title']
     ];
     $("#fields").DataTable({
         info: false,
@@ -90,12 +85,15 @@ $(document).ready(function () {
         paging: true,
         columns: [
             {
-                title: 'Course ID',
-                data: 'id'
+                title: 'Course Run Key',
+                data: 'key',
+                fnCreatedCell: function (nTd, sData, oData, iRow, iCol) {
+                    $(nTd).html("<a href='/api/v1/course_runs/" + oData.key + "/' target='_blank'>" + oData.key + "</a>");
+                }
             },
             {
-                title: 'Name',
-                data: 'name'
+                title: 'Title',
+                data: 'title'
             }
         ],
         oLanguage: {

course_discovery/templates/catalogs/preview.html

@@ -65,10 +65,6 @@
         <td><a class="example">org:(-MITx OR -HarvardX)</a></td>
       </tr>
       <tr>
-        <td>Courses of a particular type. Options include audit, credit, honor, professional, verified.</td>
-        <td><a class="example">type:credit</a></td>
-      </tr>
-      <tr>
         <td>Courses starting in a specific time period</td>
         <td><a class="example">start:[2016-01-01 TO 2016-12-31]</a></td>
       </tr>
@@ -108,7 +104,7 @@
       <table id="courses" class="table table-striped table-bordered" cellspacing="0">
         <thead>
         <tr>
-          <th>Course ID</th>
+          <th>Course Run Key</th>
           <th>Name</th>
         </tr>
         </thead>

course_discovery/templates/search/indexes/course_metadata/courserun_text.txt

+{{ object.key }}
+{{ object.title }}
+{{ object.short_description|default:'' }}
+{{ object.full_description|default:'' }}
+{{ object.pacing_type|default:'' }}