Using django-haystack for Elasticsearch integration

Haystack replaces the direct access to Elasticsearch. This gives us the ability to automatically index models as they are updated and/or deleted.

Additionally, the following changes were made:
- Moved ES-related management command and utils to core app
- Replaced TEST_ELASTICSEARCH_HOST with TEST_ELASTICSEARCH_URL

ECOM-3972

course_discovery/apps/api/v1/tests/test_views.py

@@ -2,7 +2,6 @@
 import json
 import urllib
 from time import time
-from unittest import skip
 
 import ddt
 import jwt
@@ -141,7 +140,6 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
         self.mock_user_info_response(self.user)
         self.assert_catalog_created(HTTP_AUTHORIZATION=self.generate_oauth2_token_header(self.user))
 
-    @skip('Re-enable once we switch to Haystack')
     def test_courses(self):
         """ Verify the endpoint returns the list of courses contained in the catalog. """
         url = reverse('api:v1:catalog-courses', kwargs={'id': self.catalog.id})
@@ -151,7 +149,6 @@ class CatalogViewSetTests(ElasticsearchTestMixin, SerializationMixin, OAuth2Mixi
         self.assertEqual(response.status_code, 200)
         self.assertListEqual(response.data['results'], self.serialize_course(courses, many=True))
 
-    @skip('Re-enable once we switch to Haystack')
     def test_contains(self):
         """ Verify the endpoint returns a filtered list of courses contained in the catalog. """
         course_key = self.course.key

course_discovery/apps/catalogs/models.py

-import json
-
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from haystack.query import SearchQuerySet
+
+from course_discovery.apps.course_metadata.models import Course
 
 
 class Catalog(TimeStampedModel):
@@ -12,9 +13,14 @@ class Catalog(TimeStampedModel):
     def __str__(self):
         return 'Catalog #{id}: {name}'.format(id=self.id, name=self.name)  # pylint: disable=no-member
 
-    @property
-    def query_as_dict(self):
-        return json.loads(self.query)
+    def _get_query_results(self):
+        """
+        Returns the results of this Catalog's query.
+
+        Returns:
+            SearchQuerySet
+        """
+        return SearchQuerySet().models(Course).raw_search(self.query)
 
     def courses(self):
         """ Returns the list of courses contained within this catalog.
@@ -22,9 +28,8 @@ class Catalog(TimeStampedModel):
         Returns:
             Course[]
         """
-
-        # TODO: Course.search no longer exists.  Figure out what goes here.
-        # return Course.search(self.query_as_dict)['results']
+        results = self._get_query_results().load_all()
+        return [result.object for result in results]
 
     def contains(self, course_ids):  # pylint: disable=unused-argument
         """ Determines if the given courses are contained in this catalog.
@@ -36,29 +41,9 @@ class Catalog(TimeStampedModel):
             dict: Mapping of course IDs to booleans indicating if course is
                   contained in this catalog.
         """
-        # query = self.query_as_dict['query']
-
-        # # Create a filtered query that includes that uses the catalog's query against a
-        # # collection of courses filtered using the passed in course IDs.
-        # filtered_query = {
-        #     "query": {
-        #         "filtered": {
-        #             "query": query,
-        #             "filter": {
-        #                 "ids": {
-        #                     "values": course_ids
-        #                 }
-        #             }
-        #         }
-        #     }
-        # }
-
-        # contains = {course_id: False for course_id in course_ids}
-
-        # TODO: Course.search no longer exists.  Figure out what goes here.
-        # courses = Course.search(filtered_query)['results']
-        # for course in courses:
-        #     contains[course.id] = True
+        contains = {course_id: False for course_id in course_ids}
+        results = self._get_query_results().filter(key__in=course_ids)
+        for result in results:
+            contains[result.get_stored_fields()['key']] = True
 
-        # return contains
-        pass
+        return contains

course_discovery/apps/catalogs/tests/test_models.py

-import json
-from unittest import skip
-
 from django.test import TestCase
 
 from course_discovery.apps.catalogs.tests import factories
@@ -13,20 +10,7 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
 
     def setUp(self):
         super(CatalogTests, self).setUp()
-        query = {
-            'query': {
-                'bool': {
-                    'must': [
-                        {
-                            'wildcard': {
-                                'course.title': 'abc*'
-                            }
-                        }
-                    ]
-                }
-            }
-        }
-        self.catalog = factories.CatalogFactory(query=json.dumps(query))
+        self.catalog = factories.CatalogFactory(query='title:abc*')
         self.course = CourseFactory(key='a/b/c', title='ABCs of Ͳҽʂէìղց')
         self.refresh_index()
 
@@ -39,16 +23,14 @@ class CatalogTests(ElasticsearchTestMixin, TestCase):
         expected = 'Catalog #{id}: {name}'.format(id=self.catalog.id, name=name)
         self.assertEqual(str(self.catalog), expected)
 
-    @skip('Skip until searching in ES is resolved')
     def test_courses(self):
         """ Verify the method returns a list of courses contained in the catalog. """
         self.assertEqual(self.catalog.courses(), [self.course])
 
-    @skip('Skip until searching in ES is resolved')
     def test_contains(self):
         """ Verify the method returns a mapping of course IDs to booleans. """
-        other_id = 'd/e/f'
+        uncontained_course = CourseFactory(key='d/e/f', title='ABDEF')
         self.assertDictEqual(
-            self.catalog.contains([self.course.key, other_id]),
-            {self.course.key: True, other_id: False}
+            self.catalog.contains([self.course.key, uncontained_course.key]),
+            {self.course.key: True, uncontained_course.key: False}
         )

course_discovery/apps/course_metadata/management/commands/install_es_indexes.py → course_discovery/apps/core/management/commands/install_es_indexes.py

@@ -13,8 +13,8 @@ class Command(BaseCommand):
     help = 'Install any required Elasticsearch indexes'
 
     def handle(self, *args, **options):
-        host = settings.ELASTICSEARCH['host']
-        alias = settings.ELASTICSEARCH['index']
+        host = settings.HAYSTACK_CONNECTIONS['default']['URL']
+        alias = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
 
         logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host)
         es = Elasticsearch(host)

course_discovery/apps/core/tests/mixins.py

@@ -12,8 +12,8 @@ class ElasticsearchTestMixin(object):
     @classmethod
     def setUpClass(cls):
         super(ElasticsearchTestMixin, cls).setUpClass()
-        host = settings.ELASTICSEARCH['host']
-        cls.index = settings.ELASTICSEARCH['index']
+        host = settings.HAYSTACK_CONNECTIONS['default']['URL']
+        cls.index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
         cls.es = Elasticsearch(host)
 
     def setUp(self):

course_discovery/apps/course_metadata/tests/test_install_es_indexes.py → course_discovery/apps/core/tests/test_install_es_indexes.py

@@ -10,7 +10,7 @@ LOGGER_NAME = 'courses.management.commands.install_es_indexes'
 class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase):
     def test_create_index(self):
         """ Verify the app sets the alias and creates a new index. """
-        index = settings.ELASTICSEARCH['index']
+        index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
 
         # Delete the index
         self.es.indices.delete(index=index, ignore=404)  # pylint: disable=unexpected-keyword-arg
@@ -23,7 +23,7 @@ class InstallEsIndexesCommandTests(ElasticsearchTestMixin, TestCase):
 
     def test_alias_exists(self):
         """ Verify the app does not setup a new Elasticsearch index if the alias is already set. """
-        index = settings.ELASTICSEARCH['index']
+        index = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
 
         # Verify the index exists
         self.assertTrue(self.es.indices.exists(index=index))

course_discovery/apps/course_metadata/search_indexes.py

+from haystack import indexes
+
+from course_discovery.apps.course_metadata.models import Course
+
+
+class CourseIndex(indexes.SearchIndex, indexes.Indexable):
+    text = indexes.CharField(document=True, use_template=True)
+    key = indexes.CharField(model_attr='key', stored=True)
+    title = indexes.CharField(model_attr='title')
+    organizations = indexes.MultiValueField()
+    short_description = indexes.CharField(model_attr='short_description', null=True)
+    full_description = indexes.CharField(model_attr='full_description', null=True)
+    level_type = indexes.CharField(model_attr='level_type__name', null=True)
+
+    def prepare_organizations(self, obj):
+        return [organization.name for organization in obj.organizations.all()]
+
+    def get_model(self):
+        return Course
+
+    def index_queryset(self, using=None):
+        return self.get_model().objects.all()
+
+    def get_updated_field(self):  # pragma: no cover
+        return 'modified'

course_discovery/settings/base.py

@@ -35,6 +35,7 @@ THIRD_PARTY_APPS = (
     'waffle',
     'sortedm2m',
     'simple_history',
+    'haystack',
 )
 
 PROJECT_APPS = (
@@ -276,11 +277,19 @@ SWAGGER_SETTINGS = {
     'doc_expansion': 'list',
 }
 
-ELASTICSEARCH = {
-    'host': 'localhost:9200',
-    'index': 'course_discovery',
+ELASTICSEARCH_URL = 'http://127.0.0.1:9200/'
+ELASTICSEARCH_INDEX_NAME = 'catalog'
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': ELASTICSEARCH_URL,
+        'INDEX_NAME': ELASTICSEARCH_INDEX_NAME,
+    },
 }
 
+HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
+
 # TODO Replace with None and document.
 ECOMMERCE_API_URL = 'https://ecommerce.stage.edx.org/api/v2/'
 COURSES_API_URL = 'https://courses.stage.edx.org/api/courses/v1/'

course_discovery/settings/devstack.py

@@ -20,6 +20,15 @@ if os.environ.get('ENABLE_DJANGO_TOOLBAR', False):
 INTERNAL_IPS = ('127.0.0.1',)
 # END TOOLBAR CONFIGURATION
 
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': 'http://es:9200/',
+        'INDEX_NAME': 'catalog',
+    },
+}
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')):

course_discovery/settings/test.py

-import os
 from course_discovery.settings.base import *
 
 # TEST SETTINGS
@@ -30,9 +29,12 @@ DATABASES = {
 }
 # END IN-MEMORY TEST DATABASE
 
-ELASTICSEARCH = {
-    'host': os.environ.get('TEST_ELASTICSEARCH_HOST', 'localhost'),
-    'index': 'course_discovery_test',
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': os.environ.get('TEST_ELASTICSEARCH_URL', 'http://127.0.0.1:9200/'),
+        'INDEX_NAME': 'catalog_test',
+    },
 }
 
 JWT_AUTH['JWT_SECRET_KEY'] = 'course-discovery-jwt-secret-key'

course_discovery/templates/search/indexes/course_metadata/course_text.txt

+{{ object.key }}
+{{ object.title }}
+{{ object.organizations.all|default:'' }}
+{{ object.short_description|default:'' }}
+{{ object.full_description|default:'' }}
+{{ object.level_type|default:'' }}

docker-compose.yml

@@ -35,7 +35,7 @@ course-discovery:
     - .:/edx/app/course_discovery/discovery
   command: /edx/app/discovery/devstack.sh start
   environment:
-    TEST_ELASTICSEARCH_HOST: "es"
+    TEST_ELASTICSEARCH_URL: "http://es:9200"
   ports:
     - "18381:18381"
     - "8381:8381"

requirements/base.txt

 django==1.8.7
 django-extensions==1.5.9
+django-haystack==2.4.1
 django-simple-history==1.8.1
 django-sortedm2m==1.1.1
 django-waffle==0.11