Merge pull request #11 from cpennington/configurable-elasticsearch

Make ELASTICSEARCH behave more like DATABASES for the purposes of int…

Makefile

@@ -54,6 +54,7 @@ validate: test quality
 
 migrate:
 	python manage.py migrate --noinput
+	python manage.py install_es_indexes --noinput
 
 html_coverage:
 	coverage html && open htmlcov/index.html

course_discovery/apps/courses/apps.py

-import logging
-
 from django.apps import AppConfig
-from django.conf import settings
-from elasticsearch import Elasticsearch, TransportError
-
-from course_discovery.apps.courses.config import COURSES_INDEX_CONFIG
-
-logger = logging.getLogger(__name__)
 
 
 class CoursesConfig(AppConfig):
     name = 'courses'
     verbose_name = 'Courses'
-
-    def ready(self):
-        if settings.ELASTICSEARCH.get('connect_on_startup', True):
-            host = settings.ELASTICSEARCH['host']
-            index = settings.ELASTICSEARCH['index']
-
-            logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host)
-            es = Elasticsearch(host, sniff_on_start=True)
-            logger.info('...success!')
-
-            logger.info('Making sure index [%s] exists...', index)
-            try:
-                es.indices.create(index=index, body=COURSES_INDEX_CONFIG)
-                logger.info('...index created.')
-            except TransportError as e:
-                if e.status_code == 400:
-                    logger.info('...index already exists.')
-                else:
-                    raise

course_discovery/apps/courses/management/commands/install_es_indexes.py

+import logging
+
+from django.conf import settings
+from django.core.management import BaseCommand
+
+from elasticsearch import Elasticsearch, TransportError
+
+from course_discovery.apps.courses.config import COURSES_INDEX_CONFIG
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Install any required elasticsearch indexes'
+
+    def handle(self, *args, **options):
+        host = settings.ELASTICSEARCH['host']
+        index = settings.ELASTICSEARCH['index']
+
+        logger.info('Attempting to establish initial connection to Elasticsearch host [%s]...', host)
+        es = Elasticsearch(host, sniff_on_start=True)
+        logger.info('...success!')
+
+        logger.info('Making sure index [%s] exists...', index)
+        try:
+            es.indices.create(index=index, body=COURSES_INDEX_CONFIG)
+            logger.info('...index created.')
+        except TransportError as e:
+            if e.status_code == 400:
+                logger.info('...index already exists.')
+            else:
+                raise

course_discovery/apps/courses/management/commands/refresh_all_courses.py

 import logging
-from optparse import make_option
 
 from django.core.management import BaseCommand, CommandError
 
@@ -11,12 +10,13 @@ logger = logging.getLogger(__name__)
 class Command(BaseCommand):
     help = 'Refresh course data from external sources.'
 
-    option_list = BaseCommand.option_list + (
-        make_option('--access_token',
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--access_token',
             action='store',
             dest='access_token',
             default=None,
-                    help='OAuth2 access token used to authenticate API calls.'),
+            help='OAuth2 access token used to authenticate API calls.'
         )
 
     def handle(self, *args, **options):

course_discovery/apps/courses/tests/test_apps.py → course_discovery/apps/courses/tests/test_install_es_indexes.py

 import mock
-from django.apps import AppConfig
 from django.conf import settings
-from django.test import TestCase, override_settings
+from django.test import TestCase
+from django.core.management import call_command
 from elasticsearch import TransportError
 from elasticsearch.client import IndicesClient
 from testfixtures import LogCapture
 
 from course_discovery.apps.core.tests.mixins import ElasticsearchTestMixin
 
-LOGGER_NAME = 'course_discovery.apps.courses.apps'
+LOGGER_NAME = 'courses.management.commands.install_es_indexes'
 
 
-class CoursesConfigTests(ElasticsearchTestMixin, TestCase):
-    def setUp(self):
-        super(CoursesConfigTests, self).setUp()
-        self.app_config = AppConfig.create('course_discovery.apps.courses')
-
+class CourseInstallEsIndexes(ElasticsearchTestMixin, TestCase):
     def test_ready_create_index(self):
         """ Verify the app does not setup a new Elasticsearch index if one exists already. """
         host = settings.ELASTICSEARCH['host']
@@ -26,7 +22,7 @@ class CoursesConfigTests(ElasticsearchTestMixin, TestCase):
         self.assertFalse(self.es.indices.exists(index=index))
 
         with LogCapture(LOGGER_NAME) as l:
-            self.app_config.ready()
+            call_command('install_es_indexes')
 
             # Verify the index was created
             self.assertTrue(self.es.indices.exists(index=index))
@@ -52,7 +48,7 @@ class CoursesConfigTests(ElasticsearchTestMixin, TestCase):
 
             with LogCapture(LOGGER_NAME) as l:
                 # This call should NOT raise an exception.
-                self.app_config.ready()
+                call_command('install_es_indexes')
 
         # Verify the index still exists
         self.assertTrue(self.es.indices.exists(index=index))
@@ -71,13 +67,4 @@ class CoursesConfigTests(ElasticsearchTestMixin, TestCase):
             mock_create.side_effect = TransportError(500)
 
             with self.assertRaises(TransportError):
-                self.app_config.ready()
-
-    @override_settings(ELASTICSEARCH={'connect_on_startup': False})
-    def test_ready_without_connect_on_startup(self):
-        """
-        Verify the app does not attempt to connect to Elasticsearch if the connect_on_startup setting is not set.
-        """
-        with mock.patch.object(IndicesClient, 'create') as mock_create:
-            self.app_config.ready()
-            mock_create.assert_not_called()
+                call_command('install_es_indexes')

course_discovery/settings/base.py

@@ -249,9 +249,8 @@ SWAGGER_SETTINGS = {
 }
 
 ELASTICSEARCH = {
-    'host': 'es',
+    'host': '',
     'index': 'course_discovery',
-    'connect_on_startup': True
 }
 
 # TODO Replace with None and document.

course_discovery/settings/production.py

@@ -48,5 +48,11 @@ DB_OVERRIDES = dict(
     PORT=environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
 )
 
+ES_OVERRIDES = dict(
+    HOST=environ.get('ES_HOST', ELASTICSEARCH['host']),
+    INDEX=environ.get('ES_INDEX', ELASTICSEARCH['index']),
+)
+
+
 for override, value in DB_OVERRIDES.items():
     DATABASES['default'][override] = value

course_discovery/settings/test.py

@@ -33,5 +33,4 @@ DATABASES = {
 ELASTICSEARCH = {
     'host': os.environ.get('TEST_ELASTICSEARCH_HOST', 'localhost'),
     'index': 'course_discovery_test',
-    'connect_on_startup': True
 }