Add model.

.travis.yml

 language: python
 python: 2.7
-
-services:
-    - elasticsearch
+sudo: false
 
 install:
+    - scripts/travis/install.sh
+    - scripts/travis/setup.sh
     - pip install -r requirements/test.txt
     - git fetch origin master:refs/remotes/origin/master # https://github.com/edx/diff-cover#troubleshooting
     - pip install coveralls
@@ -14,3 +14,8 @@ script:
 
 after_success:
     - coveralls
+
+env:
+- ESVER=-
+- ESVER=0.90.11
+- ESVER=1.4.2

Makefile

@@ -3,9 +3,16 @@ PACKAGES = notesserver notesapi
 
 validate: test.requirements test coverage
 
+ifeq ($(ESVER),-)
+test_settings = notesserver.settings.test_es_disabled
+else
+test_settings = notesserver.settings.test
+endif
+
 test: clean
-	./manage.py test --settings=notesserver.settings.test --with-coverage --with-ignore-docstrings \
+	./manage.py test --settings=$(test_settings) --with-coverage --with-ignore-docstrings \
 		--exclude-dir=notesserver/settings --cover-inclusive --cover-branches \
+		--ignore-files=search_indexes.py --ignore-files=highlight.py\
 		--cover-html --cover-html-dir=build/coverage/html/ \
 		--cover-xml --cover-xml-file=build/coverage/coverage.xml \
 		$(foreach package,$(PACKAGES),--cover-package=$(package)) \
@@ -34,12 +41,15 @@ diff-quality:
 coverage: diff-coverage diff-quality
 
 create-index:
-	python manage.py create_index
+	python manage.py rebuild_index
 
 requirements:
 	pip install -q -r requirements/base.txt --exists-action=w
 
 test.requirements: requirements
 	pip install -q -r requirements/test.txt --exists-action=w
+	@# unicode QUERY_PARAMS are being improperly decoded in test client
+	@# remove after https://github.com/tomchristie/django-rest-framework/issues/1891 is fixed
+	pip install -q -e git+https://github.com/tymofij/django-rest-framework.git@bugfix/test-unicode-query-params#egg=djangorestframework
 
 develop: test.requirements

README.rst

@@ -11,13 +11,12 @@ Overview
 --------
 
 The edX Notes API is designed to be compatible with the
-`Annotator <http://annotatorjs.org/>`__.
+`Annotator <http://annotatorjs.org/>`__. Can be run with up to date ElasticSearch or legacy 0.90.x.
 
 Getting Started
 ---------------
 
-1. You'll need a recent version `ElasticSearch <http://elasticsearch.org>`__ (>=1.0.0)
-installed.
+1. You'll need an `ElasticSearch <http://elasticsearch.org>`__ installed.
 
 2. Install the requirements:
 
@@ -37,6 +36,19 @@ installed.
 
        $ make run
 
+Configuration:
+--------------
+
+``CLIENT_ID`` - OAuth2 Client ID, which is to be found in ``aud`` field of IDTokens which authorize users
+
+``CLIENT_SECRET`` - secret with which IDTokens should be encoded
+
+``ES_DISABLED`` - set to True when you need to run the service without ElasticSearch support.
+                  e.g if it became corrupted and you're rebuilding the index, while still serving users
+                  through MySQL
+
+``HAYSTACK_CONNECTIONS['default']['url']`` - Your ElasticSearch URL
+
 Running Tests
 -------------
 

notesapi/management/commands/create_index.py

-from django.core.management.base import BaseCommand
-from annotator.annotation import Annotation
-
-
-class Command(BaseCommand):
-    help = 'Creates the mapping in the index.'
-
-    def handle(self, *args, **options):
-        Annotation.create_all()

notesapi/v1/models.py

+import json
+from django.db import models
+from django.core.exceptions import ValidationError
+
+
+class Note(models.Model):
+    """
+    Annotation model.
+    """
+    user_id = models.CharField(max_length=255, db_index=True, help_text="Anonymized user id, not course specific")
+    course_id = models.CharField(max_length=255, db_index=True)
+    usage_id = models.CharField(max_length=255, help_text="ID of XBlock where the text comes from")
+    quote = models.TextField(default="")
+    text = models.TextField(default="", help_text="Student's thoughts on the quote")
+    ranges = models.TextField(help_text="JSON, describes position of quote in the source text")
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+
+    @classmethod
+    def create(cls, note_dict):
+        """
+        Create the note object.
+        """
+        if not isinstance(note_dict, dict):
+            raise ValidationError('Note must be a dictionary.')
+
+        if len(note_dict) == 0:
+            raise ValidationError('Note must have a body.')
+
+        ranges = note_dict.get('ranges', list())
+
+        if len(ranges) < 1:
+            raise ValidationError('Note must contain at least one range.')
+
+        note_dict['ranges'] = json.dumps(ranges)
+        note_dict['user_id'] = note_dict.pop('user', None)
+
+        return cls(**note_dict)
+
+    def as_dict(self):
+        """
+        Returns the note object as a dictionary.
+        """
+        created = self.created.isoformat() if self.created else None
+        updated = self.updated.isoformat() if self.updated else None
+
+        return {
+            'id': str(self.pk),
+            'user': self.user_id,
+            'course_id': self.course_id,
+            'usage_id': self.usage_id,
+            'text': self.text,
+            'quote': self.quote,
+            'ranges': json.loads(self.ranges),
+            'created': created,
+            'updated': updated,
+        }

notesapi/v1/permissions.py

@@ -40,21 +40,26 @@ class HasAccessToken(BasePermission):
             auth_user = data['sub']
             if data['aud'] != settings.CLIENT_ID:
                 raise TokenWrongIssuer
+            user_found = False
             for request_field in ('GET', 'POST', 'DATA'):
                 if 'user' in getattr(request, request_field):
                     req_user = getattr(request, request_field)['user']
                     if req_user == auth_user:
-                        return True
+                        user_found = True
+                        # but we do not break or return here,
+                        # because `user` may be present in more than one field (GET, POST)
+                        # and we must make sure that all of them are correct
                     else:
-                        logger.debug("Token user {auth_user} did not match {field} user {req_user}".format(
-                            auth_user=auth_user, field=request_field, req_user=req_user
-                        ))
+                        logger.debug("Token user %s did not match %s user %s", auth_user, request_field, req_user)
                         return False
-            logger.info("No user was present to compare in GET, POST or DATA")
+            if user_found:
+                return True
+            else:
+                logger.info("No user was present to compare in GET, POST or DATA")
         except jwt.ExpiredSignature:
-            logger.debug("Token was expired: {}".format(token))
+            logger.debug("Token was expired: %s", token)
         except jwt.DecodeError:
-            logger.debug("Could not decode token {}".format(token))
+            logger.debug("Could not decode token %s", token)
         except TokenWrongIssuer:
-            logger.debug("Token has wrong issuer {}".format(token))
-        return False
+            logger.debug("Token has wrong issuer %s", token)
+        return False
\ No newline at end of file

notesapi/v1/search_indexes.py

+from haystack import indexes
+from .models import Note
+
+
+class NoteIndex(indexes.SearchIndex, indexes.Indexable):
+    user = indexes.CharField(model_attr='user_id', indexed=False)
+    course_id = indexes.CharField(model_attr='course_id', indexed=False)
+    usage_id = indexes.CharField(model_attr='usage_id', indexed=False)
+    quote = indexes.CharField(model_attr='quote')
+    text = indexes.CharField(document=True, model_attr='text')
+    ranges = indexes.CharField(model_attr='ranges', indexed=False)
+    created = indexes.DateTimeField(model_attr='created')
+    updated = indexes.DateTimeField(model_attr='updated')
+
+    def get_model(self):
+        return Note
+
+    def index_queryset(self, using=None):
+        """Used when the entire index for model is updated."""
+        return self.get_model().objects.all()

notesapi/v1/tests/test_models.py

+from unittest import TestCase
+from notesapi.v1.models import Note
+from django.core.exceptions import ValidationError
+
+
+class NoteTest(TestCase):
+    def setUp(self):
+
+        self.note_dict = {
+            "user": u"test_user_id",
+            "usage_id": u"i4x://org/course/html/52aa9816425a4ce98a07625b8cb70811",
+            "course_id": u"org/course/run",
+            "text": u"test note text",
+            "quote": u"test note quote",
+            "ranges": [
+                {
+                    "start": u"/p[1]",
+                    "end": u"/p[1]",
+                    "startOffset": 0,
+                    "endOffset": 10,
+                }
+            ],
+        }
+
+    def test_create_valid_note(self):
+        note = Note.create(self.note_dict.copy())
+        note.save()
+
+        result_note = note.as_dict()
+        del result_note['id']
+        del result_note['created']
+        del result_note['updated']
+
+        self.assertEqual(result_note, self.note_dict)
+
+    def test_create_invalid_note(self):
+        note = Note()
+        for empty_type in (None, '', []):
+            with self.assertRaises(ValidationError):
+                note.create(empty_type)
+
+    def test_must_have_fields_create(self):
+        for field in ['user', 'usage_id', 'course_id', 'ranges']:
+            payload = self.note_dict.copy()
+            payload.pop(field)
+
+            with self.assertRaises(ValidationError):
+                note = Note.create(payload)
+                note.full_clean()

notesapi/v1/urls.py

 from django.conf.urls import patterns, url
-from django.views.generic import RedirectView
-from django.core.urlresolvers import reverse_lazy
 from notesapi.v1.views import AnnotationListView, AnnotationDetailView, AnnotationSearchView
 
 urlpatterns = patterns(
     '',
     url(r'^annotations/$', AnnotationListView.as_view(), name='annotations'),
-    url(r'^annotations/(?P<annotation_id>[a-zA-Z0-9_-]+)/?$', AnnotationDetailView.as_view(), name='annotations_detail'),
+    url(
+        r'^annotations/(?P<annotation_id>[a-zA-Z0-9_-]+)/?$',
+        AnnotationDetailView.as_view(),
+        name='annotations_detail'
+    ),
     url(r'^search/$', AnnotationSearchView.as_view(), name='annotations_search'),
 )

notesapi/v1/views.py

+import logging
+import json
+
 from django.conf import settings
 from django.core.urlresolvers import reverse
+from django.core.exceptions import ValidationError
 
 from rest_framework import status
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from annotator.annotation import Annotation
+from notesapi.v1.models import Note
+
+if not settings.ES_DISABLED:
+    from notesserver.highlight import SearchQuerySet
 
-CREATE_FILTER_FIELDS = ('updated', 'created', 'consumer', 'id')
-UPDATE_FILTER_FIELDS = ('updated', 'created', 'user', 'consumer')
+log = logging.getLogger(__name__)
 
 
 class AnnotationSearchView(APIView):
     """
     Search annotations.
     """
-
     def get(self, *args, **kwargs):  # pylint: disable=unused-argument
         """
-        Search annotations.
+        Search annotations in most appropriate storage
+        """
+        # search in DB when ES is not available or there is no need to bother it
+        if settings.ES_DISABLED or 'text' not in self.request.QUERY_PARAMS.dict():
+            results = self.get_from_db(*args, **kwargs)
+        else:
+            results = self.get_from_es(*args, **kwargs)
+        return Response({'total': len(results), 'rows': results})
 
-        This method supports the limit and offset query parameters for paging
-        through results.
+    def get_from_db(self, *args, **kwargs):  # pylint: disable=unused-argument
+        """
+        Search annotations in database
         """
         params = self.request.QUERY_PARAMS.dict()
+        query = Note.objects.filter(
+            **{f: v for (f, v) in params.items() if f in ('course_id', 'usage_id')}
+        ).order_by('-updated')
 
-        if 'offset' in params:
-            kwargs['offset'] = _convert_to_int(params.pop('offset'))
-
-        if 'limit' in params and _convert_to_int(params['limit']) is not None:
-            kwargs['limit'] = _convert_to_int(params.pop('limit'))
-        elif 'limit' in params and _convert_to_int(params['limit']) is None:  # bad value
-            params.pop('limit')
-            kwargs['limit'] = settings.RESULTS_DEFAULT_SIZE
-        else:
-            # default
-            kwargs['limit'] = settings.RESULTS_DEFAULT_SIZE
+        if 'user' in params:
+            query = query.filter(user_id=params['user'])
 
-        # All remaining parameters are considered searched fields.
-        kwargs['query'] = params
+        if 'text' in params:
+            query = query.filter(text__icontains=params['text'])
 
-        results = Annotation.search(**kwargs)
-        total = Annotation.count(**kwargs)
+        return [note.as_dict() for note in query]
 
-        return Response({'total': total, 'rows': results})
+    def get_from_es(self, *args, **kwargs):  # pylint: disable=unused-argument
+        """
+        Search annotations in ElasticSearch
+        """
+        params = self.request.QUERY_PARAMS.dict()
+        query = SearchQuerySet().models(Note).filter(
+            **{f: v for (f, v) in params.items() if f in ('user', 'course_id', 'usage_id', 'text')}
+        ).order_by('-updated')
+
+        if params.get('highlight'):
+            tag = params.get('highlight_tag', 'em')
+            klass = params.get('highlight_class')
+            opts = {
+                'pre_tags': ['<{tag}{klass_str}>'.format(
+                    tag=tag,
+                    klass_str=' class="{}"'.format(klass) if klass else ''
+                )],
+                'post_tags': ['</{tag}>'.format(tag=tag)],
+            }
+            query = query.highlight(**opts)
+
+        results = []
+        for item in query:
+            note_dict = item.get_stored_fields()
+            note_dict['ranges'] = json.loads(item.ranges)
+            note_dict['id'] = str(item.pk)
+            if item.highlighted:
+                note_dict['text'] = item.highlighted[0]
+            results.append(note_dict)
+
+        return results
 
 
 class AnnotationListView(APIView):
@@ -55,11 +91,14 @@ class AnnotationListView(APIView):
         """
         Get a list of all annotations.
         """
-        self.kwargs['query'] = self.request.QUERY_PARAMS.dict()
+        params = self.request.QUERY_PARAMS.dict()
+
+        if 'course_id' not in params:
+            return Response(status=status.HTTP_400_BAD_REQUEST)
 
-        annotations = Annotation.search(**kwargs)
+        results = Note.objects.filter(course_id=params['course_id'], user_id=params['user']).order_by('-updated')
 
-        return Response(annotations)
+        return Response([result.as_dict() for result in results])
 
     def post(self, *args, **kwargs):  # pylint: disable=unused-argument
         """
@@ -70,17 +109,18 @@ class AnnotationListView(APIView):
         if 'id' in self.request.DATA:
             return Response(status=status.HTTP_400_BAD_REQUEST)
 
-        filtered_payload = _filter_input(self.request.DATA, CREATE_FILTER_FIELDS)
-
-        if len(filtered_payload) == 0:
+        try:
+            note = Note.create(self.request.DATA)
+            note.full_clean()
+        except ValidationError as error:
+            log.debug(error, exc_info=True)
             return Response(status=status.HTTP_400_BAD_REQUEST)
 
-        annotation = Annotation(filtered_payload)
-        annotation.save(refresh=True)
+        note.save()
 
-        location = reverse('api:v1:annotations_detail', kwargs={'annotation_id': annotation['id']})
+        location = reverse('api:v1:annotations_detail', kwargs={'annotation_id': note.id})
 
-        return Response(annotation, status=status.HTTP_201_CREATED, headers={'Location': location})
+        return Response(note.as_dict(), status=status.HTTP_201_CREATED, headers={'Location': location})
 
 
 class AnnotationDetailView(APIView):
@@ -94,66 +134,49 @@ class AnnotationDetailView(APIView):
         """
         Get an existing annotation.
         """
-        annotation_id = self.kwargs.get('annotation_id')
-        annotation = Annotation.fetch(annotation_id)
+        note_id = self.kwargs.get('annotation_id')
 
-        if not annotation:
-            return Response(annotation, status=status.HTTP_404_NOT_FOUND)
+        try:
+            note = Note.objects.get(id=note_id)
+        except Note.DoesNotExist:
+            return Response('Annotation not found!', status=status.HTTP_404_NOT_FOUND)
 
-        return Response(annotation)
+        return Response(note.as_dict())
 
     def put(self, *args, **kwargs):  # pylint: disable=unused-argument
         """
         Update an existing annotation.
         """
-        annotation_id = self.kwargs.get('annotation_id')
-        annotation = Annotation.fetch(annotation_id)
+        note_id = self.kwargs.get('annotation_id')
 
-        if not annotation:
+        try:
+            note = Note.objects.get(id=note_id)
+        except Note.DoesNotExist:
             return Response('Annotation not found! No update performed.', status=status.HTTP_404_NOT_FOUND)
 
-        if self.request.DATA is not None:
-            updated = _filter_input(self.request.DATA, UPDATE_FILTER_FIELDS)
-            updated['id'] = annotation_id  # use id from URL, regardless of what arrives in JSON payload.
-
-            annotation.update(updated)
+        try:
+            note.text = self.request.data['text']
+            note.full_clean()
+        except KeyError as error:
+            log.debug(error, exc_info=True)
+            return Response(status=status.HTTP_400_BAD_REQUEST)
 
-            refresh = self.kwargs.get('refresh') != 'false'
-            annotation.save(refresh=refresh)
+        note.save()
 
-        return Response(annotation)
+        return Response(note.as_dict())
 
     def delete(self, *args, **kwargs):  # pylint: disable=unused-argument
         """
         Delete an annotation.
         """
-        annotation_id = self.kwargs.get('annotation_id')
-        annotation = Annotation.fetch(annotation_id)
+        note_id = self.kwargs.get('annotation_id')
 
-        if not annotation:
-            return Response('Annotation not found! No delete performed.', status=status.HTTP_404_NOT_FOUND)
+        try:
+            note = Note.objects.get(id=note_id)
+        except Note.DoesNotExist:
+            return Response('Annotation not found! No update performed.', status=status.HTTP_404_NOT_FOUND)
 
-        annotation.delete()
+        note.delete()
 
         # Annotation deleted successfully.
         return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-def _filter_input(annotation, fields):
-    """
-    Pop given fields from annotation.
-    """
-    for field in fields:
-        annotation.pop(field, None)
-
-    return annotation
-
-
-def _convert_to_int(value, default=None):
-    """
-    Convert given value to int.
-    """
-    try:
-        return int(value or default)
-    except ValueError:
-        return default

notesserver/highlight.py

+"""
+django-haystack does not support passing additional highlighting parameters
+to backends, so we use our subclassed SearchQuerySet which does,
+and subclassed ElasticsearchSearchBackend which passes them to ES
+"""
+
+import haystack
+from haystack.backends.elasticsearch_backend import (
+    ElasticsearchSearchEngine as OrigElasticsearchSearchEngine,
+    ElasticsearchSearchQuery as OrigElasticsearchSearchQuery,
+    ElasticsearchSearchBackend as OrigElasticsearchSearchBackend)
+from haystack.query import SearchQuerySet as OrigSearchQuerySet
+
+
+class SearchQuerySet(OrigSearchQuerySet):
+    def highlight(self, **kwargs):
+        """Adds highlighting to the results."""
+        clone = self._clone()
+        clone.query.add_highlight(**kwargs)
+        return clone
+
+
+class ElasticsearchSearchQuery(OrigElasticsearchSearchQuery):
+    def add_highlight(self, **kwargs):
+        """Adds highlighting to the search results."""
+        self.highlight = kwargs or True
+
+
+class ElasticsearchSearchBackend(OrigElasticsearchSearchBackend):
+    """
+    Subclassed backend that lets user modify highlighting options
+    """
+    def build_search_kwargs(self, *args, **kwargs):
+        res = super(ElasticsearchSearchBackend, self).build_search_kwargs(*args, **kwargs)
+        index = haystack.connections[self.connection_alias].get_unified_index()
+        content_field = index.document_field
+        highlight = kwargs.get('highlight')
+        if highlight:
+            highlight_options = {
+                'fields': {
+                    content_field: {'store': 'yes'},
+                }
+            }
+            if isinstance(highlight, dict):
+                highlight_options.update(highlight)
+            res['highlight'] = highlight_options
+        return res
+
+
+class ElasticsearchSearchEngine(OrigElasticsearchSearchEngine):
+    backend = ElasticsearchSearchBackend
+    query = ElasticsearchSearchQuery

notesserver/settings/common.py

@@ -2,6 +2,8 @@ import os
 import json
 import sys
 
+from notesserver.settings.logger import get_logger_config
+
 DEBUG = False
 TEMPLATE_DEBUG = False
 DISABLE_TOKEN_CHECK = False
@@ -10,14 +12,22 @@ TIME_ZONE = 'UTC'
 
 # This value needs to be overriden in production.
 SECRET_KEY = '*^owi*4%!%9=#h@app!l^$jz8(c*q297^)4&4yn^#_m#fq=z#l'
+ALLOWED_HOSTS = ['localhost', '*.edx.org']
 
 # ID and Secret used for authenticating JWT Auth Tokens
 # should match those configured for `edx-notes` Client in EdX's /admin/oauth2/client/
 CLIENT_ID = 'edx-notes-id'
 CLIENT_SECRET = 'edx-notes-secret'
 
-ELASTICSEARCH_URL = 'http://127.0.0.1:9200'
-ELASTICSEARCH_INDEX = 'edx-notes'
+ES_DISABLED = False
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'notesserver.highlight.ElasticsearchSearchEngine',
+        'URL': 'http://127.0.0.1:9200/',
+        'INDEX_NAME': 'notes_index',
+    },
+}
+HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 
 # Number of rows to return by default in result.
 RESULTS_DEFAULT_SIZE = 25
@@ -33,73 +43,21 @@ MIDDLEWARE_CLASSES = (
     'django.middleware.csrf.CsrfViewMiddleware',
 )
 
-INSTALLED_APPS = (
+INSTALLED_APPS = [
     'django.contrib.staticfiles',
     'rest_framework',
     'rest_framework_swagger',
     'corsheaders',
+    'haystack',
     'notesapi',
     'notesapi.v1',
-)
+]
 
 STATIC_URL = '/static/'
 
 WSGI_APPLICATION = 'notesserver.wsgi.application'
 
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'handlers': {
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'stream': sys.stderr,
-            'formatter': 'standard',
-        },
-    },
-    'formatters': {
-        'standard': {
-            'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s',
-        },
-    },
-    'loggers': {
-        'django': {
-            'handlers': ['console'],
-            'level': 'ERROR',
-            'propagate': False,
-        },
-        'notesserver': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': True
-        },
-        'elasticsearch.trace': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': True
-        },
-        'elasticsearch': {
-            'handlers': ['console'],
-            'level': 'WARNING',
-            'propagate': True
-        },
-        'annotator.elasticsearch': {
-            'handlers': ['console'],
-            'level': 'WARNING',
-            'propagate': True
-        },
-        'urllib3': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': True
-        },
-        'notesapi.v1.permissions': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': True
-        },
-    },
-}
+LOGGING = get_logger_config()
 
 REST_FRAMEWORK = {
     'DEFAULT_PERMISSION_CLASSES': [

notesserver/settings/dev.py

-import annotator
-from annotator import es
 from .common import *
+from notesserver.settings.logger import get_logger_config
 
 DEBUG = True
 
-ELASTICSEARCH_INDEX = 'edx-notes-dev'
+ES_INDEXES = {'default': 'notes_index_dev'}
 
-###############################################################################
-# Override default annotator-store elasticsearch settings.
-###############################################################################
-es.host = ELASTICSEARCH_URL
-es.index = ELASTICSEARCH_INDEX
-annotator.elasticsearch.RESULTS_MAX_SIZE = RESULTS_MAX_SIZE
-###############################################################################
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': 'default.db',
+    }
+}
+
+LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG')

notesserver/settings/logger.py

+"""
+Logging configuration
+"""
+
+import os
+import platform
+import sys
+from logging.handlers import SysLogHandler
+
+
+def get_logger_config(log_dir='/var/tmp',
+                      logging_env="no_env",
+                      edx_filename="edx.log",
+                      dev_env=False,
+                      debug=False,
+                      local_loglevel='INFO',
+                      service_variant='edx-notes-api'):
+
+    """
+    Return the appropriate logging config dictionary. You should assign the
+    result of this to the LOGGING var in your settings.
+    If dev_env is set to true logging will not be done via local rsyslogd,
+    instead, application logs will be dropped in log_dir.
+    "edx_filename" is ignored unless dev_env is set to true since otherwise
+    logging is handled by rsyslogd.
+    """
+
+    # Revert to INFO if an invalid string is passed in
+    if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
+        local_loglevel = 'INFO'
+
+    hostname = platform.node().split(".")[0]
+    syslog_format = (
+        "[service_variant={service_variant}]"
+        "[%(name)s][env:{logging_env}] %(levelname)s "
+        "[{hostname}  %(process)d] [%(filename)s:%(lineno)d] "
+        "- %(message)s"
+    ).format(service_variant=service_variant, logging_env=logging_env, hostname=hostname)
+
+    if debug:
+        handlers = ['console']
+    else:
+        handlers = ['local']
+
+    logger_config = {
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'standard': {
+                'format': '%(asctime)s %(levelname)s %(process)d '
+                          '[%(name)s] %(filename)s:%(lineno)d - %(message)s',
+            },
+            'syslog_format': {'format': syslog_format},
+            'raw': {'format': '%(message)s'},
+        },
+        'handlers': {
+            'console': {
+                'level': 'DEBUG' if debug else 'INFO',
+                'class': 'logging.StreamHandler',
+                'formatter': 'standard',
+                'stream': sys.stdout,
+            },
+        },
+        'loggers': {
+            'django': {
+                'handlers': handlers,
+                'propagate': True,
+                'level': 'INFO'
+            },
+            "elasticsearch.trace": {
+                'handlers': handlers,
+                'level': 'DEBUG',
+                'propagate': False,
+            },
+            '': {
+                'handlers': handlers,
+                'level': 'DEBUG',
+                'propagate': False
+            },
+        }
+    }
+
+    if dev_env:
+        edx_file_loc = os.path.join(log_dir, edx_filename)
+        logger_config['handlers'].update({
+            'local': {
+                'class': 'logging.handlers.RotatingFileHandler',
+                'level': local_loglevel,
+                'formatter': 'standard',
+                'filename': edx_file_loc,
+                'maxBytes': 1024 * 1024 * 2,
+                'backupCount': 5,
+            },
+        })
+    else:
+        logger_config['handlers'].update({
+            'local': {
+                'level': local_loglevel,
+                'class': 'logging.handlers.SysLogHandler',
+                # Use a different address for Mac OS X
+                'address': '/var/run/syslog' if sys.platform == "darwin" else '/dev/log',
+                'formatter': 'syslog_format',
+                'facility': SysLogHandler.LOG_LOCAL0,
+            },
+        })
+
+    return logger_config

notesserver/settings/test.py

-from annotator import es
-import annotator
 from .common import *
 
 DATABASES = {
@@ -9,15 +7,37 @@ DATABASES = {
 }
 
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
-
+DISABLE_TOKEN_CHECK = False
 INSTALLED_APPS += ('django_nose',)
 
-ELASTICSEARCH_INDEX = 'edx-notes-test'
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'notesserver.highlight.ElasticsearchSearchEngine',
+        'URL': 'http://127.0.0.1:9200/',
+        'INDEX_NAME': 'notes_index_test',
+    },
+}
 
-###############################################################################
-# Override default annotator-store elasticsearch settings.
-###############################################################################
-es.host = ELASTICSEARCH_URL
-es.index = ELASTICSEARCH_INDEX
-annotator.elasticsearch.RESULTS_MAX_SIZE = RESULTS_MAX_SIZE
-###############################################################################
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'stream': sys.stderr,
+        },
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['console'],
+            'level': 'ERROR',
+            'propagate': False,
+        },
+        'elasticsearch.trace': {
+            'handlers': ['console'],
+            'level': 'ERROR',
+            'propagate': False
+        }
+    },
+}

notesserver/settings/test_es_disabled.py

+from .test import *
+
+ES_DISABLED = True
+HAYSTACK_CONNECTIONS = {}
+INSTALLED_APPS.remove('haystack')

notesserver/settings/yaml_config.py

 import yaml
-import annotator
-from annotator import es
 
 from .common import *  # pylint: disable=unused-wildcard-import, wildcard-import
+from path import path
+from django.core.exceptions import ImproperlyConfigured
 
 ###############################################################################
 # Explicitly declare here in case someone changes common.py.
@@ -12,17 +12,32 @@ TEMPLATE_DEBUG = False
 DISABLE_TOKEN_CHECK = False
 ###############################################################################
 
-CONFIG_ROOT = os.environ.get('EDXNOTES_CONFIG_ROOT')
+EDXNOTES_CONFIG_ROOT = os.environ.get('EDXNOTES_CONFIG_ROOT')
+
+if not EDXNOTES_CONFIG_ROOT:
+    raise ImproperlyConfigured("EDXNOTES_CONFIG_ROOT must be defined in the environment.")
+
+CONFIG_ROOT = path(EDXNOTES_CONFIG_ROOT)
 
 with open(CONFIG_ROOT / "edx-notes-api.yml") as yaml_file:
     config_from_yaml = yaml.load(yaml_file)
 
 vars().update(config_from_yaml)
 
-###############################################################################
-# Override default annotator-store elasticsearch settings.
-###############################################################################
-es.host = ELASTICSEARCH_URL
-es.index = ELASTICSEARCH_INDEX
-annotator.elasticsearch.RESULTS_MAX_SIZE = RESULTS_MAX_SIZE
-###############################################################################
+#
+# Support environment overrides for migrations
+DB_OVERRIDES = dict(
+    PASSWORD=environ.get('DB_MIGRATION_PASS', DATABASES['default']['PASSWORD']),
+    ENGINE=environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
+    USER=environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
+    NAME=environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
+    HOST=environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
+    PORT=environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
+)
+
+for override, value in DB_OVERRIDES.iteritems():
+    DATABASES['default'][override] = value
+
+if ES_DISABLED:
+    HAYSTACK_CONNECTIONS = {}
+    INSTALLED_APPS.remove('haystack')

notesserver/test_views.py

 import datetime
-import base64
+from unittest import skipIf
 from mock import patch, Mock
+from django.conf import settings
 from django.core.urlresolvers import reverse
 from rest_framework.test import APITestCase
 from elasticsearch.exceptions import TransportError
 
+
 class OperationalEndpointsTest(APITestCase):
     """
     Tests for operational endpoints.
@@ -17,16 +19,27 @@ class OperationalEndpointsTest(APITestCase):
         self.assertEquals(response.status_code, 200)
         self.assertEquals(response.data, {"OK": True})
 
-    @patch('annotator.elasticsearch.ElasticSearch.conn')
-    def test_heartbeat_failure(self, mocked_conn):
+    @skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
+    @patch('notesserver.views.get_es')
+    def test_heartbeat_failure_es(self, mocked_get_es):
         """
         Elasticsearch is not reachable.
         """
-        mocked_conn.ping.return_value = False
+        mocked_get_es.return_value.ping.return_value = False
         response = self.client.get(reverse('heartbeat'))
         self.assertEquals(response.status_code, 500)
         self.assertEquals(response.data, {"OK": False, "check": "es"})
 
+    @patch("django.db.backends.utils.CursorWrapper")
+    def test_heartbeat_failure_db(self, mocked_cursor_wrapper):
+        """
+        Database is not reachable.
+        """
+        mocked_cursor_wrapper.side_effect = Exception
+        response = self.client.get(reverse('heartbeat'))
+        self.assertEquals(response.status_code, 500)
+        self.assertEquals(response.data, {"OK": False, "check": "db"})
+
     def test_root(self):
         """
         Test root endpoint.
@@ -41,41 +54,67 @@ class OperationalEndpointsTest(APITestCase):
             }
         )
 
-class OperationalAuthEndpointsTest(APITestCase):
-    """
-    Tests for operational authenticated endpoints.
-    """
-
     def test_selftest_status(self):
         """
-        Test status on authorization success.
+        Test status success.
         """
         response = self.client.get(reverse('selftest'))
         self.assertEquals(response.status_code, 200)
 
+    @skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
     @patch('notesserver.views.datetime', datetime=Mock(wraps=datetime.datetime))
-    @patch('annotator.elasticsearch.ElasticSearch.conn')
-    def test_selftest_data(self, mocked_conn, mocked_datetime):
+    @patch('notesserver.views.get_es')
+    def test_selftest_data(self, mocked_get_es, mocked_datetime):
         """
         Test returned data on success.
         """
         mocked_datetime.datetime.now.return_value = datetime.datetime(2014, 12, 11)
-        mocked_conn.info.return_value = {}
+        mocked_get_es.return_value.info.return_value = {}
         response = self.client.get(reverse('selftest'))
         self.assertEquals(response.status_code, 200)
         self.assertEquals(
             response.data,
             {
                 "es": {},
+                "db": "OK",
+                "time_elapsed": 0.0
+            }
+        )
+
+    @patch('django.conf.settings.ES_DISABLED', True)
+    @patch('notesserver.views.datetime', datetime=Mock(wraps=datetime.datetime))
+    def test_selftest_data_es_disabled(self, mocked_datetime):
+        """
+        Test returned data on success.
+        """
+        mocked_datetime.datetime.now.return_value = datetime.datetime(2014, 12, 11)
+        response = self.client.get(reverse('selftest'))
+        self.assertEquals(response.status_code, 200)
+        self.assertEquals(
+            response.data,
+            {
+                "db": "OK",
                 "time_elapsed": 0.0
             }
         )
 
-    @patch('annotator.elasticsearch.ElasticSearch.conn')
-    def test_selftest_failure(self, mocked_conn):
+    @skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
+    @patch('notesserver.views.get_es')
+    def test_selftest_failure_es(self, mocked_get_es):
         """
         Elasticsearch is not reachable on selftest.
         """
-        mocked_conn.info.side_effect = TransportError()
+        mocked_get_es.return_value.info.side_effect = TransportError()
+        response = self.client.get(reverse('selftest'))
+        self.assertEquals(response.status_code, 500)
+        self.assertIn('es_error', response.data)
+
+    @patch("django.db.backends.utils.CursorWrapper")
+    def test_selftest_failure_db(self, mocked_cursor_wrapper):
+        """
+        Database is not reachable on selftest.
+        """
+        mocked_cursor_wrapper.side_effect = Exception
         response = self.client.get(reverse('selftest'))
         self.assertEquals(response.status_code, 500)
+        self.assertIn('db_error', response.data)

notesserver/views.py

 import traceback
 import datetime
+
+from django.db import connection
+from django.conf import settings
 from rest_framework import status
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
-from rest_framework.views import APIView
-from rest_framework.decorators import api_view, permission_classes, authentication_classes
+from rest_framework.decorators import api_view, permission_classes
 
 from elasticsearch.exceptions import TransportError
-from annotator import es
+
+if not settings.ES_DISABLED:
+    from haystack import connections
+
+    def get_es():
+        return connections['default'].get_backend().conn
 
 
 @api_view(['GET'])
@@ -26,13 +33,18 @@ def root(request):  # pylint: disable=unused-argument
 @permission_classes([AllowAny])
 def heartbeat(request):  # pylint: disable=unused-argument
     """
-    ElasticSearch is reachable and ready to handle requests.
+    ElasticSearch and database are reachable and ready to handle requests.
     """
-    if es.conn.ping():
-        return Response({"OK": True})
-    else:
+    try:
+        db_status()
+    except Exception:
+        return Response({"OK": False, "check": "db"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    if not settings.ES_DISABLED and not get_es().ping():
         return Response({"OK": False, "check": "es"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
+    return Response({"OK": True})
+
 
 @api_view(['GET'])
 @permission_classes([AllowAny])
@@ -41,18 +53,43 @@ def selftest(request):  # pylint: disable=unused-argument
     Manual test endpoint.
     """
     start = datetime.datetime.now()
+
+    if not settings.ES_DISABLED:
+        try:
+            es_status = get_es().info()
+        except TransportError:
+            return Response(
+                {"es_error": traceback.format_exc()},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
+
     try:
-        es_status = es.conn.info()
-    except TransportError:
+        db_status()
+        database = "OK"
+    except Exception:
         return Response(
-            {"es_error": traceback.format_exc()},
+            {"db_error": traceback.format_exc()},
             status=status.HTTP_500_INTERNAL_SERVER_ERROR
         )
 
     end = datetime.datetime.now()
     delta = end - start
 
-    return Response({
-        "es": es_status,
+    response = {
+        "db": database,
         "time_elapsed": int(delta.total_seconds() * 1000)  # In milliseconds.
-    })
+    }
+
+    if not settings.ES_DISABLED:
+        response['es'] = es_status
+
+    return Response(response)
+
+
+def db_status():
+    """
+    Return database status.
+    """
+    with connection.cursor() as cursor:
+        cursor.execute("SELECT 1")
+        cursor.fetchone()

requirements/base.txt

 Django==1.7.1
 requests==2.4.3
-djangorestframework==2.4.4
+djangorestframework==3.0.2
 django-rest-swagger==0.2.0
-elasticsearch==1.2.0
-annotator==0.12.0
+django-haystack==2.3.1
+elasticsearch==0.4.5  # only 0.4 works for ES 0.90
 django-cors-headers==0.13
 PyJWT==0.3.0
+MySQL-python==1.2.5  # GPL License
+gunicorn==19.1.1     # MIT
+path.py==3.0.1
+python-dateutil==2.4.0

scripts/travis/install.sh

+#!/bin/bash
+
+# Installs Elasticsearch
+#
+# Requires ESVER environment variable to be set.
+
+set -e
+
+if [[ $ESVER == "-" ]];
+then
+    exit 0
+fi
+
+echo "Installing ElasticSearch $ESVER" >&2
+wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ESVER.tar.gz
+tar xzvf elasticsearch-$ESVER.tar.gz

scripts/travis/setup.sh

+#!/bin/bash
+
+# Runs Elasticsearch
+#
+# Requires ESVER environment variable to be set.
+# cwd is the git repository root.
+
+set -e
+
+if [[ $ESVER == "-" ]];
+then
+    exit 0
+fi
+
+echo "Starting ElasticSearch $ESVER" >&2
+pushd elasticsearch-$ESVER
+    # Elasticsearch 0.90 daemonizes automatically, but 1.0+ requires
+    # a -d argument.
+
+    if [[ $ESVER == 0* ]];
+    then
+        ./bin/elasticsearch
+    else
+        echo "launching with -d option." >&2
+        ./bin/elasticsearch -d
+        pip install elasticsearch==1.3.0
+    fi
+popd