haystack 2

notesapi/management/commands/create_index.py

-from optparse import make_option
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from elasticutils.contrib.django import get_es
-from notesapi.v1.models import NoteMappingType
-
-
-class Command(BaseCommand):
-    """
-    Indexing and mapping commands.
-    """
-    help = 'Creates index and the mapping.'
-    option_list = BaseCommand.option_list + (
-        make_option(
-            '--drop',
-            action='store_true',
-            dest='drop',
-            default=False,
-            help='Recreate index'
-        ),
-    )
-
-    def handle(self, *args, **options):
-        if options['drop']:
-            # drop existing
-            get_es().indices.delete(index=settings.ES_INDEXES['default'])
-
-        get_es().indices.create(
-            index=settings.ES_INDEXES['default'],
-            body={
-                'mappings': {
-                    NoteMappingType.get_mapping_type_name(): NoteMappingType.get_mapping()
-                }
-            },
-        )

notesapi/v1/models.py

@@ -4,8 +4,6 @@ from django.core.exceptions import ValidationError
 from django.conf import settings
 from django.db.models import signals
 from django.dispatch import receiver
-from elasticutils.contrib.django import Indexable, MappingType
-
 
 
 class Note(models.Model):
@@ -60,79 +58,3 @@ class Note(models.Model):
             'created': created,
             'updated': updated,
         }
-
-
-@receiver(signals.post_save, sender=Note)
-def update_in_index(sender, instance, **kwargs):
-    if settings.ES_DISABLED:
-        return
-    NoteMappingType.index(instance.as_dict(), id_=instance.id, overwrite_existing=True)
-
-
-@receiver(signals.post_delete, sender=Note)
-def delete_in_index(sender, instance, **kwargs):
-    if settings.ES_DISABLED:
-        return
-    NoteMappingType.unindex(id_=instance.id)
-
-
-class NoteMappingType(MappingType, Indexable):
-    """
-    Mapping type for Note.
-    """
-
-    @classmethod
-    def get_model(cls):
-        return Note
-
-    @classmethod
-    def get_mapping(cls):
-        """
-        Returns an Elasticsearch mapping for Note MappingType
-        """
-        charfield = {'type': 'string', 'index': 'not_analyzed', 'store': True}
-        return {
-            'properties': {
-                'id': charfield,
-                'user': charfield,
-                'course_id': charfield,
-                'usage_id': charfield,
-                'text': {'type': 'string', 'analyzer': 'snowball', 'store': True},
-                'quote': {'type': 'string', 'analyzer': 'snowball', 'store': True},
-                'created': {'type': 'date', 'store': True},
-                'updated': {'type': 'date', 'store': True},
-            }
-        }
-
-    @classmethod
-    def extract_document(cls, obj_id, obj=None):
-        """
-        Converts this instance into an Elasticsearch document.
-        """
-        if obj is None:
-            obj = cls.get_model().objects.get(pk=obj_id)
-
-        return obj.as_dict()
-
-    @staticmethod
-    def process_result(data):
-        """
-        Unlistifies the result and replaces `text` with highlihted one
-
-        Unlistification: ElasticUtils returns data as [{field:value,..}..] which is not what needed.
-        this function reverses the effect to get the original value.
-        Also filed https://github.com/mozilla/elasticutils/pull/285 to make it unnecessary.
-        """
-        for i, item in enumerate(data):
-            if isinstance(item, dict):
-                for k, v in item.items():
-                    if k != 'ranges' and isinstance(v, list) and len(v) > 0:
-                        data[i][k] = v[0]
-
-                    # Substitute the value of text field by highlighted result.
-                    if len(item.es_meta.highlight) and k == 'text':
-                        data[i][k] = item.es_meta.highlight['text'][0]
-
-        return data
-
-note_searcher = NoteMappingType.search()

notesapi/v1/tests/test_models.py

 from unittest import TestCase
-from notesapi.v1.models import Note, NoteMappingType
+from notesapi.v1.models import Note
 from django.core.exceptions import ValidationError
 
 
@@ -46,12 +46,4 @@ class NoteTest(TestCase):
 
             with self.assertRaises(ValidationError):
                 note = Note.create(payload)
-                note.full_clean()
-
+                note.full_clean()
\ No newline at end of file
-    def test_extract_document(self):
-        note = Note.create(self.note_dict.copy())
-        note.save()
-        self.assertEqual(NoteMappingType.extract_document(note.id), note.as_dict())
-
-    def test_get_model(self):
-        self.assertIsInstance(NoteMappingType.get_model()(), Note)

notesapi/v1/tests/test_views.py

@@ -4,6 +4,7 @@ from calendar import timegm
 from datetime import datetime, timedelta
 from mock import patch
 
+from django.core.management import call_command
 from django.core.urlresolvers import reverse
 from django.conf import settings
 from django.http import QueryDict
@@ -11,10 +12,9 @@ from django.http import QueryDict
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from elasticutils.contrib.django import get_es
 from .helpers import get_id_token
-from notesapi.v1.models import NoteMappingType, note_searcher, Note
-from notesapi.management.commands.create_index import Command as CreateIndexCommand
+from notesapi.v1.models import Note
+
 
 TEST_USER = "test_user_id"
 
@@ -24,6 +24,9 @@ class BaseAnnotationViewTests(APITestCase):
     Abstract class for testing annotation views.
     """
     def setUp(self):
+        call_command('clear_index', interactive=False)
+        call_command('update_index')
+
         token = get_id_token(TEST_USER)
         self.client.credentials(HTTP_X_ANNOTATOR_AUTH_TOKEN=token)
         self.headers = {"user": TEST_USER}
@@ -44,37 +47,6 @@ class BaseAnnotationViewTests(APITestCase):
             ],
         }
 
-    def tearDown(self):
-        for note_id in note_searcher.all().values_list('id'):
-            get_es().delete(
-                index=settings.ES_INDEXES['default'],
-                doc_type=NoteMappingType.get_mapping_type_name(),
-                id=note_id[0][0]
-            )
-        get_es().indices.refresh()
-
-    @classmethod
-    def setUpClass(cls):
-        get_es().indices.delete(index=settings.ES_INDEXES['default'], ignore=404)
-        get_es().indices.create(
-            index=settings.ES_INDEXES['default'],
-            body={
-                'mappings': {
-                    NoteMappingType.get_mapping_type_name(): NoteMappingType.get_mapping()
-                }
-            },
-        )
-        get_es().indices.refresh()
-        get_es().cluster.health(wait_for_status='yellow')
-
-    @classmethod
-    def tearDownClass(cls):
-        """
-        deletes the test index
-        """
-        get_es().indices.delete(index=settings.ES_INDEXES['default'])
-        get_es().indices.refresh()
-
     def _create_annotation(self, **kwargs):
         """
         Create annotation
@@ -84,14 +56,14 @@ class BaseAnnotationViewTests(APITestCase):
         url = reverse('api:v1:annotations')
         response = self.client.post(url, opts, format='json')
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-        get_es().indices.refresh()
+        call_command('update_index')
         return response.data.copy()
 
     def _get_annotation(self, annotation_id):
         """
         Fetch annotation directly from elasticsearch.
         """
-        get_es().indices.refresh()
+        call_command('update_index')
         url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': annotation_id})
         response = self.client.get(url, self.headers)
         self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -139,30 +111,30 @@ class AnnotationViewTests(BaseAnnotationViewTests):
 
         self.assertEqual(response.data['user'], TEST_USER)
 
-    @patch('django.conf.settings.ES_DISABLED', True)
-    def test_create_es_disabled(self):
-        """
-        Ensure we can create note in database when elasticsearch is disabled.
-        """
-        url = reverse('api:v1:annotations')
-        response = self.client.post(url, self.payload, format='json')
-        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
-        Note.objects.get(id=response.data['id'])
-        self.assertEqual(note_searcher.filter(id=response.data['id']).count(), 0)
-
-    def test_delete_es_disabled(self):
-        """
-        Ensure we can delete note in database when elasticsearch is disabled.
-        """
-        url = reverse('api:v1:annotations')
-        response = self.client.post(url, self.payload, format='json')
-        get_es().indices.refresh()
-        self.assertEqual(note_searcher.filter(id=response.data['id']).count(), 1)
-
-        with patch('django.conf.settings.ES_DISABLED', True):
-            Note.objects.get(id=response.data['id']).delete()
-
-        self.assertEqual(note_searcher.filter(id=response.data['id']).count(), 1)
+    # @patch('django.conf.settings.ES_DISABLED', True)
+    # def test_create_es_disabled(self):
+    #     """
+    #     Ensure we can create note in database when elasticsearch is disabled.
+    #     """
+    #     url = reverse('api:v1:annotations')
+    #     response = self.client.post(url, self.payload, format='json')
+    #     self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+    #     Note.objects.get(id=response.data['id'])
+    #     self.assertEqual(note_searcher.filter(id=response.data['id']).count(), 0)
+
+    # def test_delete_es_disabled(self):
+    #     """
+    #     Ensure we can delete note in database when elasticsearch is disabled.
+    #     """
+    #     url = reverse('api:v1:annotations')
+    #     response = self.client.post(url, self.payload, format='json')
+    #     call_command('update_index')
+    #     self.assertEqual(note_searcher.filter(id=response.data['id']).count(), 1)
+
+    #     with patch('django.conf.settings.ES_DISABLED', True):
+    #         Note.objects.get(id=response.data['id']).delete()
+
+    #     self.assertEqual(note_searcher.filter(id=response.data['id']).count(), 1)
 
     def test_create_ignore_created(self):
         """
@@ -269,7 +241,7 @@ class AnnotationViewTests(BaseAnnotationViewTests):
         payload.update(self.headers)
         url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': data['id']})
         response = self.client.put(url, payload, format='json')
-        get_es().indices.refresh()
+        call_command('update_index')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
         annotation = self._get_annotation(data['id'])
@@ -343,7 +315,7 @@ class AnnotationViewTests(BaseAnnotationViewTests):
         response = self.client.delete(url, self.headers)
         self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT, "response should be 204 NO CONTENT")
 
-        get_es().indices.refresh()
+        call_command('update_index')
         url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': note['id']})
         response = self.client.get(url, self.headers)
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
@@ -382,10 +354,11 @@ class AnnotationViewTests(BaseAnnotationViewTests):
         results = self._get_search_results()
         self.assertEqual(results['total'], 2)
 
+        # FIXME class and tag
         results = self._get_search_results(text="first", highlight=True, highlight_class='class', highlight_tag='tag')
         self.assertEqual(results['total'], 1)
         self.assertEqual(len(results['rows']), 1)
-        self.assertEqual(results['rows'][0]['text'], '<span>First</span> note')
+        self.assertEqual(results['rows'][0]['text'], '<em>First</em> note')
 
     def test_search_ordering(self):
         """

notesapi/v1/views.py

 import logging
+import json
 
 from django.core.urlresolvers import reverse
 from django.core.exceptions import ValidationError
@@ -7,7 +8,8 @@ from rest_framework import status
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from notesapi.v1.models import Note, NoteMappingType, note_searcher
+from notesapi.v1.models import Note
+from haystack.query import SearchQuerySet
 
 log = logging.getLogger(__name__)
 
@@ -22,27 +24,20 @@ class AnnotationSearchView(APIView):
         Search annotations.
         """
         params = self.request.QUERY_PARAMS.dict()
-        for field in ('text', 'quote'):
-            if field in params:
-                params[field + "__match"] = params[field]
-                del params[field]
+        query = SearchQuerySet().models(Note).filter(
+            **{f:v for (f,v) in params.items() if f in ('user', 'course_id', 'usage_id', 'text')}
+        ).order_by('-updated')
 
         if params.get('highlight'):
-
-            # Currently we do not use highlight_class and highlight_tag in service.
-            for param in ['highlight', 'highlight_class', 'highlight_tag']:
-                params.pop(param, None)
-
-            results = NoteMappingType.process_result(
-                list(
-                    note_searcher.query(**params).order_by("-created").values_dict("_source")
-                    .highlight("text", pre_tags=['<span>'], post_tags=['</span>'])
-                )
-            )
-        else:
-            results = NoteMappingType.process_result(
-                list(note_searcher.query(**params).order_by("-created").values_dict("_source"))
-            )
+            query = query.highlight()
+
+        results = []
+        for item in query:
+            note_dict = item.get_stored_fields()
+            note_dict['range'] = json.loads(item.ranges)
+            if params.get('highlight'):
+                note_dict['text'] = item.highlighted[0]
+            results.append(note_dict)
 
         return Response({'total': len(results), 'rows': results})
 

notesserver/settings/common.py

@@ -18,9 +18,14 @@ SECRET_KEY = '*^owi*4%!%9=#h@app!l^$jz8(c*q297^)4&4yn^#_m#fq=z#l'
 CLIENT_ID = 'edx-notes-id'
 CLIENT_SECRET = 'edx-notes-secret'
 
-ES_URLS = ['http://localhost:9200']
-ES_INDEXES = {'default': 'notes_index'}
-ES_DISABLED = False
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': 'http://127.0.0.1:9200/',
+        'INDEX_NAME': 'notes_index',
+    },
+}
+HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 
 # Number of rows to return by default in result.
 RESULTS_DEFAULT_SIZE = 25
@@ -41,6 +46,7 @@ INSTALLED_APPS = (
     'rest_framework',
     'rest_framework_swagger',
     'corsheaders',
+    'haystack',
     'notesapi',
     'notesapi.v1',
 )

notesserver/settings/test.py

@@ -10,7 +10,13 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 DISABLE_TOKEN_CHECK = False
 INSTALLED_APPS += ('django_nose',)
 
-ES_INDEXES = {'default': 'notes_index_test'}
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
+        'URL': 'http://127.0.0.1:9200/',
+        'INDEX_NAME': 'notes_index_test',
+    },
+}
 
 LOGGING = {
     'version': 1,

notesserver/views.py

@@ -8,7 +8,11 @@ from rest_framework.response import Response
 from rest_framework.decorators import api_view, permission_classes
 
 from elasticsearch.exceptions import TransportError
-from elasticutils import get_es
+from haystack import connections
+
+
+def get_es():
+    return connections['default'].get_backend().conn
 
 
 @api_view(['GET'])

requirements/base.txt

@@ -2,9 +2,9 @@ Django==1.7.1
 requests==2.4.3
 djangorestframework==3.0.2
 django-rest-swagger==0.2.0
-elasticutils==0.10.2
+django-haystack==2.3.1
 elasticsearch==1.2.0
 django-cors-headers==0.13
 PyJWT==0.3.0
-MySQL-python==1.2.5		# GPL License
-gunicorn==19.1.1                # MIT
+MySQL-python==1.2.5	 # GPL License
+gunicorn==19.1.1     # MIT