Merge pull request #17 from edx/oleg/update-index

Allow reindexing only recent records.

README.rst

@@ -55,6 +55,13 @@ Running Tests
 Run ``make validate`` install the requirements, run the tests, and run
 lint.
 
+How To Resync The Index
+-----------------------
+edX Notes Store uses `Haystack <http://haystacksearch.org/>`_ which comes with several management commands.
+
+Please read more about ``update_index`` management command
+`here <http://django-haystack.readthedocs.org/en/latest/management_commands.html#update-index>`_.
+
 License
 -------
 

notesapi/v1/search_indexes.py

@@ -16,5 +16,17 @@ class NoteIndex(indexes.SearchIndex, indexes.Indexable):
         return Note
 
     def index_queryset(self, using=None):
-        """Used when the entire index for model is updated."""
+        """
+        Used when the entire index for model is updated.
+        """
         return self.get_model().objects.all()
+
+    def get_updated_field(self):
+        """
+        Get the field name that represents the updated date for the Note model.
+
+        This is used by the reindex command to filter out results from the QuerySet, enabling to reindex only
+        recent records. This method returns a string of the Note's field name that contains the date that the model
+        was updated.
+        """
+        return 'updated'

notesapi/v1/tests/test_update_index.py

+import factory
+import datetime
+from django.conf import settings
+from unittest import skipIf
+from django.core.management import call_command
+from django.db.models import signals
+from django.core.urlresolvers import reverse
+from .test_views import BaseAnnotationViewTests
+
+
+@skipIf(settings.ES_DISABLED, "Do not test if Elasticsearch service is disabled.")
+class UpdateIndexTest(BaseAnnotationViewTests):
+    """
+    Tests for update index command.
+    """
+
+    @factory.django.mute_signals(signals.post_save)
+    def test_create(self):
+        """
+        Ensure we can update index with created notes within specific
+        period of time.
+        """
+        start = datetime.datetime.now()
+        self._create_annotation(text=u'First note')
+
+        second_start = datetime.datetime.now()
+        self._create_annotation(text=u'Second note')
+        second_end = datetime.datetime.now()
+
+        self._create_annotation(text=u'Third note')
+        end = datetime.datetime.now()
+
+        results = self._get_search_results(text='note')
+        self.assertEqual(results, {'rows': [], 'total': 0})
+
+        # When second note was created.
+        call_command('update_index', start_date=second_start.isoformat(), end_date=second_end.isoformat(), verbosity=0)
+        results = self._get_search_results(text='note')
+        self.assertEqual(results['total'], 1)
+        self.assertEqual(results['rows'][0]['text'], 'Second note')
+
+        # All notes.
+        call_command('update_index', start_date=start.isoformat(), end_date=end.isoformat(), verbosity=0)
+        results = self._get_search_results(text='note')
+        self.assertEqual(results['total'], 3)
+
+    @factory.django.mute_signals(signals.post_delete)
+    def test_delete(self):
+        """
+        Ensure we can update index with deleted notes.
+        """
+
+        first_note = self._create_annotation(text=u'First note')
+        second_note = self._create_annotation(text=u'Second note')
+        self._create_annotation(text=u'Third note')
+
+        results = self._get_search_results(text='note')
+        self.assertEqual(results['total'], 3)
+
+        # Delete first note.
+        url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': first_note['id']})
+        response = self.client.delete(url, self.headers)
+
+        # Delete second note.
+        second_start = datetime.datetime.now()
+        url = reverse('api:v1:annotations_detail', kwargs={'annotation_id': second_note['id']})
+        response = self.client.delete(url, self.headers)
+        second_end = datetime.datetime.now() + datetime.timedelta(minutes=10)
+
+        # Try to update when only second note was deleted.
+        call_command(
+            'update_index', remove=True, start_date=second_start.isoformat(),
+            end_date=second_end.isoformat(), verbosity=0
+        )
+        results = self._get_search_results(text='note')
+
+        # When remove flag is provided, start and end flags do not play any role.
+        self.assertEqual(results['total'], 1)
+        self.assertEqual(results['rows'][0]['text'], 'Third note')

notesapi/v1/tests/test_views.py

@@ -71,6 +71,16 @@ class BaseAnnotationViewTests(APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         return response.data
 
+    def _get_search_results(self, **kwargs):
+        """
+        Helper for search method. All keyword parameters are passed in GET
+        """
+        q = QueryDict("user=" + TEST_USER, mutable=True)
+        q.update(kwargs)
+        url = reverse('api:v1:annotations_search') + '?{}'.format(q.urlencode())
+        result = self.client.get(url)
+        return result.data
+
 
 class AnnotationListViewTests(BaseAnnotationViewTests):
     """
@@ -363,15 +373,6 @@ class AnnotationSearchViewTests(BaseAnnotationViewTests):
     """
     Test annotation searching by user, course_id, usage_id and text
     """
-    def _get_search_results(self, **kwargs):
-        """
-        Helper for search method. All keyword parameters are passed in GET
-        """
-        q = QueryDict("user=" + TEST_USER, mutable=True)
-        q.update(kwargs)
-        url = reverse('api:v1:annotations_search') + '?{}'.format(q.urlencode())
-        result = self.client.get(url)
-        return result.data
 
     def test_search(self):
         """

notesserver/settings/test_es_disabled.py

 from .test import *
 
 ES_DISABLED = True
-HAYSTACK_CONNECTIONS = {}
+HAYSTACK_CONNECTIONS = {'default':{}}
 INSTALLED_APPS.remove('haystack')

requirements/test.txt

@@ -8,3 +8,4 @@ nose-ignore-docstring==0.2
 pep8==1.5.7
 pylint==1.4.0
 diff-cover==0.7.2
+factory_boy==2.4.1