Merge pull request #41 from edx/ammar/tnl-4119-fix-unicode-search

fix search when text and/or tags contain unicode data

Makefile

@@ -13,7 +13,7 @@ test: clean
 	./manage.py test --settings=$(test_settings) --with-coverage --with-ignore-docstrings \
 		--exclude-dir=notesserver/settings --cover-inclusive --cover-branches \
 		--cover-html --cover-html-dir=build/coverage/html/ \
-		--cover-xml --cover-xml-file=build/coverage/coverage.xml \
+		--cover-xml --cover-xml-file=build/coverage/coverage.xml --verbosity=2 \
 		$(foreach package,$(PACKAGES),--cover-package=$(package)) \
 		$(PACKAGES)
 

notesapi/v1/models.py

@@ -35,6 +35,6 @@ class Note(models.Model):
 
         note_dict['ranges'] = json.dumps(ranges)
         note_dict['user_id'] = note_dict.pop('user', None)
-        note_dict['tags'] = json.dumps(note_dict.get('tags', list()))
+        note_dict['tags'] = json.dumps(note_dict.get('tags', list()), ensure_ascii=False)
 
         return cls(**note_dict)

notesapi/v1/tests/test_views.py

@@ -959,6 +959,26 @@ class AnnotationSearchViewTests(BaseAnnotationViewTests):
             start=start
         )
 
+    @ddt.unpack
+    @ddt.data(
+        {"text": u"Ammar محمد عمار Muhammad", "search": u"محمد عمار", "tags": [u"عمار", u"Muhammad", u"محمد"]},
+        {"text": u"Ammar محمد عمار Muhammad", "search": u"محمد", "tags": [u"محمد", u"Muhammad"]},
+        {"text": u"Ammar محمد عمار Muhammad", "search": u"عمار", "tags": [ u"ammar", u"عمار"]},
+        {"text": u"Muhammad Ammar", "search": u"عمار", "tags": [ u"ammar", u"عمار"]},
+        {"text": u"محمد عمار", "search": u"Muhammad", "tags": [ u"Muhammad", u"عمار"]}
+    )
+    @unittest.skipIf(settings.ES_DISABLED, "MySQL cannot do highlighting")
+    def test_search_unicode_text_and_tags(self, text, search, tags):
+        """
+        Verify that search works as expected with unicode and non-unicode text and tags.
+        """
+        self._create_annotation(text=text, tags=tags)
+
+        response = self._get_search_results(text=search)
+        self.assertEqual(response["total"], 1)
+        self.assertEqual(response["rows"][0]["text"], text)
+        self.assertEqual(response["rows"][0]["tags"], tags)
+
 
 @patch('django.conf.settings.DISABLE_TOKEN_CHECK', True)
 class AllowAllAnnotationViewTests(BaseAnnotationViewTests):