Merge pull request #943 from trwired/utf8safedocs

Make browsable API views play nice with utf-8

rest_framework/tests/test_description.py

@@ -3,8 +3,10 @@
 from __future__ import unicode_literals
 from django.test import TestCase
 from rest_framework.views import APIView
-from rest_framework.compat import apply_markdown
+from rest_framework.compat import apply_markdown, smart_text
 from rest_framework.utils.formatting import get_view_name, get_view_description
+from rest_framework.tests.views import (
+    ViewWithNonASCIICharactersInDocstring, UTF8_TEST_DOCSTRING)
 
 # We check that docstrings get nicely un-indented.
 DESCRIPTION = """an example docstring
@@ -83,11 +85,10 @@ class TestViewNamesAndDescriptions(TestCase):
         Unicode in docstrings should be respected.
         """
 
-        class MockView(APIView):
-            """Проверка"""
-            pass
-
-        self.assertEqual(get_view_description(MockView), "Проверка")
+        self.assertEqual(
+            get_view_description(ViewWithNonASCIICharactersInDocstring),
+            smart_text(UTF8_TEST_DOCSTRING)
+        )
 
     def test_view_description_can_be_empty(self):
         """

rest_framework/tests/views.py

+# -*- coding: utf-8 -*-
+
+from rest_framework.views import APIView
+
+
+# test strings snatched from http://www.columbia.edu/~fdc/utf8/,
+# http://winrus.com/utf8-jap.htm and memory
+UTF8_TEST_DOCSTRING = (
+    'zażółć gęślą jaźń'
+    'Sîne klâwen durh die wolken sint geslagen'
+    'Τη γλώσσα μου έδωσαν ελληνική'
+    'யாமறிந்த மொழிகளிலே தமிழ்மொழி'
+    'На берегу пустынных волн'
+    'てすと'
+    'ｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃ'
+)
+
+
+# Apparently there is an issue where docstrings of imported view classes
+# do not retain their encoding information even if a module has a proper
+# encoding declaration at the top of its source file. Therefore for tests
+# to catch unicode related errors, a mock view has to be declared in a separate
+# module.
+class ViewWithNonASCIICharactersInDocstring(APIView):
+    __doc__ = UTF8_TEST_DOCSTRING

rest_framework/utils/formatting.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
-from rest_framework.compat import apply_markdown
+from rest_framework.compat import apply_markdown, smart_text
 import re
 
 
@@ -63,7 +63,7 @@ def get_view_description(cls, html=False):
     Return a description for an `APIView` class or `@api_view` function.
     """
     description = cls.__doc__ or ''
-    description = _remove_leading_indent(description)
+    description = _remove_leading_indent(smart_text(description))
     if html:
         return markup_description(description)
     return description