Port django.middleware.locale.LocaleMiddleware from Django 1.8

cms/envs/common.py

@@ -308,7 +308,9 @@ MIDDLEWARE_CLASSES = (
     'embargo.middleware.EmbargoMiddleware',
 
     # Detects user-requested locale from 'accept-language' header in http request
-    'django.middleware.locale.LocaleMiddleware',
+    # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671]
+    # 'django.middleware.locale.LocaleMiddleware',
+    'django_locale.middleware.LocaleMiddleware',
 
     'django.middleware.transaction.TransactionMiddleware',
     # needs to run after locale middleware (or anything that modifies the request context)

common/djangoapps/django_locale/__init__.py

+"""
+TODO: This module is imported from the stable Django 1.8 branch, as a
+copy of https://github.com/django/django/blob/stable/1.8.x/django/middleware/locale.py.
+
+Remove this file and re-import this middleware from Django once the
+codebase is upgraded with a modern version of Django. [PLAT-671]
+"""

common/djangoapps/django_locale/middleware.py

+# TODO: This file is imported from the stable Django 1.8 branch. Remove this file
+# and re-import this middleware from Django once the codebase is upgraded. [PLAT-671]
+# pylint: disable=invalid-name, missing-docstring
+"This is the locale selecting middleware that will look at accept headers"
+
+from django.conf import settings
+from django.core.urlresolvers import (
+    LocaleRegexURLResolver, get_resolver, get_script_prefix, is_valid_path,
+)
+from django.http import HttpResponseRedirect
+from django.utils import translation
+from django.utils.cache import patch_vary_headers
+# Override the Django 1.4 implementation with the 1.8 implementation
+from django_locale.trans_real import get_language_from_request
+
+
+class LocaleMiddleware(object):
+    """
+    This is a very simple middleware that parses a request
+    and decides what translation object to install in the current
+    thread context. This allows pages to be dynamically
+    translated to the language the user desires (if the language
+    is available, of course).
+    """
+    response_redirect_class = HttpResponseRedirect
+
+    def __init__(self):
+        self._is_language_prefix_patterns_used = False
+        for url_pattern in get_resolver(None).url_patterns:
+            if isinstance(url_pattern, LocaleRegexURLResolver):
+                self._is_language_prefix_patterns_used = True
+                break
+
+    def process_request(self, request):
+        check_path = self.is_language_prefix_patterns_used()
+        # This call is broken in Django 1.4:
+        # https://github.com/django/django/blob/stable/1.4.x/django/utils/translation/trans_real.py#L399
+        # (we override parse_accept_lang_header to a fixed version in dark_lang.middleware)
+        language = get_language_from_request(
+            request, check_path=check_path)
+        translation.activate(language)
+        request.LANGUAGE_CODE = translation.get_language()
+
+    def process_response(self, request, response):
+        language = translation.get_language()
+        language_from_path = translation.get_language_from_path(request.path_info)
+        if (response.status_code == 404 and not language_from_path
+                and self.is_language_prefix_patterns_used()):
+            urlconf = getattr(request, 'urlconf', None)
+            language_path = '/%s%s' % (language, request.path_info)
+            path_valid = is_valid_path(language_path, urlconf)
+            if (not path_valid and settings.APPEND_SLASH
+                    and not language_path.endswith('/')):
+                path_valid = is_valid_path("%s/" % language_path, urlconf)
+
+            if path_valid:
+                script_prefix = get_script_prefix()
+                language_url = "%s://%s%s" % (
+                    request.scheme,
+                    request.get_host(),
+                    # insert language after the script prefix and before the
+                    # rest of the URL
+                    request.get_full_path().replace(
+                        script_prefix,
+                        '%s%s/' % (script_prefix, language),
+                        1
+                    )
+                )
+                return self.response_redirect_class(language_url)
+
+        if not (self.is_language_prefix_patterns_used()
+                and language_from_path):
+            patch_vary_headers(response, ('Accept-Language',))
+        if 'Content-Language' not in response:
+            response['Content-Language'] = language
+        return response
+
+    def is_language_prefix_patterns_used(self):
+        """
+        Returns `True` if the `LocaleRegexURLResolver` is used
+        at root level of the urlpatterns, else it returns `False`.
+        """
+        return self._is_language_prefix_patterns_used

common/djangoapps/django_locale/tests.py

+# pylint: disable=invalid-name, line-too-long, super-method-not-called
+"""
+Tests taken from Django upstream:
+https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py
+"""
+from django.conf import settings
+from django.test import TestCase, RequestFactory
+from django_locale.trans_real import (
+    parse_accept_lang_header, get_language_from_request, LANGUAGE_SESSION_KEY
+)
+
+# Added to test middleware around dark lang
+from django.contrib.auth.models import User
+from django.test.utils import override_settings
+from dark_lang.models import DarkLangConfig
+
+
+# Adding to support test differences between Django and our own settings
+@override_settings(LANGUAGES=[
+    ('pt', 'Portuguese'),
+    ('pt-br', 'Portuguese-Brasil'),
+    ('es', 'Spanish'),
+    ('es-ar', 'Spanish (Argentina)'),
+    ('de', 'Deutch'),
+    ('zh-cn', 'Chinese (China)'),
+    ('ar-sa', 'Arabic (Saudi Arabia)'),
+])
+class MiscTests(TestCase):
+    """
+    Tests taken from Django upstream:
+    https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py
+    """
+    def setUp(self):
+        self.rf = RequestFactory()
+        # Added to test middleware around dark lang
+        user = User()
+        user.save()
+        DarkLangConfig(
+            released_languages='pt, pt-br, es, de, es-ar, zh-cn, ar-sa',
+            changed_by=user,
+            enabled=True
+        ).save()
+
+    def test_parse_spec_http_header(self):
+        """
+        Testing HTTP header parsing. First, we test that we can parse the
+        values according to the spec (and that we extract all the pieces in
+        the right order).
+        """
+        p = parse_accept_lang_header
+        # Good headers.
+        self.assertEqual([('de', 1.0)], p('de'))
+        self.assertEqual([('en-AU', 1.0)], p('en-AU'))
+        self.assertEqual([('es-419', 1.0)], p('es-419'))
+        self.assertEqual([('*', 1.0)], p('*;q=1.00'))
+        self.assertEqual([('en-AU', 0.123)], p('en-AU;q=0.123'))
+        self.assertEqual([('en-au', 0.5)], p('en-au;q=0.5'))
+        self.assertEqual([('en-au', 1.0)], p('en-au;q=1.0'))
+        self.assertEqual([('da', 1.0), ('en', 0.5), ('en-gb', 0.25)], p('da, en-gb;q=0.25, en;q=0.5'))
+        self.assertEqual([('en-au-xx', 1.0)], p('en-au-xx'))
+        self.assertEqual([('de', 1.0), ('en-au', 0.75), ('en-us', 0.5), ('en', 0.25), ('es', 0.125), ('fa', 0.125)], p('de,en-au;q=0.75,en-us;q=0.5,en;q=0.25,es;q=0.125,fa;q=0.125'))
+        self.assertEqual([('*', 1.0)], p('*'))
+        self.assertEqual([('de', 1.0)], p('de;q=0.'))
+        self.assertEqual([('en', 1.0), ('*', 0.5)], p('en; q=1.0, * ; q=0.5'))
+        self.assertEqual([], p(''))
+
+        # Bad headers; should always return [].
+        self.assertEqual([], p('en-gb;q=1.0000'))
+        self.assertEqual([], p('en;q=0.1234'))
+        self.assertEqual([], p('en;q=.2'))
+        self.assertEqual([], p('abcdefghi-au'))
+        self.assertEqual([], p('**'))
+        self.assertEqual([], p('en,,gb'))
+        self.assertEqual([], p('en-au;q=0.1.0'))
+        self.assertEqual([], p('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXZ,en'))
+        self.assertEqual([], p('da, en-gb;q=0.8, en;q=0.7,#'))
+        self.assertEqual([], p('de;q=2.0'))
+        self.assertEqual([], p('de;q=0.a'))
+        self.assertEqual([], p('12-345'))
+        self.assertEqual([], p(''))
+
+    def test_parse_literal_http_header(self):
+        """
+        Now test that we parse a literal HTTP header correctly.
+        """
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-br'}
+        self.assertEqual('pt-br', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt'}
+        self.assertEqual('pt', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es,de'}
+        self.assertEqual('es', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-ar,de'}
+        self.assertEqual('es-ar', g(r))
+
+        # This test assumes there won't be a Django translation to a US
+        # variation of the Spanish language, a safe assumption. When the
+        # user sets it as the preferred language, the main 'es'
+        # translation should be selected instead.
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-us'}
+        self.assertEqual(g(r), 'es')
+
+        # This tests the following scenario: there isn't a main language (zh)
+        # translation of Django but there is a translation to variation (zh_CN)
+        # the user sets zh-cn as the preferred language, it should be selected
+        # by Django without falling back nor ignoring it.
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-cn,de'}
+        self.assertEqual(g(r), 'zh-cn')
+
+    def test_logic_masked_by_darklang(self):
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'ar-qa'}
+        self.assertEqual('ar-sa', g(r))
+
+        r.session = {LANGUAGE_SESSION_KEY: 'es'}
+        self.assertEqual('es', g(r))
+
+    def test_parse_language_cookie(self):
+        """
+        Now test that we parse language preferences stored in a cookie correctly.
+        """
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt-br'}
+        r.META = {}
+        self.assertEqual('pt-br', g(r))
+
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt'}
+        r.META = {}
+        self.assertEqual('pt', g(r))
+
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es'}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
+        self.assertEqual('es', g(r))
+
+        # This test assumes there won't be a Django translation to a US
+        # variation of the Spanish language, a safe assumption. When the
+        # user sets it as the preferred language, the main 'es'
+        # translation should be selected instead.
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es-us'}
+        r.META = {}
+        self.assertEqual(g(r), 'es')
+
+        # This tests the following scenario: there isn't a main language (zh)
+        # translation of Django but there is a translation to variation (zh_CN)
+        # the user sets zh-cn as the preferred language, it should be selected
+        # by Django without falling back nor ignoring it.
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'zh-cn'}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
+        self.assertEqual(g(r), 'zh-cn')

common/djangoapps/django_locale/trans_real.py

+"""Translation helper functions."""
+# Imported from Django 1.8
+# pylint: disable=invalid-name
+import re
+from django.conf import settings
+from django.conf.locale import LANG_INFO
+from django.utils import translation
+
+
+# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.
+# and RFC 3066, section 2.1
+accept_language_re = re.compile(r'''
+    ([A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*|\*)         # "en", "en-au", "x-y-z", "*"
+    (?:\s*;\s*q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
+    (?:\s*,\s*|$)                                 # Multiple accepts per header.
+    ''', re.VERBOSE)
+
+
+language_code_re = re.compile(r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*$', re.IGNORECASE)
+
+
+LANGUAGE_SESSION_KEY = '_language'
+
+
+def parse_accept_lang_header(lang_string):
+    """
+    Parses the lang_string, which is the body of an HTTP Accept-Language
+    header, and returns a list of (lang, q-value), ordered by 'q' values.
+
+    Any format errors in lang_string results in an empty list being returned.
+    """
+    # parse_accept_lang_header is broken until we are on Django 1.5 or greater
+    # See https://code.djangoproject.com/ticket/19381
+    result = []
+    pieces = accept_language_re.split(lang_string)
+    if pieces[-1]:
+        return []
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i: i + 3]
+        if first:
+            return []
+        priority = priority and float(priority) or 1.0
+        result.append((lang, priority))
+    result.sort(key=lambda k: k[1], reverse=True)
+    return result
+
+
+def get_supported_language_variant(lang_code, strict=False):
+    """
+    Returns the language-code that's listed in supported languages, possibly
+    selecting a more generic variant. Raises LookupError if nothing found.
+    If `strict` is False (the default), the function will look for an alternative
+    country-specific variant when the currently checked is not found.
+    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
+    as the provided language codes are taken from the HTTP request. See also
+    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
+    """
+    if lang_code:
+        # If 'fr-ca' is not supported, try special fallback or language-only 'fr'.
+        possible_lang_codes = [lang_code]
+        try:
+            # TODO skip this, or import updated LANG_INFO format from __future__
+            # (fallback option wasn't added until
+            # https://github.com/django/django/commit/5dcdbe95c749d36072f527e120a8cb463199ae0d)
+            possible_lang_codes.extend(LANG_INFO[lang_code]['fallback'])
+        except KeyError:
+            pass
+        generic_lang_code = lang_code.split('-')[0]
+        possible_lang_codes.append(generic_lang_code)
+        supported_lang_codes = dict(settings.LANGUAGES)
+
+        for code in possible_lang_codes:
+            # Note: django 1.4 implementation of check_for_language is OK to use
+            if code in supported_lang_codes and translation.check_for_language(code):
+                return code
+        if not strict:
+            # if fr-fr is not supported, try fr-ca.
+            for supported_code in supported_lang_codes:
+                if supported_code.startswith(generic_lang_code + '-'):
+                    return supported_code
+    raise LookupError(lang_code)
+
+
+def get_language_from_request(request, check_path=False):
+    """
+    Analyzes the request to find what language the user wants the system to
+    show. Only languages listed in settings.LANGUAGES are taken into account.
+    If the user requests a sublanguage where we have a main language, we send
+    out the main language.
+    If check_path is True, the URL path prefix will be checked for a language
+    code, otherwise this is skipped for backwards compatibility.
+    """
+    if check_path:
+        # Note: django 1.4 implementation of get_language_from_path is OK to use
+        lang_code = translation.get_language_from_path(request.path_info)
+        if lang_code is not None:
+            return lang_code
+
+    supported_lang_codes = dict(settings.LANGUAGES)
+
+    if hasattr(request, 'session'):
+        lang_code = request.session.get(LANGUAGE_SESSION_KEY)
+        # Note: django 1.4 implementation of check_for_language is OK to use
+        if lang_code in supported_lang_codes and lang_code is not None and translation.check_for_language(lang_code):
+            return lang_code
+
+    lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)
+
+    try:
+        return get_supported_language_variant(lang_code)
+    except LookupError:
+        pass
+
+    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
+    # broken in 1.4, so defined above
+    for accept_lang, unused in parse_accept_lang_header(accept):
+        if accept_lang == '*':
+            break
+
+        if not language_code_re.search(accept_lang):
+            continue
+
+        try:
+            return get_supported_language_variant(accept_lang)
+        except LookupError:
+            continue
+
+    try:
+        return get_supported_language_variant(settings.LANGUAGE_CODE)
+    except LookupError:
+        return settings.LANGUAGE_CODE

lms/envs/common.py

@@ -1152,7 +1152,9 @@ MIDDLEWARE_CLASSES = (
     'lang_pref.middleware.LanguagePreferenceMiddleware',
 
     # Detects user-requested locale from 'accept-language' header in http request
-    'django.middleware.locale.LocaleMiddleware',
+    # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671]
+    # 'django.middleware.locale.LocaleMiddleware',
+    'django_locale.middleware.LocaleMiddleware',
 
     'django.middleware.transaction.TransactionMiddleware',
     # 'debug_toolbar.middleware.DebugToolbarMiddleware',