Merge pull request #8402 from edx/sarina/annotate-middleware

Fix dark lang and django.middleware.locale behaviors LOC-72 LOC-85

cms/envs/common.py

@@ -305,7 +305,9 @@ MIDDLEWARE_CLASSES = (
     'embargo.middleware.EmbargoMiddleware',
 
     # Detects user-requested locale from 'accept-language' header in http request
-    'django.middleware.locale.LocaleMiddleware',
+    # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671]
+    # 'django.middleware.locale.LocaleMiddleware',
+    'django_locale.middleware.LocaleMiddleware',
 
     'django.middleware.transaction.TransactionMiddleware',
     # needs to run after locale middleware (or anything that modifies the request context)

common/djangoapps/dark_lang/middleware.py

@@ -12,10 +12,12 @@ the SessionMiddleware.
 """
 from django.conf import settings
 
-from django.utils.translation.trans_real import parse_accept_lang_header
-
 from dark_lang.models import DarkLangConfig
 
+# TODO re-import this once we're on Django 1.5 or greater. [PLAT-671]
+# from django.utils.translation.trans_real import parse_accept_lang_header
+from django_locale.trans_real import parse_accept_lang_header
+
 
 def dark_parse_accept_lang_header(accept):
     '''
@@ -81,11 +83,17 @@ class DarkLangMiddleware(object):
         self._clean_accept_headers(request)
         self._activate_preview_language(request)
 
-    def _is_released(self, lang_code):
-        """
-        ``True`` iff one of the values in ``self.released_langs`` is a prefix of ``lang_code``.
-        """
-        return any(lang_code.lower().startswith(released_lang.lower()) for released_lang in self.released_langs)
+    def _fuzzy_match(self, lang_code):
+        """Returns a fuzzy match for lang_code"""
+        if lang_code in self.released_langs:
+            return lang_code
+
+        lang_prefix = lang_code.partition('-')[0]
+        for released_lang in self.released_langs:
+            released_prefix = released_lang.partition('-')[0]
+            if lang_prefix == released_prefix:
+                return released_lang
+        return None
 
     def _format_accept_value(self, lang, priority=1.0):
         """
@@ -102,12 +110,13 @@ class DarkLangMiddleware(object):
         if accept is None or accept == '*':
             return
 
-        new_accept = ", ".join(
-            self._format_accept_value(lang, priority)
-            for lang, priority
-            in dark_parse_accept_lang_header(accept)
-            if self._is_released(lang)
-        )
+        new_accept = []
+        for lang, priority in dark_parse_accept_lang_header(accept):
+            fuzzy_code = self._fuzzy_match(lang.lower())
+            if fuzzy_code:
+                new_accept.append(self._format_accept_value(fuzzy_code, priority))
+
+        new_accept = ", ".join(new_accept)
 
         request.META['HTTP_ACCEPT_LANGUAGE'] = new_accept
 

common/djangoapps/dark_lang/models.py

@@ -25,7 +25,7 @@ class DarkLangConfig(ConfigurationModel):
         if not self.released_languages.strip():  # pylint: disable=no-member
             return []
 
-        languages = [lang.strip() for lang in self.released_languages.split(',')]  # pylint: disable=no-member
+        languages = [lang.lower().strip() for lang in self.released_languages.split(',')]  # pylint: disable=no-member
         # Put in alphabetical order
         languages.sort()
         return languages

common/djangoapps/dark_lang/tests.py

@@ -4,8 +4,10 @@ Tests of DarkLangMiddleware
 from django.contrib.auth.models import User
 from django.http import HttpRequest
 
+import ddt
 from django.test import TestCase
 from mock import Mock
+import unittest
 
 from dark_lang.middleware import DarkLangMiddleware
 from dark_lang.models import DarkLangConfig
@@ -23,6 +25,7 @@ def set_if_set(dct, key, value):
         dct[key] = value
 
 
+@ddt.ddt
 class DarkLangMiddlewareTests(TestCase):
     """
     Tests of DarkLangMiddleware
@@ -82,6 +85,10 @@ class DarkLangMiddlewareTests(TestCase):
     def test_wildcard_accept(self):
         self.assertAcceptEquals('*', self.process_request(accept='*'))
 
+    def test_malformed_accept(self):
+        self.assertAcceptEquals('', self.process_request(accept='xxxxxxxxxxxx'))
+        self.assertAcceptEquals('', self.process_request(accept='en;q=1.0, es-419:q-0.8'))
+
     def test_released_accept(self):
         self.assertAcceptEquals(
             'rel;q=1.0',
@@ -123,14 +130,17 @@ class DarkLangMiddlewareTests(TestCase):
         )
 
     def test_accept_released_territory(self):
+        # We will munge 'rel-ter' to be 'rel', so the 'rel-ter'
+        # user will actually receive the released language 'rel'
+        # (Otherwise, the user will actually end up getting the server default)
         self.assertAcceptEquals(
-            'rel-ter;q=1.0, rel;q=0.5',
+            'rel;q=1.0, rel;q=0.5',
             self.process_request(accept='rel-ter;q=1.0, rel;q=0.5')
         )
 
     def test_accept_mixed_case(self):
         self.assertAcceptEquals(
-            'rel-TER;q=1.0, REL;q=0.5',
+            'rel;q=1.0, rel;q=0.5',
             self.process_request(accept='rel-TER;q=1.0, REL;q=0.5')
         )
 
@@ -140,11 +150,85 @@ class DarkLangMiddlewareTests(TestCase):
             enabled=True
         ).save()
 
+        # Since we have only released "rel-ter", the requested code "rel" will
+        # fuzzy match to "rel-ter", in addition to "rel-ter" exact matching "rel-ter"
         self.assertAcceptEquals(
-            'rel-ter;q=1.0',
+            'rel-ter;q=1.0, rel-ter;q=0.5',
             self.process_request(accept='rel-ter;q=1.0, rel;q=0.5')
         )
 
+    @ddt.data(
+        ('es;q=1.0, pt;q=0.5', 'es-419;q=1.0'),  # 'es' should get 'es-419', not English
+        ('es-AR;q=1.0, pt;q=0.5', 'es-419;q=1.0'),  # 'es-AR' should get 'es-419', not English
+    )
+    @ddt.unpack
+    def test_partial_match_es419(self, accept_header, expected):
+        # Release es-419
+        DarkLangConfig(
+            released_languages=('es-419, en'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+
+        self.assertAcceptEquals(
+            expected,
+            self.process_request(accept=accept_header)
+        )
+
+    def test_partial_match_esar_es(self):
+        # If I release 'es', 'es-AR' should get 'es', not English
+        DarkLangConfig(
+            released_languages=('es, en'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+
+        self.assertAcceptEquals(
+            'es;q=1.0',
+            self.process_request(accept='es-AR;q=1.0, pt;q=0.5')
+        )
+
+    @ddt.data(
+        # Test condition: If I release 'es-419, es, es-es'...
+        ('es;q=1.0, pt;q=0.5', 'es;q=1.0'),          # 1. es should get es
+        ('es-419;q=1.0, pt;q=0.5', 'es-419;q=1.0'),  # 2. es-419 should get es-419
+        ('es-es;q=1.0, pt;q=0.5', 'es-es;q=1.0'),    # 3. es-es should get es-es
+    )
+    @ddt.unpack
+    def test_exact_match_gets_priority(self, accept_header, expected):
+        # Release 'es-419, es, es-es'
+        DarkLangConfig(
+            released_languages=('es-419, es, es-es'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+        self.assertAcceptEquals(
+            expected,
+            self.process_request(accept=accept_header)
+        )
+
+    @unittest.skip("This won't work until fallback is implemented for LA country codes. See LOC-86")
+    @ddt.data(
+        'es-AR',  # Argentina
+        'es-PY',  # Paraguay
+    )
+    def test_partial_match_es_la(self, latin_america_code):
+        # We need to figure out the best way to implement this. There are a ton of LA country
+        # codes that ought to fall back to 'es-419' rather than 'es-es'.
+        # http://unstats.un.org/unsd/methods/m49/m49regin.htm#americas
+        # If I release 'es, es-419'
+        # Latin American codes should get es-419
+        DarkLangConfig(
+            released_languages=('es, es-419'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+
+        self.assertAcceptEquals(
+            'es-419;q=1.0',
+            self.process_request(accept='{};q=1.0, pt;q=0.5'.format(latin_america_code))
+        )
+
     def assertSessionLangEquals(self, value, request):
         """
         Assert that the 'django_language' set in request.session is equal to value
@@ -224,6 +308,6 @@ class DarkLangMiddlewareTests(TestCase):
         ).save()
 
         self.assertAcceptEquals(
-            'zh-CN;q=1.0, zh-TW;q=0.5, zh-HK;q=0.3',
+            'zh-cn;q=1.0, zh-tw;q=0.5, zh-hk;q=0.3',
             self.process_request(accept='zh-Hans;q=1.0, zh-Hant-TW;q=0.5, zh-HK;q=0.3')
         )

common/djangoapps/django_locale/__init__.py

+"""
+TODO: This module is imported from the stable Django 1.8 branch, as a
+copy of https://github.com/django/django/blob/stable/1.8.x/django/middleware/locale.py.
+
+Remove this file and re-import this middleware from Django once the
+codebase is upgraded with a modern version of Django. [PLAT-671]
+"""

common/djangoapps/django_locale/middleware.py

+# TODO: This file is imported from the stable Django 1.8 branch. Remove this file
+# and re-import this middleware from Django once the codebase is upgraded. [PLAT-671]
+# pylint: disable=invalid-name, missing-docstring
+"This is the locale selecting middleware that will look at accept headers"
+
+from django.conf import settings
+from django.core.urlresolvers import (
+    LocaleRegexURLResolver, get_resolver, get_script_prefix, is_valid_path,
+)
+from django.http import HttpResponseRedirect
+from django.utils import translation
+from django.utils.cache import patch_vary_headers
+# Override the Django 1.4 implementation with the 1.8 implementation
+from django_locale.trans_real import get_language_from_request
+
+
+class LocaleMiddleware(object):
+    """
+    This is a very simple middleware that parses a request
+    and decides what translation object to install in the current
+    thread context. This allows pages to be dynamically
+    translated to the language the user desires (if the language
+    is available, of course).
+    """
+    response_redirect_class = HttpResponseRedirect
+
+    def __init__(self):
+        self._is_language_prefix_patterns_used = False
+        for url_pattern in get_resolver(None).url_patterns:
+            if isinstance(url_pattern, LocaleRegexURLResolver):
+                self._is_language_prefix_patterns_used = True
+                break
+
+    def process_request(self, request):
+        check_path = self.is_language_prefix_patterns_used()
+        # This call is broken in Django 1.4:
+        # https://github.com/django/django/blob/stable/1.4.x/django/utils/translation/trans_real.py#L399
+        # (we override parse_accept_lang_header to a fixed version in dark_lang.middleware)
+        language = get_language_from_request(
+            request, check_path=check_path)
+        translation.activate(language)
+        request.LANGUAGE_CODE = translation.get_language()
+
+    def process_response(self, request, response):
+        language = translation.get_language()
+        language_from_path = translation.get_language_from_path(request.path_info)
+        if (response.status_code == 404 and not language_from_path
+                and self.is_language_prefix_patterns_used()):
+            urlconf = getattr(request, 'urlconf', None)
+            language_path = '/%s%s' % (language, request.path_info)
+            path_valid = is_valid_path(language_path, urlconf)
+            if (not path_valid and settings.APPEND_SLASH
+                    and not language_path.endswith('/')):
+                path_valid = is_valid_path("%s/" % language_path, urlconf)
+
+            if path_valid:
+                script_prefix = get_script_prefix()
+                language_url = "%s://%s%s" % (
+                    request.scheme,
+                    request.get_host(),
+                    # insert language after the script prefix and before the
+                    # rest of the URL
+                    request.get_full_path().replace(
+                        script_prefix,
+                        '%s%s/' % (script_prefix, language),
+                        1
+                    )
+                )
+                return self.response_redirect_class(language_url)
+
+        if not (self.is_language_prefix_patterns_used()
+                and language_from_path):
+            patch_vary_headers(response, ('Accept-Language',))
+        if 'Content-Language' not in response:
+            response['Content-Language'] = language
+        return response
+
+    def is_language_prefix_patterns_used(self):
+        """
+        Returns `True` if the `LocaleRegexURLResolver` is used
+        at root level of the urlpatterns, else it returns `False`.
+        """
+        return self._is_language_prefix_patterns_used

common/djangoapps/django_locale/tests.py

+# pylint: disable=invalid-name, line-too-long, super-method-not-called
+"""
+Tests taken from Django upstream:
+https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py
+"""
+from django.conf import settings
+from django.test import TestCase, RequestFactory
+from django_locale.trans_real import (
+    parse_accept_lang_header, get_language_from_request, LANGUAGE_SESSION_KEY
+)
+
+# Added to test middleware around dark lang
+from django.contrib.auth.models import User
+from django.test.utils import override_settings
+from dark_lang.models import DarkLangConfig
+
+
+# Adding to support test differences between Django and our own settings
+@override_settings(LANGUAGES=[
+    ('pt', 'Portuguese'),
+    ('pt-br', 'Portuguese-Brasil'),
+    ('es', 'Spanish'),
+    ('es-ar', 'Spanish (Argentina)'),
+    ('de', 'Deutch'),
+    ('zh-cn', 'Chinese (China)'),
+    ('ar-sa', 'Arabic (Saudi Arabia)'),
+])
+class MiscTests(TestCase):
+    """
+    Tests taken from Django upstream:
+    https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py
+    """
+    def setUp(self):
+        self.rf = RequestFactory()
+        # Added to test middleware around dark lang
+        user = User()
+        user.save()
+        DarkLangConfig(
+            released_languages='pt, pt-br, es, de, es-ar, zh-cn, ar-sa',
+            changed_by=user,
+            enabled=True
+        ).save()
+
+    def test_parse_spec_http_header(self):
+        """
+        Testing HTTP header parsing. First, we test that we can parse the
+        values according to the spec (and that we extract all the pieces in
+        the right order).
+        """
+        p = parse_accept_lang_header
+        # Good headers.
+        self.assertEqual([('de', 1.0)], p('de'))
+        self.assertEqual([('en-AU', 1.0)], p('en-AU'))
+        self.assertEqual([('es-419', 1.0)], p('es-419'))
+        self.assertEqual([('*', 1.0)], p('*;q=1.00'))
+        self.assertEqual([('en-AU', 0.123)], p('en-AU;q=0.123'))
+        self.assertEqual([('en-au', 0.5)], p('en-au;q=0.5'))
+        self.assertEqual([('en-au', 1.0)], p('en-au;q=1.0'))
+        self.assertEqual([('da', 1.0), ('en', 0.5), ('en-gb', 0.25)], p('da, en-gb;q=0.25, en;q=0.5'))
+        self.assertEqual([('en-au-xx', 1.0)], p('en-au-xx'))
+        self.assertEqual([('de', 1.0), ('en-au', 0.75), ('en-us', 0.5), ('en', 0.25), ('es', 0.125), ('fa', 0.125)], p('de,en-au;q=0.75,en-us;q=0.5,en;q=0.25,es;q=0.125,fa;q=0.125'))
+        self.assertEqual([('*', 1.0)], p('*'))
+        self.assertEqual([('de', 1.0)], p('de;q=0.'))
+        self.assertEqual([('en', 1.0), ('*', 0.5)], p('en; q=1.0, * ; q=0.5'))
+        self.assertEqual([], p(''))
+
+        # Bad headers; should always return [].
+        self.assertEqual([], p('en-gb;q=1.0000'))
+        self.assertEqual([], p('en;q=0.1234'))
+        self.assertEqual([], p('en;q=.2'))
+        self.assertEqual([], p('abcdefghi-au'))
+        self.assertEqual([], p('**'))
+        self.assertEqual([], p('en,,gb'))
+        self.assertEqual([], p('en-au;q=0.1.0'))
+        self.assertEqual([], p('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXZ,en'))
+        self.assertEqual([], p('da, en-gb;q=0.8, en;q=0.7,#'))
+        self.assertEqual([], p('de;q=2.0'))
+        self.assertEqual([], p('de;q=0.a'))
+        self.assertEqual([], p('12-345'))
+        self.assertEqual([], p(''))
+
+    def test_parse_literal_http_header(self):
+        """
+        Now test that we parse a literal HTTP header correctly.
+        """
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-br'}
+        self.assertEqual('pt-br', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt'}
+        self.assertEqual('pt', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es,de'}
+        self.assertEqual('es', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-ar,de'}
+        self.assertEqual('es-ar', g(r))
+
+        # This test assumes there won't be a Django translation to a US
+        # variation of the Spanish language, a safe assumption. When the
+        # user sets it as the preferred language, the main 'es'
+        # translation should be selected instead.
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-us'}
+        self.assertEqual(g(r), 'es')
+
+        # This tests the following scenario: there isn't a main language (zh)
+        # translation of Django but there is a translation to variation (zh_CN)
+        # the user sets zh-cn as the preferred language, it should be selected
+        # by Django without falling back nor ignoring it.
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-cn,de'}
+        self.assertEqual(g(r), 'zh-cn')
+
+    def test_logic_masked_by_darklang(self):
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'ar-qa'}
+        self.assertEqual('ar-sa', g(r))
+
+        r.session = {LANGUAGE_SESSION_KEY: 'es'}
+        self.assertEqual('es', g(r))
+
+    def test_parse_language_cookie(self):
+        """
+        Now test that we parse language preferences stored in a cookie correctly.
+        """
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt-br'}
+        r.META = {}
+        self.assertEqual('pt-br', g(r))
+
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt'}
+        r.META = {}
+        self.assertEqual('pt', g(r))
+
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es'}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
+        self.assertEqual('es', g(r))
+
+        # This test assumes there won't be a Django translation to a US
+        # variation of the Spanish language, a safe assumption. When the
+        # user sets it as the preferred language, the main 'es'
+        # translation should be selected instead.
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es-us'}
+        r.META = {}
+        self.assertEqual(g(r), 'es')
+
+        # This tests the following scenario: there isn't a main language (zh)
+        # translation of Django but there is a translation to variation (zh_CN)
+        # the user sets zh-cn as the preferred language, it should be selected
+        # by Django without falling back nor ignoring it.
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'zh-cn'}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
+        self.assertEqual(g(r), 'zh-cn')

common/djangoapps/django_locale/trans_real.py

+"""Translation helper functions."""
+# Imported from Django 1.8
+# pylint: disable=invalid-name
+import re
+from django.conf import settings
+from django.conf.locale import LANG_INFO
+from django.utils import translation
+
+
+# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.
+# and RFC 3066, section 2.1
+accept_language_re = re.compile(r'''
+    ([A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*|\*)         # "en", "en-au", "x-y-z", "*"
+    (?:\s*;\s*q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
+    (?:\s*,\s*|$)                                 # Multiple accepts per header.
+    ''', re.VERBOSE)
+
+
+language_code_re = re.compile(r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*$', re.IGNORECASE)
+
+
+LANGUAGE_SESSION_KEY = '_language'
+
+
+def parse_accept_lang_header(lang_string):
+    """
+    Parses the lang_string, which is the body of an HTTP Accept-Language
+    header, and returns a list of (lang, q-value), ordered by 'q' values.
+
+    Any format errors in lang_string results in an empty list being returned.
+    """
+    # parse_accept_lang_header is broken until we are on Django 1.5 or greater
+    # See https://code.djangoproject.com/ticket/19381
+    result = []
+    pieces = accept_language_re.split(lang_string)
+    if pieces[-1]:
+        return []
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i: i + 3]
+        if first:
+            return []
+        priority = priority and float(priority) or 1.0
+        result.append((lang, priority))
+    result.sort(key=lambda k: k[1], reverse=True)
+    return result
+
+
+def get_supported_language_variant(lang_code, strict=False):
+    """
+    Returns the language-code that's listed in supported languages, possibly
+    selecting a more generic variant. Raises LookupError if nothing found.
+    If `strict` is False (the default), the function will look for an alternative
+    country-specific variant when the currently checked is not found.
+    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
+    as the provided language codes are taken from the HTTP request. See also
+    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
+    """
+    if lang_code:
+        # If 'fr-ca' is not supported, try special fallback or language-only 'fr'.
+        possible_lang_codes = [lang_code]
+        try:
+            # TODO skip this, or import updated LANG_INFO format from __future__
+            # (fallback option wasn't added until
+            # https://github.com/django/django/commit/5dcdbe95c749d36072f527e120a8cb463199ae0d)
+            possible_lang_codes.extend(LANG_INFO[lang_code]['fallback'])
+        except KeyError:
+            pass
+        generic_lang_code = lang_code.split('-')[0]
+        possible_lang_codes.append(generic_lang_code)
+        supported_lang_codes = dict(settings.LANGUAGES)
+
+        for code in possible_lang_codes:
+            # Note: django 1.4 implementation of check_for_language is OK to use
+            if code in supported_lang_codes and translation.check_for_language(code):
+                return code
+        if not strict:
+            # if fr-fr is not supported, try fr-ca.
+            for supported_code in supported_lang_codes:
+                if supported_code.startswith(generic_lang_code + '-'):
+                    return supported_code
+    raise LookupError(lang_code)
+
+
+def get_language_from_request(request, check_path=False):
+    """
+    Analyzes the request to find what language the user wants the system to
+    show. Only languages listed in settings.LANGUAGES are taken into account.
+    If the user requests a sublanguage where we have a main language, we send
+    out the main language.
+    If check_path is True, the URL path prefix will be checked for a language
+    code, otherwise this is skipped for backwards compatibility.
+    """
+    if check_path:
+        # Note: django 1.4 implementation of get_language_from_path is OK to use
+        lang_code = translation.get_language_from_path(request.path_info)
+        if lang_code is not None:
+            return lang_code
+
+    supported_lang_codes = dict(settings.LANGUAGES)
+
+    if hasattr(request, 'session'):
+        lang_code = request.session.get(LANGUAGE_SESSION_KEY)
+        # Note: django 1.4 implementation of check_for_language is OK to use
+        if lang_code in supported_lang_codes and lang_code is not None and translation.check_for_language(lang_code):
+            return lang_code
+
+    lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)
+
+    try:
+        return get_supported_language_variant(lang_code)
+    except LookupError:
+        pass
+
+    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
+    # broken in 1.4, so defined above
+    for accept_lang, unused in parse_accept_lang_header(accept):
+        if accept_lang == '*':
+            break
+
+        if not language_code_re.search(accept_lang):
+            continue
+
+        try:
+            return get_supported_language_variant(accept_lang)
+        except LookupError:
+            continue
+
+    try:
+        return get_supported_language_variant(settings.LANGUAGE_CODE)
+    except LookupError:
+        return settings.LANGUAGE_CODE

lms/djangoapps/courseware/tests/test_i18n.py

@@ -4,37 +4,59 @@ Tests i18n in courseware
 import re
 from nose.plugins.attrib import attr
 
+from django.contrib.auth.models import User
 from django.test import TestCase
-from django.test.utils import override_settings
 
+from dark_lang.models import DarkLangConfig
 
-@attr('shard_1')
-@override_settings(LANGUAGES=[('eo', 'Esperanto'), ('ar', 'Arabic')])
-class I18nTestCase(TestCase):
+
+class BaseI18nTestCase(TestCase):
     """
-    Tests for i18n
+    Base utilities for i18n test classes to derive from
     """
     def assert_tag_has_attr(self, content, tag, attname, value):
         """Assert that a tag in `content` has a certain value in a certain attribute."""
-        regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d ]+)['"][^>]*>""".format(tag=tag, attname=attname)
+        regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d\- ]+)['"][^>]*>""".format(tag=tag, attname=attname)
         match = re.search(regex, content)
-        self.assertTrue(match, "Couldn't find desired tag in %r" % content)
+        self.assertTrue(match, "Couldn't find desired tag '%s' with attr '%s' in %r" % (tag, attname, content))
         attvalues = match.group(1).split()
         self.assertIn(value, attvalues)
 
+    def release_languages(self, languages):
+        """
+        Release a set of languages using the dark lang interface.
+        languages is a list of comma-separated lang codes, eg, 'ar, es-419'
+        """
+        user = User()
+        user.save()
+        DarkLangConfig(
+            released_languages=languages,
+            changed_by=user,
+            enabled=True
+        ).save()
+
+
+@attr('shard_1')
+class I18nTestCase(BaseI18nTestCase):
+    """
+    Tests for i18n
+    """
     def test_default_is_en(self):
+        self.release_languages('fr')
         response = self.client.get('/')
         self.assert_tag_has_attr(response.content, "html", "lang", "en")
         self.assertEqual(response['Content-Language'], 'en')
         self.assert_tag_has_attr(response.content, "body", "class", "lang_en")
 
     def test_esperanto(self):
+        self.release_languages('fr, eo')
         response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='eo')
         self.assert_tag_has_attr(response.content, "html", "lang", "eo")
         self.assertEqual(response['Content-Language'], 'eo')
         self.assert_tag_has_attr(response.content, "body", "class", "lang_eo")
 
     def test_switching_languages_bidi(self):
+        self.release_languages('ar, eo')
         response = self.client.get('/')
         self.assert_tag_has_attr(response.content, "html", "lang", "en")
         self.assertEqual(response['Content-Language'], 'en')
@@ -46,3 +68,26 @@ class I18nTestCase(TestCase):
         self.assertEqual(response['Content-Language'], 'ar')
         self.assert_tag_has_attr(response.content, "body", "class", "lang_ar")
         self.assert_tag_has_attr(response.content, "body", "class", "rtl")
+
+
+@attr('shard_1')
+class I18nRegressionTests(BaseI18nTestCase):
+    """
+    Tests for i18n
+    """
+    def test_es419_acceptance(self):
+        # Regression test; LOC-72, and an issue with Django
+        self.release_languages('es-419')
+        response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='es-419')
+        self.assert_tag_has_attr(response.content, "html", "lang", "es-419")
+
+    def test_unreleased_lang_resolution(self):
+        # Regression test; LOC-85
+        self.release_languages('fa')
+
+        # We've released 'fa', AND we have language files for 'fa-ir' but
+        # we want to keep 'fa-ir' as a dark language. Requesting 'fa-ir'
+        # in the http request (NOT with the ?preview-lang query param) should
+        # receive files for 'fa'
+        response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='fa-ir')
+        self.assert_tag_has_attr(response.content, "html", "lang", "fa")

lms/envs/common.py

@@ -1156,7 +1156,9 @@ MIDDLEWARE_CLASSES = (
     'lang_pref.middleware.LanguagePreferenceMiddleware',
 
     # Detects user-requested locale from 'accept-language' header in http request
-    'django.middleware.locale.LocaleMiddleware',
+    # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671]
+    # 'django.middleware.locale.LocaleMiddleware',
+    'django_locale.middleware.LocaleMiddleware',
 
     'django.middleware.transaction.TransactionMiddleware',
     # 'debug_toolbar.middleware.DebugToolbarMiddleware',