Merge pull request #8667 from edx/sarina/update-middleware-LOC-87

Fix dark lang and django.middleware.locale behaviors 

cms/envs/common.py

@@ -308,7 +308,9 @@ MIDDLEWARE_CLASSES = (
     'embargo.middleware.EmbargoMiddleware',
 
     # Detects user-requested locale from 'accept-language' header in http request
-    'django.middleware.locale.LocaleMiddleware',
+    # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671]
+    # 'django.middleware.locale.LocaleMiddleware',
+    'django_locale.middleware.LocaleMiddleware',
 
     'django.middleware.transaction.TransactionMiddleware',
     # needs to run after locale middleware (or anything that modifies the request context)

common/djangoapps/dark_lang/__init__.py

@@ -17,3 +17,6 @@ Run migrations to install the configuration table.
 Use the admin site to add a new ``DarkLangConfig`` that is enabled, and lists the
 languages that should be released.
 """
+
+# this is the UserPreference key for the currently-active dark language, if any
+DARK_LANGUAGE_KEY = 'dark-lang'

common/djangoapps/dark_lang/middleware.py

@@ -12,9 +12,18 @@ the SessionMiddleware.
 """
 from django.conf import settings
 
-from django.utils.translation.trans_real import parse_accept_lang_header
-
+from dark_lang import DARK_LANGUAGE_KEY
 from dark_lang.models import DarkLangConfig
+from openedx.core.djangoapps.user_api.preferences.api import (
+    delete_user_preference, get_user_preference, set_user_preference
+)
+from openedx.core.djangoapps.user_api.errors import UserNotFound
+from lang_pref import LANGUAGE_KEY
+
+# TODO re-import this once we're on Django 1.5 or greater. [PLAT-671]
+# from django.utils.translation.trans_real import parse_accept_lang_header
+# from django.utils.translation import LANGUAGE_SESSION_KEY
+from django_locale.trans_real import parse_accept_lang_header, LANGUAGE_SESSION_KEY
 
 
 def dark_parse_accept_lang_header(accept):
@@ -81,11 +90,17 @@ class DarkLangMiddleware(object):
         self._clean_accept_headers(request)
         self._activate_preview_language(request)
 
-    def _is_released(self, lang_code):
-        """
-        ``True`` iff one of the values in ``self.released_langs`` is a prefix of ``lang_code``.
-        """
-        return any(lang_code.lower().startswith(released_lang.lower()) for released_lang in self.released_langs)
+    def _fuzzy_match(self, lang_code):
+        """Returns a fuzzy match for lang_code"""
+        if lang_code in self.released_langs:
+            return lang_code
+
+        lang_prefix = lang_code.partition('-')[0]
+        for released_lang in self.released_langs:
+            released_prefix = released_lang.partition('-')[0]
+            if lang_prefix == released_prefix:
+                return released_lang
+        return None
 
     def _format_accept_value(self, lang, priority=1.0):
         """
@@ -102,12 +117,13 @@ class DarkLangMiddleware(object):
         if accept is None or accept == '*':
             return
 
-        new_accept = ", ".join(
-            self._format_accept_value(lang, priority)
-            for lang, priority
-            in dark_parse_accept_lang_header(accept)
-            if self._is_released(lang)
-        )
+        new_accept = []
+        for lang, priority in dark_parse_accept_lang_header(accept):
+            fuzzy_code = self._fuzzy_match(lang.lower())
+            if fuzzy_code:
+                new_accept.append(self._format_accept_value(fuzzy_code, priority))
+
+        new_accept = ", ".join(new_accept)
 
         request.META['HTTP_ACCEPT_LANGUAGE'] = new_accept
 
@@ -115,15 +131,29 @@ class DarkLangMiddleware(object):
         """
         If the request has the get parameter ``preview-lang``,
         and that language doesn't appear in ``self.released_langs``,
-        then set the session ``django_language`` to that language.
+        then set the session LANGUAGE_SESSION_KEY to that language.
         """
         if 'clear-lang' in request.GET:
-            if 'django_language' in request.session:
-                del request.session['django_language']
+            # Reset dark lang
+            delete_user_preference(request.user, DARK_LANGUAGE_KEY)
+            # Reset user's language to their language preference, if they have one
+            user_pref = get_user_preference(request.user, LANGUAGE_KEY)
+            if user_pref:
+                request.session[LANGUAGE_SESSION_KEY] = user_pref
+            elif LANGUAGE_SESSION_KEY in request.session:
+                del request.session[LANGUAGE_SESSION_KEY]
+            return
 
         preview_lang = request.GET.get('preview-lang', None)
+        if not preview_lang:
+            try:
+                # Try to get the request user's preference (might not have a user, though)
+                preview_lang = get_user_preference(request.user, DARK_LANGUAGE_KEY)
+            except UserNotFound:
+                return
 
         if not preview_lang:
             return
 
-        request.session['django_language'] = preview_lang
+        request.session[LANGUAGE_SESSION_KEY] = preview_lang
+        set_user_preference(request.user, DARK_LANGUAGE_KEY, preview_lang)

common/djangoapps/dark_lang/models.py

@@ -25,7 +25,7 @@ class DarkLangConfig(ConfigurationModel):
         if not self.released_languages.strip():  # pylint: disable=no-member
             return []
 
-        languages = [lang.strip() for lang in self.released_languages.split(',')]  # pylint: disable=no-member
+        languages = [lang.lower().strip() for lang in self.released_languages.split(',')]  # pylint: disable=no-member
         # Put in alphabetical order
         languages.sort()
         return languages

common/djangoapps/dark_lang/tests.py

@@ -4,11 +4,17 @@ Tests of DarkLangMiddleware
 from django.contrib.auth.models import User
 from django.http import HttpRequest
 
+import ddt
 from django.test import TestCase
 from mock import Mock
+import unittest
 
 from dark_lang.middleware import DarkLangMiddleware
 from dark_lang.models import DarkLangConfig
+# TODO PLAT-671 Import from Django 1.8
+# from django.utils.translation import LANGUAGE_SESSION_KEY
+from django_locale.trans_real import LANGUAGE_SESSION_KEY
+from student.tests.factories import UserFactory
 
 
 UNSET = object()
@@ -23,6 +29,7 @@ def set_if_set(dct, key, value):
         dct[key] = value
 
 
+@ddt.ddt
 class DarkLangMiddlewareTests(TestCase):
     """
     Tests of DarkLangMiddleware
@@ -37,18 +44,18 @@ class DarkLangMiddlewareTests(TestCase):
             enabled=True
         ).save()
 
-    def process_request(self, django_language=UNSET, accept=UNSET, preview_lang=UNSET, clear_lang=UNSET):
+    def process_request(self, language_session_key=UNSET, accept=UNSET, preview_lang=UNSET, clear_lang=UNSET):
         """
         Build a request and then process it using the ``DarkLangMiddleware``.
 
         Args:
-            django_language (str): The language code to set in request.session['django_language']
+            language_session_key (str): The language code to set in request.session[LANUGAGE_SESSION_KEY]
             accept (str): The accept header to set in request.META['HTTP_ACCEPT_LANGUAGE']
             preview_lang (str): The value to set in request.GET['preview_lang']
             clear_lang (str): The value to set in request.GET['clear_lang']
         """
         session = {}
-        set_if_set(session, 'django_language', django_language)
+        set_if_set(session, LANGUAGE_SESSION_KEY, language_session_key)
 
         meta = {}
         set_if_set(meta, 'HTTP_ACCEPT_LANGUAGE', accept)
@@ -61,7 +68,8 @@ class DarkLangMiddlewareTests(TestCase):
             spec=HttpRequest,
             session=session,
             META=meta,
-            GET=get
+            GET=get,
+            user=UserFactory()
         )
         self.assertIsNone(DarkLangMiddleware().process_request(request))
         return request
@@ -82,6 +90,10 @@ class DarkLangMiddlewareTests(TestCase):
     def test_wildcard_accept(self):
         self.assertAcceptEquals('*', self.process_request(accept='*'))
 
+    def test_malformed_accept(self):
+        self.assertAcceptEquals('', self.process_request(accept='xxxxxxxxxxxx'))
+        self.assertAcceptEquals('', self.process_request(accept='en;q=1.0, es-419:q-0.8'))
+
     def test_released_accept(self):
         self.assertAcceptEquals(
             'rel;q=1.0',
@@ -123,14 +135,17 @@ class DarkLangMiddlewareTests(TestCase):
         )
 
     def test_accept_released_territory(self):
+        # We will munge 'rel-ter' to be 'rel', so the 'rel-ter'
+        # user will actually receive the released language 'rel'
+        # (Otherwise, the user will actually end up getting the server default)
         self.assertAcceptEquals(
-            'rel-ter;q=1.0, rel;q=0.5',
+            'rel;q=1.0, rel;q=0.5',
             self.process_request(accept='rel-ter;q=1.0, rel;q=0.5')
         )
 
     def test_accept_mixed_case(self):
         self.assertAcceptEquals(
-            'rel-TER;q=1.0, REL;q=0.5',
+            'rel;q=1.0, rel;q=0.5',
             self.process_request(accept='rel-TER;q=1.0, REL;q=0.5')
         )
 
@@ -140,18 +155,92 @@ class DarkLangMiddlewareTests(TestCase):
             enabled=True
         ).save()
 
+        # Since we have only released "rel-ter", the requested code "rel" will
+        # fuzzy match to "rel-ter", in addition to "rel-ter" exact matching "rel-ter"
         self.assertAcceptEquals(
-            'rel-ter;q=1.0',
+            'rel-ter;q=1.0, rel-ter;q=0.5',
             self.process_request(accept='rel-ter;q=1.0, rel;q=0.5')
         )
 
+    @ddt.data(
+        ('es;q=1.0, pt;q=0.5', 'es-419;q=1.0'),  # 'es' should get 'es-419', not English
+        ('es-AR;q=1.0, pt;q=0.5', 'es-419;q=1.0'),  # 'es-AR' should get 'es-419', not English
+    )
+    @ddt.unpack
+    def test_partial_match_es419(self, accept_header, expected):
+        # Release es-419
+        DarkLangConfig(
+            released_languages=('es-419, en'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+
+        self.assertAcceptEquals(
+            expected,
+            self.process_request(accept=accept_header)
+        )
+
+    def test_partial_match_esar_es(self):
+        # If I release 'es', 'es-AR' should get 'es', not English
+        DarkLangConfig(
+            released_languages=('es, en'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+
+        self.assertAcceptEquals(
+            'es;q=1.0',
+            self.process_request(accept='es-AR;q=1.0, pt;q=0.5')
+        )
+
+    @ddt.data(
+        # Test condition: If I release 'es-419, es, es-es'...
+        ('es;q=1.0, pt;q=0.5', 'es;q=1.0'),          # 1. es should get es
+        ('es-419;q=1.0, pt;q=0.5', 'es-419;q=1.0'),  # 2. es-419 should get es-419
+        ('es-es;q=1.0, pt;q=0.5', 'es-es;q=1.0'),    # 3. es-es should get es-es
+    )
+    @ddt.unpack
+    def test_exact_match_gets_priority(self, accept_header, expected):
+        # Release 'es-419, es, es-es'
+        DarkLangConfig(
+            released_languages=('es-419, es, es-es'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+        self.assertAcceptEquals(
+            expected,
+            self.process_request(accept=accept_header)
+        )
+
+    @unittest.skip("This won't work until fallback is implemented for LA country codes. See LOC-86")
+    @ddt.data(
+        'es-AR',  # Argentina
+        'es-PY',  # Paraguay
+    )
+    def test_partial_match_es_la(self, latin_america_code):
+        # We need to figure out the best way to implement this. There are a ton of LA country
+        # codes that ought to fall back to 'es-419' rather than 'es-es'.
+        # http://unstats.un.org/unsd/methods/m49/m49regin.htm#americas
+        # If I release 'es, es-419'
+        # Latin American codes should get es-419
+        DarkLangConfig(
+            released_languages=('es, es-419'),
+            changed_by=self.user,
+            enabled=True
+        ).save()
+
+        self.assertAcceptEquals(
+            'es-419;q=1.0',
+            self.process_request(accept='{};q=1.0, pt;q=0.5'.format(latin_america_code))
+        )
+
     def assertSessionLangEquals(self, value, request):
         """
-        Assert that the 'django_language' set in request.session is equal to value
+        Assert that the LANGUAGE_SESSION_KEY set in request.session is equal to value
         """
         self.assertEquals(
             value,
-            request.session.get('django_language', UNSET)
+            request.session.get(LANGUAGE_SESSION_KEY, UNSET)
         )
 
     def test_preview_lang_with_released_language(self):
@@ -163,7 +252,7 @@ class DarkLangMiddlewareTests(TestCase):
 
         self.assertSessionLangEquals(
             'rel',
-            self.process_request(preview_lang='rel', django_language='notrel')
+            self.process_request(preview_lang='rel', language_session_key='notrel')
         )
 
     def test_preview_lang_with_dark_language(self):
@@ -174,7 +263,7 @@ class DarkLangMiddlewareTests(TestCase):
 
         self.assertSessionLangEquals(
             'unrel',
-            self.process_request(preview_lang='unrel', django_language='notrel')
+            self.process_request(preview_lang='unrel', language_session_key='notrel')
         )
 
     def test_clear_lang(self):
@@ -185,12 +274,12 @@ class DarkLangMiddlewareTests(TestCase):
 
         self.assertSessionLangEquals(
             UNSET,
-            self.process_request(clear_lang=True, django_language='rel')
+            self.process_request(clear_lang=True, language_session_key='rel')
         )
 
         self.assertSessionLangEquals(
             UNSET,
-            self.process_request(clear_lang=True, django_language='unrel')
+            self.process_request(clear_lang=True, language_session_key='unrel')
         )
 
     def test_disabled(self):
@@ -203,17 +292,17 @@ class DarkLangMiddlewareTests(TestCase):
 
         self.assertSessionLangEquals(
             'rel',
-            self.process_request(clear_lang=True, django_language='rel')
+            self.process_request(clear_lang=True, language_session_key='rel')
         )
 
         self.assertSessionLangEquals(
             'unrel',
-            self.process_request(clear_lang=True, django_language='unrel')
+            self.process_request(clear_lang=True, language_session_key='unrel')
         )
 
         self.assertSessionLangEquals(
             'rel',
-            self.process_request(preview_lang='unrel', django_language='rel')
+            self.process_request(preview_lang='unrel', language_session_key='rel')
         )
 
     def test_accept_chinese_language_codes(self):
@@ -224,6 +313,6 @@ class DarkLangMiddlewareTests(TestCase):
         ).save()
 
         self.assertAcceptEquals(
-            'zh-CN;q=1.0, zh-TW;q=0.5, zh-HK;q=0.3',
+            'zh-cn;q=1.0, zh-tw;q=0.5, zh-hk;q=0.3',
             self.process_request(accept='zh-Hans;q=1.0, zh-Hant-TW;q=0.5, zh-HK;q=0.3')
         )

common/djangoapps/django_locale/__init__.py

+"""
+TODO: This module is imported from the stable Django 1.8 branch, as a
+copy of https://github.com/django/django/blob/stable/1.8.x/django/middleware/locale.py.
+
+Remove this file and re-import this middleware from Django once the
+codebase is upgraded with a modern version of Django. [PLAT-671]
+"""

common/djangoapps/django_locale/middleware.py

+# TODO: This file is imported from the stable Django 1.8 branch. Remove this file
+# and re-import this middleware from Django once the codebase is upgraded. [PLAT-671]
+# pylint: disable=invalid-name, missing-docstring
+"This is the locale selecting middleware that will look at accept headers"
+
+from django.conf import settings
+from django.core.urlresolvers import (
+    LocaleRegexURLResolver, get_resolver, get_script_prefix, is_valid_path,
+)
+from django.http import HttpResponseRedirect
+from django.utils import translation
+from django.utils.cache import patch_vary_headers
+# Override the Django 1.4 implementation with the 1.8 implementation
+from django_locale.trans_real import get_language_from_request
+
+
+class LocaleMiddleware(object):
+    """
+    This is a very simple middleware that parses a request
+    and decides what translation object to install in the current
+    thread context. This allows pages to be dynamically
+    translated to the language the user desires (if the language
+    is available, of course).
+    """
+    response_redirect_class = HttpResponseRedirect
+
+    def __init__(self):
+        self._is_language_prefix_patterns_used = False
+        for url_pattern in get_resolver(None).url_patterns:
+            if isinstance(url_pattern, LocaleRegexURLResolver):
+                self._is_language_prefix_patterns_used = True
+                break
+
+    def process_request(self, request):
+        check_path = self.is_language_prefix_patterns_used()
+        # This call is broken in Django 1.4:
+        # https://github.com/django/django/blob/stable/1.4.x/django/utils/translation/trans_real.py#L399
+        # (we override parse_accept_lang_header to a fixed version in dark_lang.middleware)
+        language = get_language_from_request(
+            request, check_path=check_path)
+        translation.activate(language)
+        request.LANGUAGE_CODE = translation.get_language()
+
+    def process_response(self, request, response):
+        language = translation.get_language()
+        language_from_path = translation.get_language_from_path(request.path_info)
+        if (response.status_code == 404 and not language_from_path
+                and self.is_language_prefix_patterns_used()):
+            urlconf = getattr(request, 'urlconf', None)
+            language_path = '/%s%s' % (language, request.path_info)
+            path_valid = is_valid_path(language_path, urlconf)
+            if (not path_valid and settings.APPEND_SLASH
+                    and not language_path.endswith('/')):
+                path_valid = is_valid_path("%s/" % language_path, urlconf)
+
+            if path_valid:
+                script_prefix = get_script_prefix()
+                language_url = "%s://%s%s" % (
+                    request.scheme,
+                    request.get_host(),
+                    # insert language after the script prefix and before the
+                    # rest of the URL
+                    request.get_full_path().replace(
+                        script_prefix,
+                        '%s%s/' % (script_prefix, language),
+                        1
+                    )
+                )
+                return self.response_redirect_class(language_url)
+
+        if not (self.is_language_prefix_patterns_used()
+                and language_from_path):
+            patch_vary_headers(response, ('Accept-Language',))
+        if 'Content-Language' not in response:
+            response['Content-Language'] = language
+        return response
+
+    def is_language_prefix_patterns_used(self):
+        """
+        Returns `True` if the `LocaleRegexURLResolver` is used
+        at root level of the urlpatterns, else it returns `False`.
+        """
+        return self._is_language_prefix_patterns_used

common/djangoapps/django_locale/tests.py

+# pylint: disable=invalid-name, line-too-long, super-method-not-called
+"""
+Tests taken from Django upstream:
+https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py
+"""
+from django.conf import settings
+from django.test import TestCase, RequestFactory
+from django_locale.trans_real import (
+    parse_accept_lang_header, get_language_from_request, LANGUAGE_SESSION_KEY
+)
+
+# Added to test middleware around dark lang
+from django.contrib.auth.models import User
+from django.test.utils import override_settings
+from dark_lang.models import DarkLangConfig
+
+
+# Adding to support test differences between Django and our own settings
+@override_settings(LANGUAGES=[
+    ('pt', 'Portuguese'),
+    ('pt-br', 'Portuguese-Brasil'),
+    ('es', 'Spanish'),
+    ('es-ar', 'Spanish (Argentina)'),
+    ('de', 'Deutch'),
+    ('zh-cn', 'Chinese (China)'),
+    ('ar-sa', 'Arabic (Saudi Arabia)'),
+])
+class MiscTests(TestCase):
+    """
+    Tests taken from Django upstream:
+    https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py
+    """
+    def setUp(self):
+        self.rf = RequestFactory()
+        # Added to test middleware around dark lang
+        user = User()
+        user.save()
+        DarkLangConfig(
+            released_languages='pt, pt-br, es, de, es-ar, zh-cn, ar-sa',
+            changed_by=user,
+            enabled=True
+        ).save()
+
+    def test_parse_spec_http_header(self):
+        """
+        Testing HTTP header parsing. First, we test that we can parse the
+        values according to the spec (and that we extract all the pieces in
+        the right order).
+        """
+        p = parse_accept_lang_header
+        # Good headers.
+        self.assertEqual([('de', 1.0)], p('de'))
+        self.assertEqual([('en-AU', 1.0)], p('en-AU'))
+        self.assertEqual([('es-419', 1.0)], p('es-419'))
+        self.assertEqual([('*', 1.0)], p('*;q=1.00'))
+        self.assertEqual([('en-AU', 0.123)], p('en-AU;q=0.123'))
+        self.assertEqual([('en-au', 0.5)], p('en-au;q=0.5'))
+        self.assertEqual([('en-au', 1.0)], p('en-au;q=1.0'))
+        self.assertEqual([('da', 1.0), ('en', 0.5), ('en-gb', 0.25)], p('da, en-gb;q=0.25, en;q=0.5'))
+        self.assertEqual([('en-au-xx', 1.0)], p('en-au-xx'))
+        self.assertEqual([('de', 1.0), ('en-au', 0.75), ('en-us', 0.5), ('en', 0.25), ('es', 0.125), ('fa', 0.125)], p('de,en-au;q=0.75,en-us;q=0.5,en;q=0.25,es;q=0.125,fa;q=0.125'))
+        self.assertEqual([('*', 1.0)], p('*'))
+        self.assertEqual([('de', 1.0)], p('de;q=0.'))
+        self.assertEqual([('en', 1.0), ('*', 0.5)], p('en; q=1.0, * ; q=0.5'))
+        self.assertEqual([], p(''))
+
+        # Bad headers; should always return [].
+        self.assertEqual([], p('en-gb;q=1.0000'))
+        self.assertEqual([], p('en;q=0.1234'))
+        self.assertEqual([], p('en;q=.2'))
+        self.assertEqual([], p('abcdefghi-au'))
+        self.assertEqual([], p('**'))
+        self.assertEqual([], p('en,,gb'))
+        self.assertEqual([], p('en-au;q=0.1.0'))
+        self.assertEqual([], p('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXZ,en'))
+        self.assertEqual([], p('da, en-gb;q=0.8, en;q=0.7,#'))
+        self.assertEqual([], p('de;q=2.0'))
+        self.assertEqual([], p('de;q=0.a'))
+        self.assertEqual([], p('12-345'))
+        self.assertEqual([], p(''))
+
+    def test_parse_literal_http_header(self):
+        """
+        Now test that we parse a literal HTTP header correctly.
+        """
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-br'}
+        self.assertEqual('pt-br', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt'}
+        self.assertEqual('pt', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es,de'}
+        self.assertEqual('es', g(r))
+
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-ar,de'}
+        self.assertEqual('es-ar', g(r))
+
+        # This test assumes there won't be a Django translation to a US
+        # variation of the Spanish language, a safe assumption. When the
+        # user sets it as the preferred language, the main 'es'
+        # translation should be selected instead.
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-us'}
+        self.assertEqual(g(r), 'es')
+
+        # This tests the following scenario: there isn't a main language (zh)
+        # translation of Django but there is a translation to variation (zh_CN)
+        # the user sets zh-cn as the preferred language, it should be selected
+        # by Django without falling back nor ignoring it.
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-cn,de'}
+        self.assertEqual(g(r), 'zh-cn')
+
+    def test_logic_masked_by_darklang(self):
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'ar-qa'}
+        self.assertEqual('ar-sa', g(r))
+
+        r.session = {LANGUAGE_SESSION_KEY: 'es'}
+        self.assertEqual('es', g(r))
+
+    def test_parse_language_cookie(self):
+        """
+        Now test that we parse language preferences stored in a cookie correctly.
+        """
+        g = get_language_from_request
+        r = self.rf.get('/')
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt-br'}
+        r.META = {}
+        self.assertEqual('pt-br', g(r))
+
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt'}
+        r.META = {}
+        self.assertEqual('pt', g(r))
+
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es'}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
+        self.assertEqual('es', g(r))
+
+        # This test assumes there won't be a Django translation to a US
+        # variation of the Spanish language, a safe assumption. When the
+        # user sets it as the preferred language, the main 'es'
+        # translation should be selected instead.
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es-us'}
+        r.META = {}
+        self.assertEqual(g(r), 'es')
+
+        # This tests the following scenario: there isn't a main language (zh)
+        # translation of Django but there is a translation to variation (zh_CN)
+        # the user sets zh-cn as the preferred language, it should be selected
+        # by Django without falling back nor ignoring it.
+        r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'zh-cn'}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'}
+        self.assertEqual(g(r), 'zh-cn')

common/djangoapps/django_locale/trans_real.py

+"""Translation helper functions."""
+# Imported from Django 1.8
+# pylint: disable=invalid-name
+import re
+from django.conf import settings
+from django.conf.locale import LANG_INFO
+from django.utils import translation
+
+
+# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.
+# and RFC 3066, section 2.1
+accept_language_re = re.compile(r'''
+    ([A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*|\*)         # "en", "en-au", "x-y-z", "*"
+    (?:\s*;\s*q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
+    (?:\s*,\s*|$)                                 # Multiple accepts per header.
+    ''', re.VERBOSE)
+
+
+language_code_re = re.compile(r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*$', re.IGNORECASE)
+
+
+LANGUAGE_SESSION_KEY = '_language'
+
+
+def parse_accept_lang_header(lang_string):
+    """
+    Parses the lang_string, which is the body of an HTTP Accept-Language
+    header, and returns a list of (lang, q-value), ordered by 'q' values.
+
+    Any format errors in lang_string results in an empty list being returned.
+    """
+    # parse_accept_lang_header is broken until we are on Django 1.5 or greater
+    # See https://code.djangoproject.com/ticket/19381
+    result = []
+    pieces = accept_language_re.split(lang_string)
+    if pieces[-1]:
+        return []
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i: i + 3]
+        if first:
+            return []
+        priority = priority and float(priority) or 1.0
+        result.append((lang, priority))
+    result.sort(key=lambda k: k[1], reverse=True)
+    return result
+
+
+def get_supported_language_variant(lang_code, strict=False):
+    """
+    Returns the language-code that's listed in supported languages, possibly
+    selecting a more generic variant. Raises LookupError if nothing found.
+    If `strict` is False (the default), the function will look for an alternative
+    country-specific variant when the currently checked is not found.
+    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
+    as the provided language codes are taken from the HTTP request. See also
+    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
+    """
+    if lang_code:
+        # If 'fr-ca' is not supported, try special fallback or language-only 'fr'.
+        possible_lang_codes = [lang_code]
+        try:
+            # TODO skip this, or import updated LANG_INFO format from __future__
+            # (fallback option wasn't added until
+            # https://github.com/django/django/commit/5dcdbe95c749d36072f527e120a8cb463199ae0d)
+            possible_lang_codes.extend(LANG_INFO[lang_code]['fallback'])
+        except KeyError:
+            pass
+        generic_lang_code = lang_code.split('-')[0]
+        possible_lang_codes.append(generic_lang_code)
+        supported_lang_codes = dict(settings.LANGUAGES)
+
+        for code in possible_lang_codes:
+            # Note: django 1.4 implementation of check_for_language is OK to use
+            if code in supported_lang_codes and translation.check_for_language(code):
+                return code
+        if not strict:
+            # if fr-fr is not supported, try fr-ca.
+            for supported_code in supported_lang_codes:
+                if supported_code.startswith(generic_lang_code + '-'):
+                    return supported_code
+    raise LookupError(lang_code)
+
+
+def get_language_from_request(request, check_path=False):
+    """
+    Analyzes the request to find what language the user wants the system to
+    show. Only languages listed in settings.LANGUAGES are taken into account.
+    If the user requests a sublanguage where we have a main language, we send
+    out the main language.
+    If check_path is True, the URL path prefix will be checked for a language
+    code, otherwise this is skipped for backwards compatibility.
+    """
+    if check_path:
+        # Note: django 1.4 implementation of get_language_from_path is OK to use
+        lang_code = translation.get_language_from_path(request.path_info)
+        if lang_code is not None:
+            return lang_code
+
+    supported_lang_codes = dict(settings.LANGUAGES)
+
+    if hasattr(request, 'session'):
+        lang_code = request.session.get(LANGUAGE_SESSION_KEY)
+        # Note: django 1.4 implementation of check_for_language is OK to use
+        if lang_code in supported_lang_codes and lang_code is not None and translation.check_for_language(lang_code):
+            return lang_code
+
+    lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)
+
+    try:
+        return get_supported_language_variant(lang_code)
+    except LookupError:
+        pass
+
+    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
+    # broken in 1.4, so defined above
+    for accept_lang, unused in parse_accept_lang_header(accept):
+        if accept_lang == '*':
+            break
+
+        if not language_code_re.search(accept_lang):
+            continue
+
+        try:
+            return get_supported_language_variant(accept_lang)
+        except LookupError:
+            continue
+
+    try:
+        return get_supported_language_variant(settings.LANGUAGE_CODE)
+    except LookupError:
+        return settings.LANGUAGE_CODE

common/djangoapps/lang_pref/middleware.py

@@ -4,6 +4,9 @@ Middleware for Language Preferences
 
 from openedx.core.djangoapps.user_api.preferences.api import get_user_preference
 from lang_pref import LANGUAGE_KEY
+# TODO PLAT-671 Import from Django 1.8
+# from django.utils.translation import LANGUAGE_SESSION_KEY
+from django_locale.trans_real import LANGUAGE_SESSION_KEY
 
 
 class LanguagePreferenceMiddleware(object):
@@ -16,10 +19,12 @@ class LanguagePreferenceMiddleware(object):
 
     def process_request(self, request):
         """
-        If a user's UserPreference contains a language preference and there is
-        no language set on the session (i.e. from dark language overrides), use the user's preference.
+        If a user's UserPreference contains a language preference, use the user's preference.
         """
-        if request.user.is_authenticated() and 'django_language' not in request.session:
+        # If the user is logged in, check for their language preference
+        if request.user.is_authenticated():
+            # Get the user's language preference
             user_pref = get_user_preference(request.user, LANGUAGE_KEY)
+            # Set it to the LANGUAGE_SESSION_KEY (Django-specific session setting governing language pref)
             if user_pref:
-                request.session['django_language'] = user_pref
+                request.session[LANGUAGE_SESSION_KEY] = user_pref

common/djangoapps/lang_pref/tests/test_middleware.py

 from django.test import TestCase
 from django.test.client import RequestFactory
 from django.contrib.sessions.middleware import SessionMiddleware
+# TODO PLAT-671 Import from Django 1.8
+# from django.utils.translation import LANGUAGE_SESSION_KEY
+from django_locale.trans_real import LANGUAGE_SESSION_KEY
 
 from lang_pref.middleware import LanguagePreferenceMiddleware
 from openedx.core.djangoapps.user_api.preferences.api import set_user_preference
@@ -25,19 +28,23 @@ class TestUserPreferenceMiddleware(TestCase):
     def test_no_language_set_in_session_or_prefs(self):
         # nothing set in the session or the prefs
         self.middleware.process_request(self.request)
-        self.assertNotIn('django_language', self.request.session)
+        self.assertNotIn(LANGUAGE_SESSION_KEY, self.request.session)
 
     def test_language_in_user_prefs(self):
         # language set in the user preferences and not the session
         set_user_preference(self.user, LANGUAGE_KEY, 'eo')
         self.middleware.process_request(self.request)
-        self.assertEquals(self.request.session['django_language'], 'eo')
+        self.assertEquals(self.request.session[LANGUAGE_SESSION_KEY], 'eo')
 
     def test_language_in_session(self):
         # language set in both the user preferences and session,
-        # session should get precedence
-        self.request.session['django_language'] = 'en'
+        # preference should get precedence. The session will hold the last value,
+        # which is probably the user's last preference. Look up the updated preference.
+
+        # Dark lang middleware should run after this middleware, so it can
+        # set a session language as an override of the user's preference.
+        self.request.session[LANGUAGE_SESSION_KEY] = 'en'
         set_user_preference(self.user, LANGUAGE_KEY, 'eo')
         self.middleware.process_request(self.request)
 
-        self.assertEquals(self.request.session['django_language'], 'en')
+        self.assertEquals(self.request.session[LANGUAGE_SESSION_KEY], 'eo')

lms/djangoapps/courseware/tests/test_i18n.py

@@ -4,37 +4,65 @@ Tests i18n in courseware
 import re
 from nose.plugins.attrib import attr
 
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse, NoReverseMatch
 from django.test import TestCase
-from django.test.utils import override_settings
+from django.test.client import Client
 
+from dark_lang.models import DarkLangConfig
+from lang_pref import LANGUAGE_KEY
+from openedx.core.djangoapps.user_api.preferences.api import set_user_preference
+from student.tests.factories import UserFactory, RegistrationFactory, UserProfileFactory
 
-@attr('shard_1')
-@override_settings(LANGUAGES=[('eo', 'Esperanto'), ('ar', 'Arabic')])
-class I18nTestCase(TestCase):
+
+class BaseI18nTestCase(TestCase):
     """
-    Tests for i18n
+    Base utilities for i18n test classes to derive from
     """
     def assert_tag_has_attr(self, content, tag, attname, value):
         """Assert that a tag in `content` has a certain value in a certain attribute."""
-        regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d ]+)['"][^>]*>""".format(tag=tag, attname=attname)
+        regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d\- ]+)['"][^>]*>""".format(tag=tag, attname=attname)
         match = re.search(regex, content)
-        self.assertTrue(match, "Couldn't find desired tag in %r" % content)
+        self.assertTrue(match, "Couldn't find desired tag '%s' with attr '%s' in %r" % (tag, attname, content))
         attvalues = match.group(1).split()
         self.assertIn(value, attvalues)
 
+    def release_languages(self, languages):
+        """
+        Release a set of languages using the dark lang interface.
+        languages is a list of comma-separated lang codes, eg, 'ar, es-419'
+        """
+        user = User()
+        user.save()
+        DarkLangConfig(
+            released_languages=languages,
+            changed_by=user,
+            enabled=True
+        ).save()
+
+
+@attr('shard_1')
+class I18nTestCase(BaseI18nTestCase):
+    """
+    Tests for i18n
+    """
     def test_default_is_en(self):
+        self.release_languages('fr')
         response = self.client.get('/')
         self.assert_tag_has_attr(response.content, "html", "lang", "en")
         self.assertEqual(response['Content-Language'], 'en')
         self.assert_tag_has_attr(response.content, "body", "class", "lang_en")
 
     def test_esperanto(self):
+        self.release_languages('fr, eo')
         response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='eo')
         self.assert_tag_has_attr(response.content, "html", "lang", "eo")
         self.assertEqual(response['Content-Language'], 'eo')
         self.assert_tag_has_attr(response.content, "body", "class", "lang_eo")
 
     def test_switching_languages_bidi(self):
+        self.release_languages('ar, eo')
         response = self.client.get('/')
         self.assert_tag_has_attr(response.content, "html", "lang", "en")
         self.assertEqual(response['Content-Language'], 'en')
@@ -46,3 +74,122 @@ class I18nTestCase(TestCase):
         self.assertEqual(response['Content-Language'], 'ar')
         self.assert_tag_has_attr(response.content, "body", "class", "lang_ar")
         self.assert_tag_has_attr(response.content, "body", "class", "rtl")
+
+
+@attr('shard_1')
+class I18nRegressionTests(BaseI18nTestCase):
+    """
+    Tests for i18n
+    """
+    def test_es419_acceptance(self):
+        # Regression test; LOC-72, and an issue with Django
+        self.release_languages('es-419')
+        response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='es-419')
+        self.assert_tag_has_attr(response.content, "html", "lang", "es-419")
+
+    def test_unreleased_lang_resolution(self):
+        # Regression test; LOC-85
+        self.release_languages('fa')
+
+        # We've released 'fa', AND we have language files for 'fa-ir' but
+        # we want to keep 'fa-ir' as a dark language. Requesting 'fa-ir'
+        # in the http request (NOT with the ?preview-lang query param) should
+        # receive files for 'fa'
+        response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='fa-ir')
+        self.assert_tag_has_attr(response.content, "html", "lang", "fa")
+
+        # Now try to access with dark lang
+        response = self.client.get('/?preview-lang=fa-ir')
+        self.assert_tag_has_attr(response.content, "html", "lang", "fa-ir")
+
+    def test_preview_lang(self):
+        # Regression test; LOC-87
+        self.release_languages('es-419')
+        site_lang = settings.LANGUAGE_CODE
+        # Visit the front page; verify we see site default lang
+        response = self.client.get('/')
+        self.assert_tag_has_attr(response.content, "html", "lang", site_lang)
+
+        # Verify we can switch language using the preview-lang query param
+        response = self.client.get('/?preview-lang=eo')
+        self.assert_tag_has_attr(response.content, "html", "lang", "eo")
+        # We should be able to see released languages using preview-lang, too
+        response = self.client.get('/?preview-lang=es-419')
+        self.assert_tag_has_attr(response.content, "html", "lang", "es-419")
+
+        # Clearing the language should go back to site default
+        response = self.client.get('/?clear-lang')
+        self.assert_tag_has_attr(response.content, "html", "lang", site_lang)
+
+
+@attr('shard_1')
+class I18nLangPrefTests(BaseI18nTestCase):
+    """
+    Regression tests of language presented to the user, when they
+    choose a language preference, and when they have a preference
+    and use the dark lang preview functionality.
+    """
+    def setUp(self):
+        super(I18nLangPrefTests, self).setUp()
+        # Create one user and save it to the database
+        email = 'test@edx.org'
+        pwd = 'test_password'
+        self.user = UserFactory.build(username='test', email=email)
+        self.user.set_password(pwd)
+        self.user.save()
+
+        # Create a registration for the user
+        RegistrationFactory(user=self.user)
+
+        # Create a profile for the user
+        UserProfileFactory(user=self.user)
+
+        # Create the test client
+        self.client = Client()
+
+        # Get the login url & log in our user
+        try:
+            login_url = reverse('login_post')
+        except NoReverseMatch:
+            login_url = reverse('login')
+        self.client.post(login_url, {'email': email, 'password': pwd})
+
+        # Url and site lang vars for tests to use
+        self.url = reverse('dashboard')
+        self.site_lang = settings.LANGUAGE_CODE
+
+    def test_lang_preference(self):
+        # Regression test; LOC-87
+        self.release_languages('ar, es-419')
+
+        # Visit the front page; verify we see site default lang
+        response = self.client.get(self.url)
+        self.assert_tag_has_attr(response.content, "html", "lang", self.site_lang)
+
+        # Set user language preference
+        set_user_preference(self.user, LANGUAGE_KEY, 'ar')
+        # and verify we now get an ar response
+        response = self.client.get(self.url)
+        self.assert_tag_has_attr(response.content, "html", "lang", 'ar')
+
+        # Verify that switching language preference gives the right language
+        set_user_preference(self.user, LANGUAGE_KEY, 'es-419')
+        response = self.client.get(self.url)
+        self.assert_tag_has_attr(response.content, "html", "lang", 'es-419')
+
+    def test_preview_precedence(self):
+        # Regression test; LOC-87
+        self.release_languages('ar, es-419')
+
+        # Set user language preference
+        set_user_preference(self.user, LANGUAGE_KEY, 'ar')
+        # Verify preview-lang takes precedence
+        response = self.client.get('{}?preview-lang=eo'.format(self.url))
+        self.assert_tag_has_attr(response.content, "html", "lang", 'eo')
+        # Hitting another page should keep the dark language set.
+        response = self.client.get(reverse('courses'))
+        self.assert_tag_has_attr(response.content, "html", "lang", "eo")
+
+        # Clearing language must set language back to preference language
+        response = self.client.get('{}?clear-lang'.format(self.url))
+        self.assert_tag_has_attr(response.content, "html", "lang", 'ar')

lms/envs/common.py

@@ -1142,17 +1142,23 @@ MIDDLEWARE_CLASSES = (
 
     'splash.middleware.SplashMiddleware',
 
-    # Allows us to dark-launch particular languages
-    'dark_lang.middleware.DarkLangMiddleware',
+
     'geoinfo.middleware.CountryMiddleware',
     'embargo.middleware.EmbargoMiddleware',
 
     # Allows us to set user preferences
-    # should be after DarkLangMiddleware
     'lang_pref.middleware.LanguagePreferenceMiddleware',
 
-    # Detects user-requested locale from 'accept-language' header in http request
-    'django.middleware.locale.LocaleMiddleware',
+    # Allows us to dark-launch particular languages.
+    # Must be after LangPrefMiddleware, so ?preview-lang query params can override
+    # user's language preference. ?clear-lang resets to user's language preference.
+    'dark_lang.middleware.DarkLangMiddleware',
+
+    # Detects user-requested locale from 'accept-language' header in http request.
+    # Must be after DarkLangMiddleware.
+    # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671]
+    # 'django.middleware.locale.LocaleMiddleware',
+    'django_locale.middleware.LocaleMiddleware',
 
     'django.middleware.transaction.TransactionMiddleware',
     # 'debug_toolbar.middleware.DebugToolbarMiddleware',