mattdrayer/xblock-translations: Add I18N/L10N support to XBlocks via the runtime

* mattdrayer:  Latest proto code
* mattdrayer:  Add translation.py
* asadiqbal08: Xblock translation ugettext update, remove translation.py
* mattdrayer:  Additional I18N work -- starting to see some translations!
* asadiqbal08: Trying to make xBlock message catalog files path dynamic
* mattdrayer:  Remove unnecessary modifications
* mattdrayer:  Cleaned up implementation
* mattdrayer:  Moved import statement
* asadiqbal08: update as suggested
* asadiqbal08: xblock its own domain
* asadiqbal08: translation: secure none object
* asadiqbal08: pylint
* asadiqbal08: get locale from xblock
* asadiqbal08: update
* mattdrayer:  Determine XBlock locale path within runtime service
* mattdrayer:  Determine module location via the runtime
* mattdrayer:  Remove ModuleI18nService reference
* asadiqbal08: override the service in studio
* asadiqbal08: remove import
* asadiqbal08: update the Modulei18nService
* asadiqbal08: update the Modulei18nService
* mattdrayer:  Remove redundant __class__ reference
* asadiqbal08: update the docstring
* asadiqbal08: tests
* mattdrayer:  Remove specific ugettext override from ModuleI18nService
* mattdrayer:  Move service operation to base class
* mattdrayer:  Address quality violations
* asadiqbal08: Investigating the test failure issue on jenkins and solved
* asadiqbal08: First utilizing the parent class method
* mattdrayer:  Use recommended callable approach
* asadiqbal08: remove unused code
* asadiqbal08: Updated the test to use cms preview module system runtime in order to get i18n service.
* asadiqbal08: Pylint quality
* asadiqbal08: update the service call to check xblock declarations
* asadiqbal08: update doc string
* asadiqbal08: i18n callable test in studio
* asadiqbal08: test lms runtime for modulei18n service
* asadiqbal08: add doc strings
* asadiqbal08: Rename locale and domain to Flask-Babel convention

cms/djangoapps/contentstore/tests/test_i18n.py

+"""
+Tests for validate Internationalization and Module i18n service.
+"""
+import mock
+import gettext
 from unittest import skip
-
 from django.contrib.auth.models import User
-
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from contentstore.tests.utils import AjaxEnabledTestClient
+from xmodule.modulestore.django import ModuleI18nService
+from django.utils import translation
+from django.utils.translation import get_language
+from django.conf import settings
+from xmodule.modulestore.tests.factories import ItemFactory, CourseFactory
+from contentstore.views.preview import _preview_module_system
+
+
+class FakeTranslations(ModuleI18nService):
+    """A test GNUTranslations class that takes a map of msg -> translations."""
+
+    def __init__(self, translations):  # pylint: disable=super-init-not-called
+        self.translations = translations
+
+    def ugettext(self, msgid):
+        """
+        Mock override for ugettext translation operation
+        """
+        return self.translations.get(msgid, msgid)
+
+    @staticmethod
+    def translator(locales_map):  # pylint: disable=method-hidden
+        """Build mock translator for the given locales.
+        Returns a mock gettext.translation function that uses
+        individual TestTranslations to translate in the given locales.
+        :param locales_map: A map from locale name to a translations map.
+                            {
+                             'es': {'Hi': 'Hola', 'Bye': 'Adios'},
+                             'zh': {'Hi': 'Ni Hao', 'Bye': 'Zaijian'}
+                            }
+        """
+        def _translation(domain, localedir=None, languages=None):  # pylint: disable=unused-argument
+            """
+            return gettext.translation for given language
+            """
+            if languages:
+                language = languages[0]
+                if language in locales_map:
+                    return FakeTranslations(locales_map[language])
+            return gettext.NullTranslations()
+        return _translation
+
+
+class TestModuleI18nService(ModuleStoreTestCase):
+    """ Test ModuleI18nService """
+
+    def setUp(self):
+        """ Setting up tests """
+        super(TestModuleI18nService, self).setUp()
+        self.test_language = 'dummy language'
+        self.request = mock.Mock()
+        self.course = CourseFactory.create()
+        self.field_data = mock.Mock()
+        self.descriptor = ItemFactory(category="pure", parent=self.course)
+        self.runtime = _preview_module_system(
+            self.request,
+            self.descriptor,
+            self.field_data,
+        )
+        self.addCleanup(translation.activate, settings.LANGUAGE_CODE)
+
+    def get_module_i18n_service(self, descriptor):
+        """
+        return the module i18n service.
+        """
+        i18n_service = self.runtime.service(descriptor, 'i18n')
+        self.assertIsNotNone(i18n_service)
+        self.assertIsInstance(i18n_service, ModuleI18nService)
+        return i18n_service
+
+    def test_django_service_translation_works(self):
+        """
+        Test django translation service works fine.
+        """
+
+        def wrap_with_xyz(func):
+            """
+            A decorator function that just adds 'XYZ ' to the front of all strings
+            """
+            def new_func(*args, **kwargs):
+                """ custom function """
+                output = func(*args, **kwargs)
+                return "XYZ " + output
+            return new_func
+
+        old_lang = translation.get_language()
+
+        i18n_service = self.get_module_i18n_service(self.descriptor)
+
+        # Activate french, so that if the fr files haven't been loaded, they will be loaded now.
+        translation.activate("fr")
+        french_translation = translation.trans_real._active.value  # pylint: disable=protected-access
+
+        # wrap the ugettext functions so that 'TEST ' will prefix each translation
+        french_translation.ugettext = wrap_with_xyz(french_translation.ugettext)
+        self.assertEqual(i18n_service.ugettext(self.test_language), 'XYZ dummy language')
+
+        # Turn back on our old translations
+        translation.activate(old_lang)
+        del old_lang
+        self.assertEqual(i18n_service.ugettext(self.test_language), 'dummy language')
+
+    @mock.patch('django.utils.translation.ugettext', mock.Mock(return_value='XYZ-TEST-LANGUAGE'))
+    def test_django_translator_in_use_with_empty_block(self):
+        """
+        Test: Django default translator should in use if we have an empty block
+        """
+        i18n_service = ModuleI18nService(None)
+        self.assertEqual(i18n_service.ugettext(self.test_language), 'XYZ-TEST-LANGUAGE')
+
+    @mock.patch('django.utils.translation.ugettext', mock.Mock(return_value='XYZ-TEST-LANGUAGE'))
+    def test_message_catalog_translations(self):
+        """
+        Test: Message catalog from FakeTranslation should return required translations.
+        """
+        _translator = FakeTranslations.translator(
+            {
+                'es': {'Hello': 'es-hello-world'},
+                'fr': {'Hello': 'fr-hello-world'},
+            },
+        )
+        localedir = '/translations'
+        translation.activate("es")
+        with mock.patch('gettext.translation', return_value=_translator(domain='text', localedir=localedir,
+                                                                        languages=[get_language()])):
+            i18n_service = self.get_module_i18n_service(self.descriptor)
+            self.assertEqual(i18n_service.ugettext('Hello'), 'es-hello-world')
+
+        translation.activate("ar")
+        with mock.patch('gettext.translation', return_value=_translator(domain='text', localedir=localedir,
+                                                                        languages=[get_language()])):
+            i18n_service = self.get_module_i18n_service(self.descriptor)
+            self.assertEqual(i18n_service.ugettext('Hello'), 'Hello')
+            self.assertNotEqual(i18n_service.ugettext('Hello'), 'fr-hello-world')
+            self.assertNotEqual(i18n_service.ugettext('Hello'), 'es-hello-world')
+
+        translation.activate("fr")
+        with mock.patch('gettext.translation', return_value=_translator(domain='text', localedir=localedir,
+                                                                        languages=[get_language()])):
+            i18n_service = self.get_module_i18n_service(self.descriptor)
+            self.assertEqual(i18n_service.ugettext('Hello'), 'fr-hello-world')
+
+    def test_i18n_service_callable(self):
+        """
+        Test: i18n service should be callable in studio.
+        """
+        self.assertTrue(callable(self.runtime._services.get('i18n')))  # pylint: disable=protected-access
 
 
 class InternationalizationTest(ModuleStoreTestCase):

cms/djangoapps/contentstore/views/preview.py

@@ -104,6 +104,9 @@ class PreviewModuleSystem(ModuleSystem):  # pylint: disable=abstract-method
     # they are being rendered for preview (i.e. in Studio)
     is_author_mode = True
 
+    def __init__(self, **kwargs):
+        super(PreviewModuleSystem, self).__init__(**kwargs)
+
     def handler_url(self, block, handler_name, suffix='', query='', thirdparty=False):
         return reverse('preview_handler', kwargs={
             'usage_key_string': unicode(block.scope_ids.usage_id),
@@ -236,8 +239,8 @@ def _preview_module_system(request, descriptor, field_data):
         # Get the raw DescriptorSystem, not the CombinedSystem
         descriptor_runtime=descriptor._runtime,  # pylint: disable=protected-access
         services={
-            "i18n": ModuleI18nService(),
             "field-data": field_data,
+            "i18n": ModuleI18nService,
             "library_tools": LibraryToolsService(modulestore()),
             "settings": SettingsService(),
             "user": DjangoXBlockUserService(request.user),

common/lib/xmodule/xmodule/modulestore/django.py

@@ -7,9 +7,11 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
 from __future__ import absolute_import
 
 from importlib import import_module
+import gettext
 import logging
-
+from pkg_resources import resource_filename
 import re
+
 from django.conf import settings
 
 # This configuration must be executed BEFORE any additional Django imports. Otherwise, the imports may fail due to
@@ -20,6 +22,7 @@ if not settings.configured:
 from django.core.cache import caches, InvalidCacheBackendError
 import django.dispatch
 import django.utils
+from django.utils.translation import get_language, to_locale
 
 from pymongo import ReadPreference
 from xmodule.contentstore.django import contentstore
@@ -28,7 +31,6 @@ from xmodule.modulestore.mixed import MixedModuleStore
 from xmodule.util.django import get_current_request_hostname
 import xblock.reference.plugins
 
-
 try:
     # We may not always have the request_cache module available
     from request_cache.middleware import RequestCache
@@ -243,9 +245,35 @@ class ModuleI18nService(object):
     i18n service.
 
     """
+    def __init__(self, block=None):
+        """
+        Attempt to load an XBlock-specific GNU gettext translator using the XBlock's own domain
+        translation catalog, currently expected to be found at:
+            <xblock_root>/conf/locale/<language>/LC_MESSAGES/<domain>.po|mo
+        If we can't locate the domain translation catalog then we fall-back onto
+        django.utils.translation, which will point to the system's own domain translation catalog
+        This effectively achieves translations by coincidence for an XBlock which does not provide
+        its own dedicated translation catalog along with its implementation.
+        """
+        self.translator = django.utils.translation
+        if block:
+            xblock_resource = block.unmixed_class.__module__
+            xblock_locale_dir = '/translations'
+            xblock_locale_path = resource_filename(xblock_resource, xblock_locale_dir)
+            xblock_domain = 'text'
+            selected_language = get_language()
+            try:
+                self.translator = gettext.translation(
+                    xblock_domain,
+                    xblock_locale_path,
+                    [to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)]
+                )
+            except IOError:
+                # Fall back to the default Django translator if the XBlock translator is not found.
+                pass
 
     def __getattr__(self, name):
-        return getattr(django.utils.translation, name)
+        return getattr(self.translator, name)
 
     def strftime(self, *args, **kwargs):
         """

common/lib/xmodule/xmodule/x_module.py

@@ -24,6 +24,7 @@ from xblock.fields import (
     String, Dict, ScopeIds, Reference, ReferenceList,
     ReferenceValueDict, UserScope
 )
+
 from xblock.fragment import Fragment
 from xblock.runtime import Runtime, IdReader, IdGenerator
 from xmodule import course_metadata_utils
@@ -1776,6 +1777,28 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):
     def publish(self, block, event_type, event):
         pass
 
+    def service(self, block, service_name):
+        """
+        Runtime-specific override for the XBlock service manager.  If a service is not currently
+        instantiated and is declared as a critical requirement, an attempt is made to load the
+        module.
+
+        Arguments:
+            block (an XBlock): this block's class will be examined for service
+                decorators.
+            service_name (string): the name of the service requested.
+
+        Returns:
+            An object implementing the requested service, or None.
+        """
+        # getting the service from parent module. making sure of block service declarations.
+        service = super(ModuleSystem, self).service(block=block, service_name=service_name)
+        # Passing the block to service if it is callable e.g. ModuleI18nService. It is the responsibility of calling
+        # service to handle the passing argument.
+        if callable(service):
+            return service(block)
+        return service
+
 
 class CombinedSystem(object):
     """

lms/djangoapps/courseware/module_render.py

@@ -75,7 +75,7 @@ from xmodule.error_module import ErrorDescriptor, NonStaffErrorDescriptor
 from xmodule.exceptions import NotFoundError, ProcessingError
 from xmodule.lti_module import LTIModule
 from xmodule.mixin import wrap_with_license
-from xmodule.modulestore.django import modulestore, ModuleI18nService
+from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.x_module import XModuleDescriptor
 from .field_overrides import OverrideFieldData
@@ -713,7 +713,6 @@ def get_module_system_for_user(user, student_data,  # TODO  # pylint: disable=to
         wrappers=block_wrappers,
         get_real_user=user_by_anonymous_id,
         services={
-            'i18n': ModuleI18nService(),
             'fs': FSService(),
             'field-data': field_data,
             'user': DjangoXBlockUserService(user, user_is_staff=user_is_staff),

lms/djangoapps/lms_xblock/runtime.py

 """
 Module implementing `xblock.runtime.Runtime` functionality for the LMS
 """
-
 import re
-import xblock.reference.plugins
 
 from django.core.urlresolvers import reverse
 from django.conf import settings
-from request_cache.middleware import RequestCache
-from lms.djangoapps.lms_xblock.models import XBlockAsidesConfig
+
 from openedx.core.djangoapps.user_api.course_tag import api as user_course_tag_api
-from xmodule.modulestore.django import modulestore
-from xmodule.services import SettingsService
+from request_cache.middleware import RequestCache
+import xblock.reference.plugins
 from xmodule.library_tools import LibraryToolsService
-from xmodule.x_module import ModuleSystem
+from xmodule.modulestore.django import modulestore, ModuleI18nService
 from xmodule.partitions.partitions_service import PartitionService
+from xmodule.services import SettingsService
+from xmodule.x_module import ModuleSystem
+
+from lms.djangoapps.lms_xblock.models import XBlockAsidesConfig
 
 
 def _quote_slashes(match):
@@ -201,16 +202,17 @@ class LmsModuleSystem(ModuleSystem):  # pylint: disable=abstract-method
     def __init__(self, **kwargs):
         request_cache_dict = RequestCache.get_request_cache().data
         services = kwargs.setdefault('services', {})
-        services['user_tags'] = UserTagsService(self)
+        services['fs'] = xblock.reference.plugins.FSService()
+        services['i18n'] = ModuleI18nService
+        services['library_tools'] = LibraryToolsService(modulestore())
         services['partitions'] = LmsPartitionService(
             user=kwargs.get('user'),
             course_id=kwargs.get('course_id'),
             track_function=kwargs.get('track_function', None),
             cache=request_cache_dict
         )
-        services['library_tools'] = LibraryToolsService(modulestore())
-        services['fs'] = xblock.reference.plugins.FSService()
         services['settings'] = SettingsService()
+        services['user_tags'] = UserTagsService(self)
         self.request_token = kwargs.pop('request_token', None)
         super(LmsModuleSystem, self).__init__(**kwargs)
 

lms/djangoapps/lms_xblock/test/test_runtime.py

@@ -11,6 +11,10 @@ from urlparse import urlparse
 from opaque_keys.edx.locations import BlockUsageLocator, CourseLocator, SlashSeparatedCourseKey
 from lms.djangoapps.lms_xblock.runtime import quote_slashes, unquote_slashes, LmsModuleSystem
 from xblock.fields import ScopeIds
+from xmodule.modulestore.django import ModuleI18nService
+from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xblock.exceptions import NoSuchServiceError
 
 TEST_STRINGS = [
     '',
@@ -181,3 +185,61 @@ class TestUserServiceAPI(TestCase):
         # Try to get tag in wrong scope
         with self.assertRaises(ValueError):
             self.runtime.service(self.mock_block, 'user_tags').get_tag('fake_scope', self.key)
+
+
+class TestI18nService(ModuleStoreTestCase):
+    """ Test ModuleI18nService """
+
+    def setUp(self):
+        """ Setting up tests """
+        super(TestI18nService, self).setUp()
+        self.course = CourseFactory.create()
+        self.test_language = 'dummy language'
+        self.runtime = LmsModuleSystem(
+            static_url='/static',
+            track_function=Mock(),
+            get_module=Mock(),
+            render_template=Mock(),
+            replace_urls=str,
+            course_id=self.course.id,
+            descriptor_runtime=Mock(),
+        )
+
+        self.mock_block = Mock()
+        self.mock_block.service_declaration.return_value = 'need'
+
+    def test_module_i18n_lms_service(self):
+        """
+        Test: module i18n service in LMS
+        """
+        i18n_service = self.runtime.service(self.mock_block, 'i18n')
+        self.assertIsNotNone(i18n_service)
+        self.assertIsInstance(i18n_service, ModuleI18nService)
+
+    def test_no_service_exception_with_none_declaration_(self):
+        """
+        Test: NoSuchServiceError should be raised block declaration returns none
+        """
+        self.mock_block.service_declaration.return_value = None
+        with self.assertRaises(NoSuchServiceError):
+            self.runtime.service(self.mock_block, 'i18n')
+
+    def test_no_service_exception_(self):
+        """
+        Test: NoSuchServiceError should be raised if i18n service is none.
+        """
+        self.runtime._services['i18n'] = None  # pylint: disable=protected-access
+        with self.assertRaises(NoSuchServiceError):
+            self.runtime.service(self.mock_block, 'i18n')
+
+    def test_i18n_service_callable(self):
+        """
+        Test: _services dict should contain the callable i18n service in LMS.
+        """
+        self.assertTrue(callable(self.runtime._services.get('i18n')))  # pylint: disable=protected-access
+
+    def test_i18n_service_not_callable(self):
+        """
+        Test: i18n service should not be callable in LMS after initialization.
+        """
+        self.assertFalse(callable(self.runtime.service(self.mock_block, 'i18n')))