Commit 7315855d by Christina Roberts

Merge pull request #1868 from MITx/feature/i18n

Feature/i18n
parents 57427f9a dd71ae81
......@@ -31,3 +31,13 @@ cover_html/
chromedriver.log
/nbproject
ghostdriver.log
/cms/doc/en/getting_started/
/conf/locale/en
/conf/locale/fr
create-dev-env.hack.sh
distribute-0.6.36.tar.gz
i18n/googleTranslate.hack.py
i18n/mitx/conf/locale/fr/LC_MESSAGES/django.po
i18n/split.py
.gitignore
from unittest import skip
from django.core.urlresolvers import reverse
from django.contrib.auth.models import User
from django.test.client import Client
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
class InternationalizationTest(ModuleStoreTestCase):
"""
Tests to validate Internationalization.
"""
def setUp(self):
"""
These tests need a user in the DB so that the django Test Client
can log them in.
They inherit from the ModuleStoreTestCase class so that the mongodb collection
will be cleared out before each test case execution and deleted
afterwards.
"""
self.uname = 'testuser'
self.email = 'test+courses@edx.org'
self.password = 'foo'
# Create the use so we can log them in.
self.user = User.objects.create_user(self.uname, self.email, self.password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
self.user.is_active = True
# Staff has access to view all courses
self.user.is_staff = True
self.user.save()
self.course_data = {
'template': 'i4x://edx/templates/course/Empty',
'org': 'MITx',
'number': '999',
'display_name': 'Robot Super Course',
}
def test_course_plain_english(self):
"""Test viewing the index page with no courses"""
self.client = Client()
self.client.login(username=self.uname, password=self.password)
resp = self.client.get(reverse('index'))
self.assertContains(resp,
'<h1 class="title-1">My Courses</h1>',
status_code=200,
html=True)
def test_course_explicit_english(self):
"""Test viewing the index page with no courses"""
self.client = Client()
self.client.login(username=self.uname, password=self.password)
resp = self.client.get(reverse('index'),
{},
HTTP_ACCEPT_LANGUAGE='en'
)
self.assertContains(resp,
'<h1 class="title-1">My Courses</h1>',
status_code=200,
html=True)
# ****
# NOTE:
# ****
#
# This test will break when we replace this fake 'test' language
# with actual French. This test will need to be updated with
# actual French at that time.
# Test temporarily disable since it depends on creation of dummy strings
@skip
def test_course_with_accents (self):
"""Test viewing the index page with no courses"""
self.client = Client()
self.client.login(username=self.uname, password=self.password)
resp = self.client.get(reverse('index'),
{},
HTTP_ACCEPT_LANGUAGE='fr'
)
TEST_STRING = u'<h1 class="title-1">' \
+ u'My \xc7\xf6\xfcrs\xe9s L#' \
+ u'</h1>'
self.assertContains(resp,
TEST_STRING,
status_code=200,
html=True)
......@@ -129,6 +129,9 @@ MIDDLEWARE_CLASSES = (
'track.middleware.TrackMiddleware',
'mitxmako.middleware.MakoMiddleware',
# Detects user-requested locale from 'accept-language' header in http request
'django.middleware.locale.LocaleMiddleware',
'django.middleware.transaction.TransactionMiddleware'
)
......@@ -173,9 +176,13 @@ STATICFILES_DIRS = [
# Locale/Internationalization
TIME_ZONE = 'America/New_York' # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
LANGUAGE_CODE = 'en' # http://www.i18nguy.com/unicode/language-identifiers.html
USE_I18N = True
USE_L10N = True
# Localization strings (e.g. django.po) are under this directory
LOCALE_PATHS = (REPO_ROOT + '/conf/locale',) # mitx/conf/locale/
# Tracking
TRACK_MAX_EVENT = 10000
......
......@@ -245,7 +245,7 @@ function showImportSubmit(e) {
$('.submit-button').show();
$('.progress').show();
} else {
$('.error-block').html('File format not supported. Please upload a file with a <code>tar.gz</code> extension.').show();
$('.error-block').html(gettext('File format not supported. Please upload a file with a <code>tar.gz</code> extension.')).show();
}
}
......@@ -406,7 +406,7 @@ function showFileSelectionMenu(e) {
}
function startUpload(e) {
$('.upload-modal h1').html('Uploading…');
$('.upload-modal h1').html(gettext('Uploading…'));
$('.upload-modal .file-name').html($('.file-input').val().replace('C:\\fakepath\\', ''));
$('.upload-modal .file-chooser').ajaxSubmit({
beforeSend: resetUploadBar,
......@@ -439,7 +439,7 @@ function displayFinishedUpload(xhr) {
$('.upload-modal .embeddable').show();
$('.upload-modal .file-name').hide();
$('.upload-modal .progress-fill').html(resp.msg);
$('.upload-modal .choose-file-button').html('Load Another File').show();
$('.upload-modal .choose-file-button').html(gettext('Load Another File')).show();
$('.upload-modal .progress-fill').width('100%');
// see if this id already exists, if so, then user must have updated an existing piece of content
......@@ -500,11 +500,11 @@ function toggleSock(e) {
});
if($sock.hasClass('is-shown')) {
$btnLabel.text('Hide Studio Help');
$btnLabel.text(gettext('Hide Studio Help'));
}
else {
$btnLabel.text('Looking for Help with Studio?');
$btnLabel.text(gettext('Looking for Help with Studio?'));
}
}
......@@ -845,7 +845,15 @@ function saveSetSectionScheduleDate(e) {
data: JSON.stringify({ 'id': id, 'metadata': {'start': start}})
}).success(function () {
var $thisSection = $('.courseware-section[data-id="' + id + '"]');
$thisSection.find('.section-published-date').html('<span class="published-status"><strong>Will Release:</strong> ' + input_date + ' at ' + input_time + ' UTC</span><a href="#" class="edit-button" data-date="' + input_date + '" data-time="' + input_time + '" data-id="' + id + '">Edit</a>');
var format = gettext('<strong>Will Release:</strong> %(date)s at $(time)s UTC');
var willReleaseAt = interpolate(format, [input_date, input_time], true);
$thisSection.find('.section-published-date').html(
'<span class="published-status">' + willReleaseAt + '</span>' +
'<a href="#" class="edit-button" ' +
'" data-date="' + input_date +
'" data-time="' + input_time +
'" data-id="' + id + '">' +
gettext('Edit') + '</a>');
$thisSection.find('.section-published-date').animate({
'background-color': 'rgb(182,37,104)'
}, 300).animate({
......
......@@ -30,6 +30,7 @@
<body class="<%block name='bodyclass'></%block> hide-wip">
<%include file="courseware_vendor_js.html"/>
<script type="text/javascript" src="${static.url('jsi18n/')}"></script>
<script type="text/javascript" src="${static.url('js/vendor/json2.js')}"></script>
<script type="text/javascript" src="${static.url('js/vendor/underscore-min.js')}"></script>
<script type="text/javascript" src="${static.url('js/vendor/backbone-min.js')}"></script>
......
<%! from django.utils.translation import ugettext as _ %>
<%inherit file="base.html" />
<%block name="title">My Courses</%block>
<%block name="title">${_("My Courses")}</%block>
<%block name="bodyclass">is-signedin index dashboard</%block>
<%block name="header_extras">
......@@ -36,18 +38,18 @@
<div class="wrapper-mast wrapper">
<header class="mast has-actions">
<div class="title">
<h1 class="title-1">My Courses</h1>
<h1 class="title-1">${_("My Courses")}</h1>
</div>
% if user.is_active:
<nav class="nav-actions">
<h3 class="sr">Page Actions</h3>
<h3 class="sr">${_("Page Actions")}</h3>
<ul>
<li class="nav-item">
% if not disable_course_creation:
<a href="#" class="button new-button new-course-button"><i class="ss-icon ss-symbolicons-standard icon icon-create">&#x002B;</i> New Course</a>
<a href="#" class="button new-button new-course-button"><i class="ss-icon ss-symbolicons-standard icon icon-create">&#x002B;</i> ${_("New Course")}</a>
% elif settings.MITX_FEATURES.get('STAFF_EMAIL',''):
<a href="mailto:${settings.MITX_FEATURES.get('STAFF_EMAIL','')}">Email staff to create course</a>
<a href="mailto:${settings.MITX_FEATURES.get('STAFF_EMAIL','')}">${_("Email staff to create course")}</a>
% endif
</li>
</ul>
......@@ -59,7 +61,9 @@
<div class="wrapper-content wrapper">
<section class="content">
<div class="introduction">
<p class="copy"><strong>Welcome, ${ user.username }</strong>. Here are all of the courses you are currently authoring in Studio:</p>
<p class="copy">
<strong>${_("Welcome, %(name)s") % dict(name= user.username)}</strong>.
${_("Here are all of the courses you are currently authoring in Studio:")}</p>
</div>
</section>
</div>
......@@ -81,11 +85,11 @@
% else:
<div class='warn-msg'>
<p>
In order to start authoring courses using edX Studio, please click on the activation link in your email.
${_("In order to start authoring courses using edX Studio, please click on the activation link in your email.")}
</p>
</div>
% endif
</article>
</div>
</div>
</%block>
\ No newline at end of file
</%block>
<%! from django.core.urlresolvers import reverse %>
<%! from django.utils.translation import ugettext as _ %>
<div class="wrapper-footer wrapper">
<footer class="primary" role="contentinfo">
<div class="colophon">
<p>&copy; 2013 <a href="http://www.edx.org" rel="external">edX</a>. All rights reserved.</p>
<p>&copy; 2013 <a href="http://www.edx.org" rel="external">edX</a>. ${ _("All rights reserved.")}</p>
</div>
<nav class="nav-peripheral">
......@@ -15,10 +17,11 @@
</li> -->
% if user.is_authenticated():
<li class="nav-item nav-peripheral-feedback">
<a href="http://help.edge.edx.org/discussion/new" class="show-tender" title="Use our feedback tool, Tender, to share your feedback">Contact Us</a>
<a href="http://help.edge.edx.org/discussion/new" class="show-tender" title="${_('Use our feedback tool, Tender, to share your feedback')}">${_("Contact Us")}</a>
</li>
% endif
</ol>
</nav>
</footer>
</div>
\ No newline at end of file
</div>
......@@ -120,6 +120,17 @@ urlpatterns += (
url(r'^ux-alerts$', 'contentstore.views.ux_alerts', name='ux-alerts')
)
js_info_dict = {
'domain': 'djangojs',
'packages': ('cms',),
}
urlpatterns += (
# Serve catalog of localized strings to be rendered by Javascript
url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict),
)
if settings.ENABLE_JASMINE:
# # Jasmine
urlpatterns = urlpatterns + (url(r'^_jasmine/', include('django_jasmine.urls')),)
......
# Extraction from Python source files
#[python: cms/**.py]
#[python: lms/**.py]
#[python: common/**.py]
# Extraction from Javscript source files
#[javascript: cms/**.js]
#[javascript: lms/**.js]
#[javascript: common/static/js/capa/**.js]
#[javascript: common/static/js/course_groups/**.js]
# do not extract from common/static/js/vendor/**
# Extraction from Mako templates
[mako: cms/templates/**.html]
input_encoding = utf-8
[mako: lms/templates/**.html]
input_encoding = utf-8
[mako: common/templates/**.html]
input_encoding = utf-8
{"locales" : ["en", "fr", "de"]}
import re
import itertools
class Converter:
"""Converter is an abstract class that transforms strings.
It hides embedded tags (HTML or Python sequences) from transformation
To implement Converter, provide implementation for inner_convert_string()
Strategy:
1. extract tags embedded in the string
a. use the index of each extracted tag to re-insert it later
b. replace tags in string with numbers (<0>, <1>, etc.)
c. save extracted tags in a separate list
2. convert string
3. re-insert the extracted tags
"""
# matches tags like these:
# HTML: <B>, </B>, <BR/>, <textformat leading="10">
# Python: %(date)s, %(name)s
tag_pattern = re.compile(r'(<[-\w" .:?=/]*>)|({[^}]*})|(%\([^)]*\)\w)', re.I)
def convert(self, string):
"""Returns: a converted tagged string
param: string (contains html tags)
Don't replace characters inside tags
"""
(string, tags) = self.detag_string(string)
string = self.inner_convert_string(string)
string = self.retag_string(string, tags)
return string
def detag_string(self, string):
"""Extracts tags from string.
returns (string, list) where
string: string has tags replaced by indices (<BR>... => <0>, <1>, <2>, etc.)
list: list of the removed tags ('<BR>', '<I>', '</I>')
"""
counter = itertools.count(0)
count = lambda m: '<%s>' % counter.next()
tags = self.tag_pattern.findall(string)
tags = [''.join(tag) for tag in tags]
(new, nfound) = self.tag_pattern.subn(count, string)
if len(tags) != nfound:
raise Exception('tags dont match:'+string)
return (new, tags)
def retag_string(self, string, tags):
"""substitutes each tag back into string, into occurrences of <0>, <1> etc"""
for (i, tag) in enumerate(tags):
p = '<%s>' % i
string = re.sub(p, tag, string, 1)
return string
# ------------------------------
# Customize this in subclasses of Converter
def inner_convert_string(self, string):
return string # do nothing by default
from converter import Converter
# Creates new localization properties files in a dummy language
# Each property file is derived from the equivalent en_US file, except
# 1. Every vowel is replaced with an equivalent with extra accent marks
# 2. Every string is padded out to +30% length to simulate verbose languages (e.g. German)
# to see if layout and flows work properly
# 3. Every string is terminated with a '#' character to make it easier to detect truncation
# --------------------------------
# Example use:
# >>> from dummy import Dummy
# >>> c = Dummy()
# >>> c.convert("hello my name is Bond, James Bond")
# u'h\xe9ll\xf6 my n\xe4m\xe9 \xefs B\xf6nd, J\xe4m\xe9s B\xf6nd Lorem i#'
#
# >>> c.convert('don\'t convert <a href="href">tag ids</a>')
# u'd\xf6n\'t \xe7\xf6nv\xe9rt <a href="href">t\xe4g \xefds</a> Lorem ipsu#'
#
# >>> c.convert('don\'t convert %(name)s tags on %(date)s')
# u"d\xf6n't \xe7\xf6nv\xe9rt %(name)s t\xe4gs \xf6n %(date)s Lorem ips#"
# Substitute plain characters with accented lookalikes.
# http://tlt.its.psu.edu/suggestions/international/web/codehtml.html#accent
TABLE = {'A': u'\xC0',
'a': u'\xE4',
'b': u'\xDF',
'C': u'\xc7',
'c': u'\xE7',
'E': u'\xC9',
'e': u'\xE9',
'I': U'\xCC',
'i': u'\xEF',
'O': u'\xD8',
'o': u'\xF6',
'u': u'\xFC'
}
# The print industry's standard dummy text, in use since the 1500s
# see http://www.lipsum.com/
LOREM = ' Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed ' \
'do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad ' \
'minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ' \
'ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate ' \
'velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat ' \
'cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. '
# To simulate more verbose languages (like German), pad the length of a string
# by a multiple of PAD_FACTOR
PAD_FACTOR = 1.3
class Dummy (Converter):
"""
A string converter that generates dummy strings with fake accents
and lorem ipsum padding.
"""
def convert(self, string):
result = Converter.convert(self, string)
return self.pad(result)
def inner_convert_string(self, string):
for (k,v) in TABLE.items():
string = string.replace(k, v)
return string
def pad(self, string):
"""add some lorem ipsum text to the end of string"""
size = len(string)
if size < 7:
target = size*3
else:
target = int(size*PAD_FACTOR)
return string + self.terminate(LOREM[:(target-size)])
def terminate(self, string):
"""replaces the final char of string with #"""
return string[:-1]+'#'
def init_msgs(self, msgs):
"""
Make sure the first msg in msgs has a plural property.
msgs is list of instances of pofile.Msg
"""
if len(msgs)==0:
return
headers = msgs[0].get_property('msgstr')
has_plural = len([header for header in headers if header.find('Plural-Forms:') == 0])>0
if not has_plural:
# Apply declaration for English pluralization rules
plural = "Plural-Forms: nplurals=2; plural=(n != 1);\\n"
headers.append(plural)
def convert_msg(self, msg):
"""
Takes one Msg object and converts it (adds a dummy translation to it)
msg is an instance of pofile.Msg
"""
source = msg.msgid
if len(source)==0:
# don't translate empty string
return
plural = msg.msgid_plural
if len(plural)>0:
# translate singular and plural
foreign_single = self.convert(source)
foreign_plural = self.convert(plural)
plural = {'0': self.final_newline(source, foreign_single),
'1': self.final_newline(plural, foreign_plural)}
msg.msgstr_plural = plural
return
else:
foreign = self.convert(source)
msg.msgstr = self.final_newline(source, foreign)
def final_newline(self, original, translated):
""" Returns a new translated string.
If last char of original is a newline, make sure translation
has a newline too.
"""
if len(original)>1:
if original[-1]=='\n' and translated[-1]!='\n':
return translated + '\n'
return translated
#!/usr/bin/python
# Generate test translation files from human-readable po files.
#
#
# po files can be generated with this:
# django-admin.py makemessages --all --extension html -l en
# Usage:
#
# $ ./make_dummy.py <sourcefile>
#
# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po
#
# generates output to
# mitx/conf/locale/vr/LC_MESSAGES/django.po
import os, sys
import polib
from dummy import Dummy
# Dummy language
# two letter language codes reference:
# see http://www.loc.gov/standards/iso639-2/php/code_list.php
#
# Django will not localize in languages that django itself has not been
# localized for. So we are using a well-known language: 'fr'.
OUT_LANG = 'fr'
def main(file):
"""
Takes a source po file, reads it, and writes out a new po file
containing a dummy translation.
"""
if not os.path.exists(file):
raise IOError('File does not exist: %s' % file)
pofile = polib.pofile(file)
converter = Dummy()
converter.init_msgs(pofile.translated_entries())
for msg in pofile:
converter.convert_msg(msg)
new_file = new_filename(file, OUT_LANG)
create_dir_if_necessary(new_file)
pofile.save(new_file)
def new_filename(original_filename, new_lang):
"""Returns a filename derived from original_filename, using new_lang as the locale"""
orig_dir = os.path.dirname(original_filename)
msgs_dir = os.path.basename(orig_dir)
orig_file = os.path.basename(original_filename)
return '%s/%s/%s/%s' % (os.path.abspath(orig_dir + '/../..'),
new_lang,
msgs_dir,
orig_file)
def create_dir_if_necessary(pathname):
dirname = os.path.dirname(pathname)
if not os.path.exists(dirname):
os.makedirs(dirname)
if __name__ == '__main__':
if len(sys.argv)<2:
raise Exception("missing file argument")
main(sys.argv[1])
#!/usr/bin/python
import os, subprocess, logging, json
from make_dummy import create_dir_if_necessary, main as dummy_main
'''
Generate or update all translation files
Usage:
$ update.py
1. extracts files from mako templates
2. extracts files from django templates and python source files
3. extracts files from django javascript files
4. generates dummy text translations
5. compiles po files to mo files
Configuration (e.g. known languages) declared in mitx/conf/locale/config
'''
# -----------------------------------
# BASE_DIR is the working directory to execute django-admin commands from.
# Typically this should be the 'mitx' directory.
BASE_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__))+'/..')
# LOCALE_DIR contains the locale files.
# Typically this should be 'mitx/conf/locale'
LOCALE_DIR = BASE_DIR + '/conf/locale'
# MSGS_DIR contains the English po files
MSGS_DIR = LOCALE_DIR + '/en/LC_MESSAGES'
# CONFIG_FILENAME contains localization configuration in json format
CONFIG_FILENAME = LOCALE_DIR + '/config'
# BABEL_CONFIG contains declarations for Babel to extract strings from mako template files
BABEL_CONFIG = LOCALE_DIR + '/babel.cfg'
# Strings from mako template files are written to BABEL_OUT
BABEL_OUT = MSGS_DIR + '/mako.po'
# These are the shell commands invoked by main()
COMMANDS = {
'babel_mako': 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT),
'make_django': 'django-admin.py makemessages --all --ignore=src/* --extension html -l en',
'make_djangojs': 'django-admin.py makemessages --all -d djangojs --ignore=src/* --extension js -l en',
'msgcat' : 'msgcat -o merged.po django.po %s' % BABEL_OUT,
'rename_django' : 'mv django.po django_old.po',
'rename_merged' : 'mv merged.po django.po',
'compile': 'django-admin.py compilemessages'
}
def execute (command_kwd, log, working_directory=BASE_DIR):
'''
Executes command_kwd, which references a shell command in COMMANDS.
'''
full_cmd = COMMANDS[command_kwd]
log.info('%s' % full_cmd)
subprocess.call(full_cmd.split(' '), cwd=working_directory)
def make_log ():
'''returns a logger'''
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
log.addHandler(log_handler)
return log
def get_config ():
'''Returns data found in config file, or returns None if file not found'''
config_path = os.path.abspath(CONFIG_FILENAME)
if not os.path.exists(config_path):
return None
with open(config_path) as stream:
return json.load(stream)
def main ():
log = make_log()
create_dir_if_necessary(LOCALE_DIR)
log.info('Executing all commands from %s' % BASE_DIR)
remove_files = ['django.po', 'djangojs.po', 'nonesuch']
for filename in remove_files:
path = MSGS_DIR + '/' + filename
log.info('Deleting file %s' % path)
if not os.path.exists(path):
log.warn("File does not exist: %s" % path)
else:
os.remove(path)
# Generate or update human-readable .po files from all source code.
execute('babel_mako', log=log)
execute('make_django', log=log)
execute('make_djangojs', log=log)
execute('msgcat', log=log, working_directory=MSGS_DIR)
execute('rename_django', log=log, working_directory=MSGS_DIR)
execute('rename_merged', log=log, working_directory=MSGS_DIR)
# Generate dummy text files from the English .po files
log.info('Generating dummy text.')
dummy_main(LOCALE_DIR + '/en/LC_MESSAGES/django.po')
dummy_main(LOCALE_DIR + '/en/LC_MESSAGES/djangojs.po')
# Generate machine-readable .mo files
execute('compile', log)
if __name__ == '__main__':
main()
......@@ -6,7 +6,7 @@
<div class="discussion-post">
<header>
%if thread['group_id']
%if thread['group_id']:
<div class="group-visibility-label">This post visible only to group ${cohort_dictionary[thread['group_id']]}. </div>
%endif
......@@ -35,4 +35,4 @@
</ol>
</article>
<%include file="_js_data.html" />
\ No newline at end of file
<%include file="_js_data.html" />
-r repo-requirements.txt
Babel==0.9.6
beautifulsoup4==4.1.3
beautifulsoup==3.2.1
boto==2.6.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment