Prep for initial language release

.gitignore

@@ -30,14 +30,14 @@ codekit-config.json
 
 ### Internationalization artifacts
 *.mo
+*.po
+!django.po
+!django.mo
+!djangojs.po
+!djangojs.mo
 conf/locale/en/LC_MESSAGES/*.po
-!messages.po
-### Remove when we have real Esperanto translations. For now, ignore
-### dummy Esperanto files.
-conf/locale/eo/*
-## Remove when we officially support these languages.
-conf/locale/fr
-conf/locale/ko_KR
+conf/locale/en/LC_MESSAGES/*.mo
+conf/locale/messages.mo
 
 ### Testing artifacts
 .testids/

cms/envs/common.py

+# -*- coding: utf-8 -*-
 """
 This is the common settings file, intended to set sane defaults. If you have a
 piece of configuration that's dependent on a set of feature flags being set,
@@ -254,10 +255,7 @@ STATICFILES_DIRS = [
 TIME_ZONE = 'America/New_York'  # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 LANGUAGE_CODE = 'en'  # http://www.i18nguy.com/unicode/language-identifiers.html
 
-LANGUAGES = (
-    ('eo', 'Esperanto'),
-)
-
+LANGUAGES = lms.envs.common.LANGUAGES
 USE_I18N = True
 USE_L10N = True
 

conf/locale/config.yaml

 # Configuration for i18n workflow.
 
 locales:
-    - en
+    - en  # English - Source Language
+    - ach  # Acoli
+    - ar  # Arabic
+    - bg_BG  # Bulgarian (Bulgaria)
+    - bn  # Bengali
+    - bn_BD  # Bengali (Bangladesh)
+    - cs  # Czech
+    - cy  # Welsh
+    - de_DE  # German (Germany)
+    - en@lolcat  # LOLCAT English
+    - en@pirate  # Pirate English
+    - es_419  # Spanish (Latin America)
+    - es_EC  # Spanish (Ecuador)
+    - es_ES  # Spanish (Spain)
+    - es_MX  # Spanish (Mexico)
+    - es_US  # Spanish (United States)
+    - et_EE  # Estonian (Estonia)
+    - fa  # Persian
+    - fa_IR  # Persian (Iran)
+    - fi_FI  # Finnish (Finland)
+    - fr  # French
+    - gl  # Galician
+    - he  # Hebrew
+    - hi  # Hindi
+    - hy_AM  # Armenian (Armenia)
+    - id  # Indonesian
+    - it_IT  # Italian (Italy)
+    - ja_JP  # Japanese (Japan)
+    - km_KH  # Khmer (Cambodia)
+    - ko_KR  # Korean (Korea)
+    - lt_LT  # Lithuanian (Lithuania)
+    - ml  # Malayalam
+    - nb  # Norwegian Bokmål
+    - nl_NL  # Dutch (Netherlands)
+    - pl  # Polish
+    - pt_BR  # Portuguese (Brazil)
+    - pt_PT  # Portuguese (Portugal)
+    - ru  # Russian
+    - si  # Sinhala
+    - sk  # Slovak
+    - sl  # Slovenian
+    - th  # Thai
+    - tr_TR  # Turkish (Turkey)
+    - uk  # Ukranian
+    - vi  # Vietnamese
+    - zh_CN  # Chinese (China)
+    - zh_CN.GB2312  # Chinese (China) (GB2312)
+    - zh_TW  # Chinese (Taiwan)
 
-    # More languages we might want someday, these have started on Transifex.
-    # fr
-    # ko_KR
-    # ru
-    # es_419
-    # ja_JP
-    # pt_BR
-    # zh_CN
-    # zh_TW
-    # ar
-    # es_ES
-    # fa_IR
-    # tr_TR
-    # de_DE
-    # id
-    # hi
-    # vi
-    # pt_PT
-    # lt_LT
-    # gl
-    # it_IT
-    # cs
-    # et_EE
-    # nb
-    # sk
 
 # The locale used for fake-accented English, for testing.
 dummy-locale: eo

i18n/config.py

@@ -69,5 +69,11 @@ class Configuration(object):
         """
         return self.get_messages_dir(self.source_locale)
 
+    @property
+    def translated_locales(self):
+        """
+        Returns the set of locales to be translated (ignoring the source_locale).
+        """
+        return sorted(set(self.locales) - set([self.source_locale]))
 
 CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config.yaml').normpath())

i18n/extract.py

@@ -32,7 +32,7 @@ BABEL_CONFIG = BASE_DIR.relpathto(LOCALE_DIR.joinpath('babel.cfg'))
 # Use relpath to reduce noise in logs
 BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako.po'))
 
-SOURCE_WARN = 'This English source file is machine-generated. Do not check it into git.'
+EDX_MARKER = "edX translation file"
 
 LOG = logging.getLogger(__name__)
 
@@ -104,8 +104,8 @@ def fix_header(po):
     po.metadata_is_fuzzy = []   # remove [u'fuzzy']
     header = po.header
     fixes = (
-        ('SOME DESCRIPTIVE TITLE', 'edX translation file\n' + SOURCE_WARN),
-        ('Translations template for PROJECT.', 'edX translation file\n' + SOURCE_WARN),
+        ('SOME DESCRIPTIVE TITLE', EDX_MARKER),
+        ('Translations template for PROJECT.', EDX_MARKER),
         ('YEAR', '%s' % datetime.utcnow().year),
         ('ORGANIZATION', 'edX'),
         ("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"),

i18n/generate.py

@@ -103,7 +103,7 @@ def validate_files(dir, files_to_merge):
 def main():
     logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
-    for locale in CONFIGURATION.locales:
+    for locale in CONFIGURATION.translated_locales:
         merge_files(locale)
     # Dummy text is not required. Don't raise exception if files are missing.
     merge_files(CONFIGURATION.dummy_locale, fail_if_missing=False)

i18n/tests/test_compiled_messages.py

@@ -13,7 +13,6 @@ from unittest import TestCase
 
 from i18n.config import CONFIGURATION, LOCALE_DIR
 
-
 @ddt.ddt
 class TestCompiledMessages(TestCase):
     """
@@ -22,7 +21,7 @@ class TestCompiledMessages(TestCase):
 
     PO_FILES = ['django.po', 'djangojs.po']
 
-    @ddt.data(*CONFIGURATION.locales)
+    @ddt.data(*CONFIGURATION.translated_locales)
     def test_translated_messages(self, locale):
         message_dir = LOCALE_DIR / locale / 'LC_MESSAGES'
         for pofile_name in self.PO_FILES:

i18n/tests/test_generate.py

@@ -8,6 +8,7 @@ from unittest import TestCase
 from polib import pofile
 from pytz import UTC
 
+from i18n import extract
 from i18n import generate
 from i18n.config import CONFIGURATION
 
@@ -27,6 +28,7 @@ class TestGenerate(TestCase):
         """
         Tests merge script on English source files.
         """
+        extract.main()
         filename = os.path.join(CONFIGURATION.source_messages_dir, random_name())
         generate.merge(CONFIGURATION.source_locale, target=filename)
         self.assertTrue(os.path.exists(filename))
@@ -41,7 +43,7 @@ class TestGenerate(TestCase):
         after start of test suite)
         """
         generate.main()
-        for locale in CONFIGURATION.locales:
+        for locale in CONFIGURATION.translated_locales:
             for filename in ('django', 'djangojs'):
                 mofile = filename+'.mo'
                 path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile)

i18n/transifex.py

@@ -4,11 +4,11 @@ import sys
 from polib import pofile
 
 from i18n.config import CONFIGURATION
-from i18n.extract import SOURCE_WARN
 from i18n.execute import execute
+from i18n.extract import EDX_MARKER
 
-TRANSIFEX_HEADER = 'Translations in this file have been downloaded from %s'
-TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-studio/'
+TRANSIFEX_HEADER = 'edX community translations have been downloaded from %s'
+TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-platform/'
 
 
 def push():
@@ -26,9 +26,9 @@ def clean_translated_locales():
     Strips out the warning from all translated po files
     about being an English source file.
     """
-    for locale in CONFIGURATION.locales:
-        if locale != CONFIGURATION.source_locale:
-            clean_locale(locale)
+    for locale in CONFIGURATION.translated_locales:
+        clean_locale(locale)
+
 
 def clean_locale(locale):
     """
@@ -40,18 +40,20 @@ def clean_locale(locale):
     for filename in ('django-partial.po', 'djangojs-partial.po', 'mako.po'):
         clean_file(dirname.joinpath(filename))
 
-def clean_file(file):
+
+def clean_file(filename):
     """
     Strips out the warning from a translated po file about being an English source file.
     Replaces warning with a note about coming from Transifex.
     """
-    po = pofile(file)
-    if po.header.find(SOURCE_WARN) != -1:
+    po = pofile(filename)
+    if po.header.find(EDX_MARKER) != -1:
         new_header = get_new_header(po)
-        new = po.header.replace(SOURCE_WARN, new_header)
+        new = po.header.replace(EDX_MARKER, new_header)
         po.header = new
         po.save()
 
+
 def get_new_header(po):
     team = po.metadata.get('Language-Team', None)
     if not team:
@@ -59,6 +61,7 @@ def get_new_header(po):
     else:
         return TRANSIFEX_HEADER % team
 
+
 if __name__ == '__main__':
     if len(sys.argv) < 2:
         raise Exception("missing argument: push or pull")

lms/envs/common.py

+# -*- coding: utf-8 -*-
 """
 This is the common settings file, intended to set sane defaults. If you have a
 piece of configuration that's dependent on a set of feature flags being set,
@@ -499,8 +500,56 @@ FAVICON_PATH = 'images/favicon.ico'
 TIME_ZONE = 'America/New_York'  # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 LANGUAGE_CODE = 'en'  # http://www.i18nguy.com/unicode/language-identifiers.html
 
+# Sourced from http://www.localeplanet.com/icu/ and wikipedia
 LANGUAGES = (
-    ('eo', 'Esperanto'),
+    ('eo', u'Dummy Language (Esperanto)'),  # Dummy languaged used for testing
+
+    ('ach', u'Acholi'),  # Acoli
+    ('ar', u'العربية'),  # Arabic
+    ('bg-bg', u'български (България)'),  # Bulgarian (Bulgaria)
+    ('bn', u'বাংলা'),  # Bengali
+    ('bn-bd', u'বাংলা (বাংলাদেশ)'),  # Bengali (Bangladesh)
+    ('cs', u'Čeština'),  # Czech
+    ('cy', u'Cymraeg'),  # Welsh
+    ('de-de', u'Deutsch (Deutschland)'),  # German (Germany)
+    ('en@lolcat', u'LOLCAT English'),  # LOLCAT English
+    ('en@pirate', u'Pirate English'),  # Pirate English
+    ('es-419', u'Español (Latinoamérica)'),  # Spanish (Latin America)
+    ('es-ec', u'Español (Ecuador)'),  # Spanish (Ecuador)
+    ('es-es', u'Español (España)'),  # Spanish (Spain)
+    ('es-mx', u'Español (México)'),  # Spanish (Mexico)
+    ('es-us', u'Español (Estados Unidos)'),  # Spanish (United States)
+    ('et-ee', u'Eesti (Eesti)'),  # Estonian (Estonia)
+    ('fa', u'فارسی'),  # Persian
+    ('fa-ir', u'فارسی (ایران)'),  # Persian (Iran)
+    ('fi-fi', u'Suomi (Suomi)'),  # Finnish (Finland)
+    ('fr', u'Français'),  # French
+    ('gl', u'Galego'),  # Galician
+    ('he', u'עברית'),  # Hebrew
+    ('hi', u'हिन्दी'),  # Hindi
+    ('hy-am', u'Հայերէն (Հայաստանի Հանրապետութիւն)'),  # Armenian (Armenia)
+    ('id', u'Bahasa Indonesia'),  # Indonesian
+    ('it-it', u'Italiano (Italia)'),  # Italian (Italy)
+    ('ja-jp', u'日本語(日本)'),  # Japanese (Japan)
+    ('km-kh', u'ភាសាខ្មែរ (កម្ពុជា)'),  # Khmer (Cambodia)
+    ('ko-kr', u'한국어(대한민국)'),  # Korean (Korea)
+    ('lt-lt', u'Lietuvių (Lietuva)'),  # Lithuanian (Lithuania)
+    ('ml', u'മലയാളം'),  # Malayalam
+    ('nb', u'Norsk bokmål'),  # Norwegian Bokmål
+    ('nl-nl', u'Nederlands (Nederland)'),  # Dutch (Netherlands)
+    ('pl', u'Polski'),  # Polish
+    ('pt-br', u'Português (Brasil)'),  # Portuguese (Brazil)
+    ('pt-pt', u'Português (Portugal)'),  # Portuguese (Portugal)
+    ('ru', u'Русский'),  # Russian
+    ('si', u'සිංහල'),  # Sinhala
+    ('sk', u'Slovenčina'),  # Slovak
+    ('sl', u'Slovenščina'),  # Slovenian
+    ('th', u'ไทย'),  # Thai
+    ('tr-tr', u'Türkçe (Türkiye)'),  # Turkish (Turkey)
+    ('uk', u'Українська'),  # Uknranian
+    ('vi', u'Tiếng Việt'),  # Vietnamese
+    ('zh-cn', u'大陆简体'),  # Chinese (China)
+    ('zh-tw', u'台灣正體'),  # Chinese (Taiwan)
 )
 
 USE_I18N = True

rakelib/i18n.rake

@@ -57,7 +57,7 @@ namespace :i18n do
   end
 
   desc "Run tests for the internationalization library"
-  task :test => ["i18n:validate:gettext", "i18n:extract", "i18n:generate"] do
+  task :test do
     test = File.join(REPO_ROOT, "i18n", "tests")
     pythonpath_prefix = "PYTHONPATH=#{REPO_ROOT}/i18n:$PYTHONPATH"
     sh("#{pythonpath_prefix} nosetests #{test}")