integrate dummy string tools; added rake tasks and unit tests

i18n/dummy.py

@@ -86,7 +86,7 @@ class Dummy (Converter):
     def init_msgs(self, msgs):
         """
         Make sure the first msg in msgs has a plural property.
-        msgs is list of instances of pofile.Msg
+        msgs is list of instances of polib.POEntry
         """
         if len(msgs)==0:
             return
@@ -100,8 +100,8 @@ class Dummy (Converter):
 
     def convert_msg(self, msg):
         """
-        Takes one Msg object and converts it (adds a dummy translation to it)
-        msg is an instance of pofile.Msg
+        Takes one POEntry object and converts it (adds a dummy translation to it)
+        msg is an instance of polib.POEntry
         """
         source = msg.msgid
         if len(source)==0:

i18n/make_dummy.py

@@ -18,20 +18,12 @@
 import os, sys
 import polib
 from dummy import Dummy
+from execute import create_dir_if_necessary
 
-# Dummy language 
-# two letter language codes reference:
-# see http://www.loc.gov/standards/iso639-2/php/code_list.php
-#
-# Django will not localize in languages that django itself has not been
-# localized for. So we are using a well-known language: 'fr'.
-
-OUT_LANG = 'fr'
-
-def main(file):
+def main(file, locale):
     """
     Takes a source po file, reads it, and writes out a new po file
-    containing a dummy translation.
+    in :param locale: containing a dummy translation.
     """
     if not os.path.exists(file):
         raise IOError('File does not exist: %s' % file)
@@ -40,29 +32,36 @@ def main(file):
     converter.init_msgs(pofile.translated_entries())
     for msg in pofile:
         converter.convert_msg(msg)
-    new_file = new_filename(file, OUT_LANG)
+    new_file = new_filename(file, locale)
     create_dir_if_necessary(new_file)
     pofile.save(new_file)
-    
-
 
-def new_filename(original_filename, new_lang):
-    """Returns a filename derived from original_filename, using new_lang as the locale"""
+def new_filename(original_filename, new_locale):
+    """Returns a filename derived from original_filename, using new_locale as the locale"""
     orig_dir = os.path.dirname(original_filename)
     msgs_dir = os.path.basename(orig_dir)
     orig_file = os.path.basename(original_filename)
-    return '%s/%s/%s/%s' % (os.path.abspath(orig_dir + '/../..'),
-                            new_lang,
-                            msgs_dir,
-                            orig_file)
+    return os.path.join(orig_dir,
+                        '/../..',
+                        new_locale,
+                        msgs_dir,
+                        orig_file)
 
 
-def create_dir_if_necessary(pathname):
-    dirname = os.path.dirname(pathname)
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+# Dummy language 
+# two letter language codes reference:
+# see http://www.loc.gov/standards/iso639-2/php/code_list.php
+#
+# Django will not localize in languages that django itself has not been
+# localized for. So we are using a well-known language: 'fr'.
+
+DEFAULT_LOCALE = 'fr'
 
 if __name__ == '__main__':
     if len(sys.argv)<2:
         raise Exception("missing file argument")
-    main(sys.argv[1])
+    if len(sys.argv)<2:
+        locale = DEFAULT_LOCALE
+    else:
+        locale = sys.argv[2]
+    main(sys.argv[1], locale)

i18n/test/__init__.py

 from test_extract import TestExtract
 from test_generate import TestGenerate
+from test_converter import TestConverter
+from test_dummy import TestDummy

i18n/update.py

-#!/usr/bin/python
-
-import os, subprocess, logging, json
-from make_dummy import create_dir_if_necessary, main as dummy_main
-
-'''
-Generate or update all translation files
- Usage:
-    $ update.py
-
-
- 1. extracts files from mako templates
- 2. extracts files from django templates and python source files
- 3. extracts files from django javascript files
- 4. generates dummy text translations
- 5. compiles po files to mo files
-
- Configuration (e.g. known languages) declared in mitx/conf/locale/config
-'''
-
-# -----------------------------------
-# BASE_DIR is the working directory to execute django-admin commands from.
-# Typically this should be the 'mitx' directory.
-BASE_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__))+'/..')
-
-# LOCALE_DIR contains the locale files.
-# Typically this should be 'mitx/conf/locale'
-LOCALE_DIR = BASE_DIR + '/conf/locale'
-
-# MSGS_DIR contains the English po files
-MSGS_DIR = LOCALE_DIR + '/en/LC_MESSAGES'
-
-# CONFIG_FILENAME contains localization configuration in json format
-CONFIG_FILENAME = LOCALE_DIR + '/config'
-
-# BABEL_CONFIG contains declarations for Babel to extract strings from mako template files
-BABEL_CONFIG = LOCALE_DIR + '/babel.cfg'
-
-# Strings from mako template files are written to BABEL_OUT
-BABEL_OUT = MSGS_DIR + '/mako.po'
-
-# These are the shell commands invoked by main()
-COMMANDS = {
-    'babel_mako': 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT),
-    'make_django': 'django-admin.py makemessages --all --ignore=src/* --extension html -l en',
-    'make_djangojs': 'django-admin.py makemessages --all -d djangojs --ignore=src/* --extension js -l en',
-    'msgcat' : 'msgcat -o merged.po django.po %s' % BABEL_OUT,
-    'rename_django' : 'mv django.po django_old.po',
-    'rename_merged' : 'mv merged.po django.po',
-    'compile': 'django-admin.py compilemessages'
-    
-    }
-
-def execute (command_kwd, log, working_directory=BASE_DIR):
-    '''
-    Executes command_kwd, which references a shell command in COMMANDS.
-    '''
-    full_cmd = COMMANDS[command_kwd]
-    log.info('%s' % full_cmd)
-    subprocess.call(full_cmd.split(' '), cwd=working_directory)
-    
-def make_log ():
-    '''returns a logger'''
-    log = logging.getLogger(__name__)
-    log.setLevel(logging.INFO)
-    log_handler = logging.StreamHandler()
-    log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
-    log.addHandler(log_handler)
-    return log
-
-def get_config ():
-    '''Returns data found in config file, or returns None if file not found'''
-    config_path = os.path.abspath(CONFIG_FILENAME)
-    if not os.path.exists(config_path):
-        return None
-    with open(config_path) as stream:
-        return json.load(stream)
-
-def main ():
-    log = make_log()
-    create_dir_if_necessary(LOCALE_DIR)
-    log.info('Executing all commands from %s' % BASE_DIR)
-
-    remove_files = ['django.po', 'djangojs.po', 'nonesuch']
-    for filename in remove_files:
-        path = MSGS_DIR + '/' + filename
-        log.info('Deleting file %s' % path)
-        if not os.path.exists(path):
-            log.warn("File does not exist: %s" % path)
-        else:
-            os.remove(path)
-
-    # Generate or update human-readable .po files from all source code.
-    execute('babel_mako', log=log)
-    execute('make_django', log=log)
-    execute('make_djangojs', log=log)
-    execute('msgcat', log=log, working_directory=MSGS_DIR)
-    execute('rename_django', log=log, working_directory=MSGS_DIR)
-    execute('rename_merged', log=log, working_directory=MSGS_DIR)
-
-    # Generate dummy text files from the English .po files
-    log.info('Generating dummy text.')
-    dummy_main(LOCALE_DIR + '/en/LC_MESSAGES/django.po')
-    dummy_main(LOCALE_DIR + '/en/LC_MESSAGES/djangojs.po')
-
-    # Generate machine-readable .mo files
-    execute('compile', log)
-
-if __name__ == '__main__':
-    main()

rakefile

@@ -516,6 +516,16 @@ task :generate_i18n do
   sh(File.join(REPO_ROOT, "i18n", "generate.py"))
 end
 
+desc "Simulate international translation by generating dummy strings corresponding to source strings."
+task :dummy_i18n do
+  source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"]
+  dummy_locale = 'fr'
+  cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py")
+  for file in source_files do
+    sh("#{cmd} #{file} #{dummy_locale}")
+  end
+end
+
 # --- Develop and public documentation ---
 desc "Invoke sphinx 'make build' to generate docs."
 task :builddocs, [:options] do |t, args|

requirements.txt

 -r repo-requirements.txt
-Babel==0.9.6
 beautifulsoup4==4.1.3
 beautifulsoup==3.2.1
 boto==2.6.0
@@ -62,6 +61,10 @@ newrelic==1.8.0.13
 # Used for documentation gathering
 sphinx==1.1.3
 
+# Used for Internationalization and localization
+Babel==0.9.6
+transifex-client==0.8
+
 # Used for testing
 coverage==3.6
 factory_boy==2.0.2