workaround for gettext parser bug

.tx/config

+[main]
+host = https://www.transifex.com
+
+[edx-studio.django-partial]
+file_filter = conf/locale/<lang>/LC_MESSAGES/django-partial.po
+source_file = conf/locale/en/LC_MESSAGES/django-partial.po
+source_lang = en
+type = PO
+
+[edx-studio.djangojs]
+file_filter = conf/locale/<lang>/LC_MESSAGES/djangojs.po
+source_file = conf/locale/en/LC_MESSAGES/djangojs.po
+source_lang = en
+type = PO
+
+[edx-studio.mako]
+file_filter = conf/locale/<lang>/LC_MESSAGES/mako.po
+source_file = conf/locale/en/LC_MESSAGES/mako.po
+source_lang = en
+type = PO
+
+[edx-studio.messages]
+file_filter = conf/locale/<lang>/LC_MESSAGES/messages.po
+source_file = conf/locale/en/LC_MESSAGES/messages.po
+source_lang = en
+type = PO

conf/locale/config

-{"locales" : ["en"]}
+{"locales" : ["en", "fr", "es"]}

i18n/execute.py

@@ -50,14 +50,29 @@ def execute (command, working_directory=BASE_DIR, log=LOG):
     """
     Executes shell command in a given working_directory.
     Command is a string to pass to the shell.
-    Output is logged to log.
+    The command is logged to log, output is ignored.
     """
+    if log:
         log.info(command)
     subprocess.call(command.split(' '), cwd=working_directory)
 
+
+def call(command, working_directory=BASE_DIR, log=LOG):
+    """
+    Executes shell command in a given working_directory.
+    Command is a string to pass to the shell.
+    Returns a tuple of two strings: (stdout, stderr)
+    """
+    if log:
+        log.info(command)
+    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory)
+    out, err = p.communicate()
+    return (out, err)
+    
+    
 def get_config():
     """Returns data found in config file, or returns None if file not found"""
-    config_path = os.path.abspath(CONFIG_FILENAME)
+    config_path = os.path.normpath(CONFIG_FILENAME)
     if not os.path.exists(config_path):
         log.warn("Configuration file cannot be found: %s" % \
                  os.path.relpath(config_path, BASE_DIR))

i18n/extract.py

@@ -79,6 +79,7 @@ def fix_header(po):
     """
     Replace default headers with edX headers
     """
+    po.metadata_is_fuzzy = []   # remove [u'fuzzy']
     header = po.header
     fixes = (
         ('SOME DESCRIPTIVE TITLE', 'edX translation file'),
@@ -119,10 +120,9 @@ def fix_metadata(po):
              'Report-Msgid-Bugs-To': 'translation_team@edx.org',
              'Project-Id-Version': '0.1a',
              'Language' : 'en',
+             'Last-Translator' : '',
              'Language-Team': 'translation team <translation_team@edx.org>',
              }
-    if po.metadata.has_key('Last-Translator'):
-        del po.metadata['Last-Translator']
     po.metadata.update(fixes)
 
 def strip_key_strings(po):

i18n/generate.py

@@ -14,6 +14,8 @@
 """
 
 import os
+from polib import pofile
+
 from execute import execute, get_config, messages_dir, remove_file, \
      BASE_DIR, LOG, SOURCE_LOCALE
 
@@ -30,11 +32,23 @@ def merge(locale, target='django.po'):
     merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge)
     execute(merge_cmd, working_directory=locale_directory)
 
-    # rename merged.po -> django.po (default)
+    # clean up redunancies in the metadata
     merged_filename = os.path.join(locale_directory, 'merged.po')
+    clean_metadata(merged_filename)
+
+    # rename merged.po -> django.po (default)
     django_filename = os.path.join(locale_directory, target)
     os.rename(merged_filename, django_filename) # can't overwrite file on Windows
 
+def clean_metadata(file):
+    """
+    Clean up redundancies in the metadata caused by merging.
+    This reads in a PO file and simply saves it back out again.
+    """
+    po = pofile(file)
+    po.save()
+    
+
 def validate_files(dir, files_to_merge):
     """
     Asserts that the given files exist.

i18n/make_dummy.py

@@ -10,15 +10,15 @@
 #
 # $ ./make_dummy.py <sourcefile>
 #
-# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po
+# $ ./make_dummy.py ../conf/locale/en/LC_MESSAGES/django.po
 #
 # generates output to
-#    mitx/conf/locale/vr/LC_MESSAGES/django.po
+#    mitx/conf/locale/fr/LC_MESSAGES/django.po
 
 import os, sys
 import polib
 from dummy import Dummy
-from execute import create_dir_if_necessary
+from execute import get_logger, create_dir_if_necessary
 
 def main(file, locale):
     """
@@ -41,11 +41,11 @@ def new_filename(original_filename, new_locale):
     orig_dir = os.path.dirname(original_filename)
     msgs_dir = os.path.basename(orig_dir)
     orig_file = os.path.basename(original_filename)
-    return os.path.join(orig_dir,
-                        '/../..',
+    return os.path.abspath(os.path.join(orig_dir,
+                                        '../..',
                                         new_locale,
                                         msgs_dir,
-                        orig_file)
+                                        orig_file))
 
 
 # Dummy language 
@@ -60,7 +60,7 @@ DEFAULT_LOCALE = 'fr'
 if __name__ == '__main__':
     if len(sys.argv)<2:
         raise Exception("missing file argument")
-    if len(sys.argv)<2:
+    if len(sys.argv)<3:
         locale = DEFAULT_LOCALE
     else:
         locale = sys.argv[2]

i18n/tests/__init__.py

@@ -2,3 +2,5 @@ from test_extract import TestExtract
 from test_generate import TestGenerate
 from test_converter import TestConverter
 from test_dummy import TestDummy
+from test_validate import TestValidate
+

i18n/tests/test_validate.py

+import os
+from unittest import TestCase
+from nose.plugins.skip import SkipTest
+
+from execute import call, LOCALE_DIR, LOG
+
+class TestValidate(TestCase):
+    """
+    Call GNU msgfmt -c on each .po file to validate its format.
+    """
+        
+    def test_validate(self):
+        # Skip this test for now because it's very noisy
+        raise SkipTest()
+        for file in self.get_po_files():
+            # Use relative paths to make output less noisy.
+            rfile = os.path.relpath(file, LOCALE_DIR)
+            (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR)
+            if err != '':
+                LOG.warn('\n'+err)
+
+    def get_po_files(self, root=LOCALE_DIR):
+        """
+        This is a generator. It yields all of the .po files under root.
+        """
+        for (dirpath, dirnames, filenames) in os.walk(root):
+            for name in filenames:
+                (base, ext) = os.path.splitext(name)
+                if ext.lower() == '.po':
+                    yield os.path.join(dirpath, name)
+
+
+    

rakefile

@@ -337,12 +337,6 @@ task :migrate, [:env] do |t, args|
     sh(django_admin(:lms, args.env, 'migrate'))
 end
 
-desc "Run tests for the internationalization library"
-task :test_i18n do
-  test = File.join(REPO_ROOT, "i18n", "tests")
-  sh("nosetests #{test}")
-end
-
 Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
     task_name = "test_#{lib}"
 
@@ -516,27 +510,49 @@ end
 
 # --- Internationalization tasks
 
-desc "Extract localizable strings from sources"
-task :extract_dev_strings do
+namespace :i18n do
+
+  desc "Extract localizable strings from sources"
+  task :extract do
     sh(File.join(REPO_ROOT, "i18n", "extract.py"))
-end
+  end
 
-desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first."
-task :generate_i18n do
+  desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first."
+  task :generate do
     if ARGV.last.downcase == 'extract'
-    Rake::Task["extract_dev_strings"].execute
+      Rake::Task["i18n:extract"].execute
     end
     sh(File.join(REPO_ROOT, "i18n", "generate.py"))
-end
+  end
 
-desc "Simulate international translation by generating dummy strings corresponding to source strings."
-task :dummy_i18n do
+  desc "Simulate international translation by generating dummy strings corresponding to source strings."
+  task :dummy do
     source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"]
     dummy_locale = 'fr'
     cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py")
     for file in source_files do
       sh("#{cmd} #{file} #{dummy_locale}")
     end
+  end
+
+  namespace :transifex do
+    desc "Push source strings to Transifex for translation"
+    task :push do
+      sh("tx push -s")
+    end
+
+    desc "Pull transated strings from Transifex"
+    task :pull do
+      sh("tx pull")
+    end
+  end
+
+  desc "Run tests for the internationalization library"
+  task :test do
+    test = File.join(REPO_ROOT, "i18n", "tests")
+    sh("nosetests #{test}")
+  end
+
 end
 
 # --- Develop and public documentation ---