Skip to content

E
edx-platform
Commit 31b64ad5 by David Baumgold
Options

Merge pull request #2933 from edx/cale/quieter-i18n-tests 

Make i18n tests quieter
parents 2d6bfbad d93238d8
Showing with 194 additions and 125 deletions
i18n/converter.py
import re
import itertools
class Converter(object):
"""Converter is an abstract class that transforms strings.
It hides embedded tags (HTML or Python sequences) from transformation
......@@ -20,7 +21,8 @@ class Converter(object):
# matches tags like these:
# HTML: <B>, </B>, <BR/>, <textformat leading="10">
# Python: %(date)s, %(name)s
tag_pattern = re.compile(r'''
tag_pattern = re.compile(
r'''
(<[^>]+>) | # <tag>
({[^}]+}) | # {tag}
(%\([\w]+\)\w) | # %(tag)s
......@@ -28,7 +30,7 @@ class Converter(object):
(&\#\d+;) | # &#1234;
(&\#x[0-9a-f]+;) # &#xABCD;
''',
re.IGNORECASE|re.VERBOSE
re.IGNORECASE | re.VERBOSE
)
def convert(self, string):
......@@ -55,7 +57,7 @@ class Converter(object):
tags = [''.join(tag) for tag in tags]
(new, nfound) = self.tag_pattern.subn(count, string)
if len(tags) != nfound:
raise Exception('tags dont match:'+string)
raise Exception('tags dont match:' + string)
return (new, tags)
def retag_string(self, string, tags):
......@@ -65,7 +67,6 @@ class Converter(object):
string = re.sub(p, tag, string, 1)
return string
# ------------------------------
# Customize this in subclasses of Converter
......
i18n/dummy.py
......@@ -22,15 +22,15 @@ $ ./dummy.py
generates output conf/locale/$DUMMY_LOCALE/LC_MESSAGES,
where $DUMMY_LOCALE is the dummy_locale value set in the i18n config
"""
from __future__ import print_function
import re
import sys
import argparse
import polib
from path import path
from i18n.config import CONFIGURATION
from i18n.execute import create_dir_if_necessary
from i18n.converter import Converter
......@@ -186,7 +186,7 @@ def make_dummy(filename, locale, converter):
pofile.metadata['Plural-Forms'] = 'nplurals=2; plural=(n != 1);'
new_file = new_filename(filename, locale)
create_dir_if_necessary(new_file)
new_file.parent.makedirs_p()
pofile.save(new_file)
......@@ -197,18 +197,25 @@ def new_filename(original_filename, new_locale):
return new_file.abspath()
def main():
def main(verbosity=1):
"""
Generate dummy strings for all source po files.
"""
SOURCE_MSGS_DIR = CONFIGURATION.source_messages_dir
for locale, converter in zip(CONFIGURATION.dummy_locales, [Dummy(), Dummy2()]):
print "Processing source language files into dummy strings, locale {}:".format(locale)
if verbosity:
print('Processing source language files into dummy strings, locale "{}"'.format(locale))
for source_file in CONFIGURATION.source_messages_dir.walkfiles('*.po'):
print ' ', source_file.relpath()
if verbosity:
print(' ', source_file.relpath())
make_dummy(SOURCE_MSGS_DIR.joinpath(source_file), locale, converter)
print
if verbosity:
print()
if __name__ == '__main__':
sys.exit(main())
# pylint: disable=invalid-name
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--verbose", "-v", action="count", default=0)
args = parser.parse_args()
main(verbosity=args.verbose)
i18n/execute.py
import os, subprocess, logging
"""
Utility library file for executing shell commands
"""
import os
import subprocess
import logging
from i18n.config import BASE_DIR
LOG = logging.getLogger(__name__)
def execute(command, working_directory=BASE_DIR):
def execute(command, working_directory=BASE_DIR, stderr=subprocess.STDOUT):
"""
Executes shell command in a given working_directory.
Command is a string to pass to the shell.
......@@ -12,7 +18,7 @@ def execute(command, working_directory=BASE_DIR):
"""
LOG.info("Executing in %s ...", working_directory)
LOG.info(command)
subprocess.check_call(command, cwd=working_directory, stderr=subprocess.STDOUT, shell=True)
subprocess.check_call(command, cwd=working_directory, stderr=stderr, shell=True)
def call(command, working_directory=BASE_DIR):
......@@ -28,12 +34,6 @@ def call(command, working_directory=BASE_DIR):
return (out, err)
def create_dir_if_necessary(pathname):
dirname = os.path.dirname(pathname)
if not os.path.exists(dirname):
os.makedirs(dirname)
def remove_file(filename, verbose=True):
"""
Attempt to delete filename.
......
i18n/extract.py
......@@ -21,49 +21,68 @@ import os
import os.path
import logging
import sys
import argparse
from path import path
from polib import pofile
from i18n.config import BASE_DIR, LOCALE_DIR, CONFIGURATION
from i18n.execute import execute, create_dir_if_necessary, remove_file
from i18n.execute import execute, remove_file
from i18n.segment import segment_pofiles
EDX_MARKER = "edX translation file"
LOG = logging.getLogger(__name__)
DEVNULL = open(os.devnull, 'wb')
def base(path1, *paths):
"""Return a relative path from BASE_DIR to path1 / paths[0] / ... """
return BASE_DIR.relpathto(path1.joinpath(*paths))
def main():
def main(verbosity=1):
"""
Main entry point of script
"""
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
create_dir_if_necessary(LOCALE_DIR)
LOCALE_DIR.parent.makedirs_p()
source_msgs_dir = CONFIGURATION.source_messages_dir
remove_file(source_msgs_dir.joinpath('django.po'))
# Extract strings from mako templates.
babel_mako_cmd = 'pybabel extract -F {config} -c "Translators:" . -o {output}'
verbosity_map = {
0: "-q",
1: "",
2: "-v",
}
babel_verbosity = verbosity_map.get(verbosity, "")
babel_mako_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}'
babel_mako_cmd = babel_mako_cmd.format(
verbosity=babel_verbosity,
config=base(LOCALE_DIR, 'babel_mako.cfg'),
output=base(CONFIGURATION.source_messages_dir, 'mako.po'),
)
execute(babel_mako_cmd, working_directory=BASE_DIR)
if verbosity:
stderr = None
else:
stderr = DEVNULL
makemessages = "django-admin.py makemessages -l en"
execute(babel_mako_cmd, working_directory=BASE_DIR, stderr=stderr)
makemessages = "django-admin.py makemessages -l en -v{}".format(verbosity)
ignores = " ".join('--ignore="{}/*"'.format(d) for d in CONFIGURATION.ignore_dirs)
if ignores:
makemessages += " " + ignores
# Extract strings from django source files, including .py files.
make_django_cmd = makemessages + ' --extension html'
execute(make_django_cmd, working_directory=BASE_DIR)
execute(make_django_cmd, working_directory=BASE_DIR, stderr=stderr)
# Extract strings from Javascript source files.
make_djangojs_cmd = makemessages + ' -d djangojs --extension js'
execute(make_djangojs_cmd, working_directory=BASE_DIR)
execute(make_djangojs_cmd, working_directory=BASE_DIR, stderr=stderr)
# makemessages creates 'django.po'. This filename is hardcoded.
# Rename it to django-partial.po to enable merging into django.po later.
......@@ -90,13 +109,14 @@ def main():
output_file = source_msgs_dir / (app_name + ".po")
files_to_clean.add(output_file)
babel_cmd = 'pybabel extract -F {config} -c "Translators:" {app} -o {output}'
babel_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" {app} -o {output}'
babel_cmd = babel_cmd.format(
verbosity=babel_verbosity,
config=LOCALE_DIR / 'babel_third_party.cfg',
app=app_name,
output=output_file,
)
execute(babel_cmd, working_directory=app_dir)
execute(babel_cmd, working_directory=app_dir, stderr=stderr)
# Segment the generated files.
segmented_files = segment_pofiles("en")
......@@ -132,20 +152,24 @@ def fix_header(po):
fixes = (
('SOME DESCRIPTIVE TITLE', EDX_MARKER),
('Translations template for PROJECT.', EDX_MARKER),
('YEAR', '%s' % datetime.utcnow().year),
('YEAR', str(datetime.utcnow().year)),
('ORGANIZATION', 'edX'),
("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"),
('This file is distributed under the same license as the PROJECT project.',
'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'),
('This file is distributed under the same license as the PACKAGE package.',
'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'),
('FIRST AUTHOR <EMAIL@ADDRESS>',
'EdX Team <info@edx.org>')
)
(
'This file is distributed under the same license as the PROJECT project.',
'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'
),
(
'This file is distributed under the same license as the PACKAGE package.',
'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'
),
('FIRST AUTHOR <EMAIL@ADDRESS>', 'EdX Team <info@edx.org>'),
)
for src, dest in fixes:
header = header.replace(src, dest)
po.header = header
def fix_metadata(po):
"""
Replace default metadata with edX metadata
......@@ -168,12 +192,13 @@ def fix_metadata(po):
'PO-Revision-Date': datetime.utcnow(),
'Report-Msgid-Bugs-To': 'openedx-translation@googlegroups.com',
'Project-Id-Version': '0.1a',
'Language' : 'en',
'Last-Translator' : '',
'Language': 'en',
'Last-Translator': '',
'Language-Team': 'openedx-translation <openedx-translation@googlegroups.com>',
}
po.metadata.update(fixes)
def strip_key_strings(po):
"""
Removes all entries in PO which are key strings.
......@@ -183,6 +208,7 @@ def strip_key_strings(po):
del po[:]
po += newlist
def is_key_string(string):
"""
returns True if string is a key string.
......@@ -190,5 +216,10 @@ def is_key_string(string):
"""
return len(string) > 1 and string[0] == '_'
if __name__ == '__main__':
main()
# pylint: disable=invalid-name
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--verbose', '-v', action='count', default=0)
args = parser.parse_args()
main(verbosity=args.verbose)
i18n/generate.py
......@@ -24,6 +24,7 @@ from i18n.config import BASE_DIR, CONFIGURATION
from i18n.execute import execute
LOG = logging.getLogger(__name__)
DEVNULL = open(os.devnull, "wb")
def merge(locale, target='django.po', sources=('django-partial.po',), fail_if_missing=True):
......@@ -45,7 +46,7 @@ def merge(locale, target='django.po', sources=('django-partial.po',), fail_if_mi
except Exception, e:
if not fail_if_missing:
return
raise e
raise
# merged file is merged.po
merge_cmd = 'msgcat -o merged.po ' + ' '.join(sources)
......@@ -110,23 +111,31 @@ def validate_files(dir, files_to_merge):
raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname))
def main(argv=None):
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
parser = argparse.ArgumentParser(description="Generate merged and compiled message files.")
parser.add_argument("--strict", action='store_true', help="Complain about missing files.")
args = parser.parse_args(argv or [])
def main(strict=True, verbosity=1):
"""
Main entry point for script
"""
for locale in CONFIGURATION.translated_locales:
merge_files(locale, fail_if_missing=args.strict)
merge_files(locale, fail_if_missing=strict)
# Dummy text is not required. Don't raise exception if files are missing.
for locale in CONFIGURATION.dummy_locales:
merge_files(locale, fail_if_missing=False)
compile_cmd = 'django-admin.py compilemessages'
execute(compile_cmd, working_directory=BASE_DIR)
compile_cmd = 'django-admin.py compilemessages -v{}'.format(verbosity)
if verbosity:
stderr = None
else:
stderr = DEVNULL
execute(compile_cmd, working_directory=BASE_DIR, stderr=stderr)
if __name__ == '__main__':
main(sys.argv[1:])
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
# pylint: disable=invalid-name
parser = argparse.ArgumentParser(description="Generate merged and compiled message files.")
parser.add_argument("--strict", action='store_true', help="Complain about missing files.")
parser.add_argument("--verbose", "-v", action="count", default=0)
args = parser.parse_args()
main(strict=args.strict, verbosity=args.verbose)
i18n/segment.py
......@@ -8,8 +8,9 @@ import copy
import fnmatch
import logging
import sys
import argparse
import polib
import textwrap
from i18n.config import CONFIGURATION
......@@ -116,27 +117,32 @@ def segment_pofile(filename, segments):
return files_written
def main(argv):
def main(locales=None, verbosity=1): # pylint: disable=unused-argument
"""
$ segment.py LOCALE [...]
Segment the .po files in LOCALE(s) based on the segmenting rules in
config.yaml.
Note that segmenting is *not* idempotent: it modifies the input file, so
be careful that you don't run it twice on the same file.
Main entry point of script
"""
# This is used as a tool only to segment translation files when adding a
# new segment. In the regular workflow, the work is done by the extract
# phase calling the functions above.
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
if len(argv) < 2:
sys.exit("Need a locale to segment")
for locale in argv[1:]:
locales = locales or []
for locale in locales:
segment_pofiles(locale)
if __name__ == "__main__":
main(sys.argv)
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
# pylint: disable=invalid-name
description = textwrap.dedent("""
Segment the .po files in LOCALE(s) based on the segmenting rules in
config.yaml.
Note that segmenting is *not* idempotent: it modifies the input file, so
be careful that you don't run it twice on the same file.
""".strip())
parser = argparse.ArgumentParser(description=description)
parser.add_argument("locale", nargs="+", help="a locale to segment")
parser.add_argument("--verbose", "-v", action="count", default=0)
args = parser.parse_args()
main(locales=args.locale, verbosity=args.verbose)
i18n/tests/test_extract.py
......@@ -33,7 +33,7 @@ class TestExtract(TestCase):
super(TestExtract, self).setUp()
if not SETUP_HAS_RUN:
# Run extraction script. Warning, this takes 1 minute or more
extract.main()
extract.main(verbosity=0)
SETUP_HAS_RUN = True
def get_files(self):
......
i18n/tests/test_generate.py
from datetime import datetime, timedelta
import os
import sys
import string
import random
import re
......@@ -23,8 +24,13 @@ class TestGenerate(TestCase):
@classmethod
def setUpClass(cls):
extract.main()
dummy.main()
sys.stderr.write(
"\nExtracting i18n strings and generating dummy translations; "
"this may take a few minutes\n"
)
sys.stderr.flush()
extract.main(verbosity=0)
dummy.main(verbosity=0)
def setUp(self):
# Subtract 1 second to help comparisons with file-modify time succeed,
......@@ -50,7 +56,7 @@ class TestGenerate(TestCase):
.mo files should exist, and be recently created (modified
after start of test suite)
"""
generate.main()
generate.main(verbosity=0, strict=False)
for locale in CONFIGURATION.translated_locales:
for filename in ('django', 'djangojs'):
mofile = filename+'.mo'
......
i18n/transifex.py
#!/usr/bin/env python
from __future__ import print_function
import sys
from polib import pofile
import argparse
from i18n.config import CONFIGURATION
from i18n.execute import execute
from i18n.extract import EDX_MARKER
TRANSIFEX_HEADER = 'edX community translations have been downloaded from %s'
TRANSIFEX_HEADER = 'edX community translations have been downloaded from {}'
TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-platform/'
......@@ -16,7 +17,7 @@ def push():
def pull():
print "Pulling languages from transifex..."
print("Pulling languages from transifex...")
execute('tx pull --mode=reviewed --all')
clean_translated_locales()
......@@ -57,18 +58,22 @@ def clean_file(filename):
def get_new_header(po):
team = po.metadata.get('Language-Team', None)
if not team:
return TRANSIFEX_HEADER % TRANSIFEX_URL
return TRANSIFEX_HEADER.format(TRANSIFEX_URL)
else:
return TRANSIFEX_HEADER % team
return TRANSIFEX_HEADER.format(team)
if __name__ == '__main__':
if len(sys.argv) < 2:
raise Exception("missing argument: push or pull")
arg = sys.argv[1]
if arg == 'push':
# pylint: disable=invalid-name
parser = argparse.ArgumentParser()
parser.add_argument("command", help="push or pull")
parser.add_argument("--verbose", "-v")
args = parser.parse_args()
# pylint: enable=invalid-name
if args.command == "push":
push()
elif arg == 'pull':
elif args.command == "pull":
pull()
else:
raise Exception("unknown argument: (%s)" % arg)
raise Exception("unknown command ({cmd})".format(cmd=args.command))
i18n/validate.py
......@@ -16,6 +16,7 @@ from i18n.converter import Converter
log = logging.getLogger(__name__)
def validate_po_files(root, report_empty=False):
"""
Validate all of the po files found in the root directory.
......@@ -148,20 +149,14 @@ def check_messages(filename, report_empty=False):
log.info(" No problems found in {0}".format(filename))
def parse_args(argv):
def get_parser():
"""
Parse command line arguments, returning a dict of
valid options:
{
'empty': BOOLEAN,
'verbose': BOOLEAN,
'language': str
}
where 'language' is a language code, eg "fr"
Returns an argument parser for this script.
"""
parser = argparse.ArgumentParser(description="Automatically finds translation errors in all edx-platform *.po files, for all languages, unless one or more language(s) is specified to check.")
parser = argparse.ArgumentParser(description=( # pylint: disable=redefined-outer-name
"Automatically finds translation errors in all edx-platform *.po files, "
"for all languages, unless one or more language(s) is specified to check."
))
parser.add_argument(
'-l', '--language',
......@@ -178,39 +173,44 @@ def parse_args(argv):
parser.add_argument(
'-v', '--verbose',
action='store_true',
action='count', default=0,
help="Turns on info-level logging."
)
return vars(parser.parse_args(argv))
return parser
def main():
"""Main entry point for the tool."""
def main(languages=None, empty=False, verbosity=1): # pylint: disable=unused-argument
"""
Main entry point for script
"""
languages = languages or []
args_dict = parse_args(sys.argv[1:])
if args_dict['verbose']:
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
else:
logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
if not languages:
root = LOCALE_DIR
validate_po_files(root, empty)
return
langs = args_dict['language']
# languages will be a list of language codes; test each language.
for language in languages:
root = LOCALE_DIR / language
# Assert that a directory for this language code exists on the system
if not root.isdir():
log.error(" {0} is not a valid directory.\nSkipping language '{1}'".format(root, language))
continue
# If we found the language code's directory, validate the files.
validate_po_files(root, empty)
if langs is not None:
# lang will be a list of language codes; test each language.
for lang in langs:
root = LOCALE_DIR / lang
# Assert that a directory for this language code exists on the system
if not os.path.isdir(root):
log.error(" {0} is not a valid directory.\nSkipping language '{1}'".format(root, lang))
continue
# If we found the language code's directory, validate the files.
validate_po_files(root, args_dict['empty'])
if __name__ == '__main__':
# pylint: disable=invalid-name
parser = get_parser()
args = parser.parse_args()
if args.verbose:
log_level = logging.INFO
else:
# If lang is None, we walk all of the .po files under root, and test each one.
root = LOCALE_DIR
validate_po_files(root, args_dict['empty'])
log_level = logging.WARNING
logging.basicConfig(stream=sys.stdout, level=log_level)
# pylint: enable=invalid-name
if __name__ == '__main__':
main()
main(languages=args.language, empty=args.empty, verbosity=args.verbose)
rakelib/i18n.rake
......@@ -10,7 +10,11 @@ namespace :i18n do
desc "Extract localizable strings from sources"
task :extract => ["i18n:validate:gettext", "assets:coffee"] do
sh(File.join(REPO_ROOT, "i18n", "extract.py"))
command = File.join(REPO_ROOT, "i18n", "extract.py")
if verbose == true
command += " -vv"
end
sh(command)
end
desc "Compile localizable strings from sources, extracting strings first."
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment