Merge branch 'develop' of github.com:nltk/nltk into bleu_fix

nltk/internals.py

@@ -16,6 +16,7 @@ import textwrap
 import types
 import sys
 import stat
+import locale
 
 # Use the c version of ElementTree, which is faster, if possible:
 try:
@@ -156,7 +157,7 @@ def java(cmd, classpath=None, stdin=None, stdout=None, stderr=None,
 
     # Check the return code.
     if p.returncode != 0:
-        print(stderr.decode(sys.stdout.encoding))
+        print(_decode_stdoutdata(stderr))
         raise OSError('Java command failed!')
 
     return (stdout, stderr)
@@ -492,7 +493,7 @@ def find_file_iter(filename, env_vars=(), searchpath=(),
             try:
                 p = subprocess.Popen(['which', alternative], stdout=subprocess.PIPE)
                 stdout, stderr = p.communicate()
-                path = stdout.decode(sys.stdout.encoding).strip()
+                path = _decode_stdoutdata(stdout).strip()
                 if path.endswith(alternative) and os.path.exists(path):
                     if verbose:
                         print('[Found %s: %s]' % (filename, path))
@@ -643,6 +644,14 @@ def find_jar(name_pattern, path_to_jar=None, env_vars=(),
     return next(find_jar_iter(name_pattern, path_to_jar, env_vars,
                          searchpath, url, verbose, is_regex))
 
+
+def _decode_stdoutdata(stdoutdata):
+    """ Convert data read from stdout/stderr to unicode """
+    if not isinstance(stdoutdata, bytes):
+        return stdoutdata
+    encoding = getattr(sys.stdout, "encoding", locale.getpreferredencoding())
+    return stdoutdata.decode(encoding)
+
 ##########################################################################
 # Import Stdlib Module
 ##########################################################################