Fix for unicode decoding error in new splitting code

Also adds a new unicode integration test

lib/ansible/utils/splitter.py

@@ -70,7 +70,11 @@ def split_args(args):
 
     # here we encode the args, so we have a uniform charset to
     # work with, and split on white space
-    args = args.encode('utf-8')
+    try:
+        args = args.encode('utf-8')
+        do_decode = True
+    except UnicodeDecodeError:
+        do_decode = False
     tokens = args.split()
 
     # iterate over the tokens, and reassemble any that may have been
@@ -146,7 +150,8 @@ def split_args(args):
         raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")
 
     # finally, we decode each param back to the unicode it was in the arg string
-    params = [x.decode('utf-8') for x in params]
+    if do_decode:
+        params = [x.decode('utf-8') for x in params]
     return params
 
 def unquote(data):

test/integration/Makefile

@@ -23,6 +23,9 @@ parsing:
 	ansible-playbook bad_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags common,scenario4; [ $$? -eq 3 ]
 	ansible-playbook good_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
+unicode:
+	ansible-playbook unicode.yml -i $(INVENTORY) -e @$(VARS_FILE) -v $(TEST_FLAGS)
+
 non_destructive:
 	ansible-playbook non_destructive.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 

test/integration/unicode.yml

+---
+- name: 'A play with unicode: ¢ £ ¤ ¥'
+  hosts: localhost
+  connection: local
+  vars:
+    test_var: 'Ī ī Ĭ ĭ Į į İ ı IJ ij Ĵ ĵ Ķ ķ ĸ Ĺ ĺ Ļ ļ Ľ ľ Ŀ ŀ Ł ł Ń ń Ņ ņ Ň ň ʼn Ŋ ŋ Ō ō Ŏ ŏ Ő ő Œ'
+    num_hosts: 5
+    hostnames:
+        - 'host-#ϬϭϮϯϰ'
+        - 'host-ͰͱͲͳʹ͵'
+        - 'host-ΙΚΛΜΝΞ'
+        - 'host-στυφχψ'
+        - 'host-ϬϭϮϯϰϱ'
+
+  tasks:
+    - name: 'A task name with unicode: è é ê ë'
+      debug: msg='hi there'
+
+    - name: 'A task with unicode parameters'
+      debug: var=test_var
+
+      # € ‚ ƒ „ … † ‡ ˆ ‰ Š ‹ Œ Ž ‘ ’ “ ” • – — ˜ ™ š › œ ž Ÿ ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ®'
+
+    - name: 'A task using with_items containing unicode'
+      debug: msg='{{item}}'
+      with_items:
+        - '¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö ×'
+        - 'Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ Ā'
+        - 'ā Ă ă Ą ą Ć ć Ĉ ĉ Ċ ċ Č č Ď ď Đ đ Ē ē Ĕ ĕ Ė ė Ę ę Ě ě Ĝ ĝ Ğ ğ Ġ ġ Ģ ģ Ĥ ĥ Ħ ħ Ĩ ĩ'
+
+    - add_host:
+        name: '{{hostnames}}.{{item}}'
+        groups: 'ĪīĬĭ'
+        ansible_connection: local
+        host_id: '{{item}}'
+      with_sequence: start=1 end={{num_hosts}} format=%d
+
+- name: 'A play for hosts in group: ĪīĬĭ'
+  hosts: 'ĪīĬĭ'
+  gather_facts: true
+  tasks:
+    - debug: msg='Unicode is a good thing ™'