Fixing up tests, removing some of the yaml error stuff from parsing

* moved old unittests for vault over to the new codebase
* reverted YAML error helpers and reverted the load() function
  in parsing/__init__.py, pending a rewrite of a new YAML loader
  class of some kind to encapsulate all of that
* fixed an error in in the module args parser regarding the shell/
  command argument parsing, where some additional arguments were
  being lost

v2/ansible/errors/__init__.py

@@ -20,10 +20,11 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
-from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
 
 class AnsibleError(Exception):
     def __init__(self, message, obj=None):
+        # we import this here to prevent an import loop with errors
+        from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
         self._obj     = obj
         if isinstance(self._obj, AnsibleBaseYAMLObject):
             extended_error = self._get_extended_error()

v2/ansible/parsing/__init__.py

@@ -28,157 +28,7 @@ from ansible.parsing.vault import VaultLib
 from ansible.parsing.yaml import safe_load
 
 
-def process_common_errors(msg, probline, column):
-    replaced = probline.replace(" ","")
-
-    if ":{{" in replaced and "}}" in replaced:
-        msg = msg + """
-This one looks easy to fix.  YAML thought it was looking for the start of a
-hash/dictionary and was confused to see a second "{".  Most likely this was
-meant to be an ansible template evaluation instead, so we have to give the
-parser a small hint that we wanted a string instead. The solution here is to
-just quote the entire value.
-
-For instance, if the original line was:
-
-    app_path: {{ base_path }}/foo
-
-It should be written as:
-
-    app_path: "{{ base_path }}/foo"
-"""
-        return msg
-
-    elif len(probline) and len(probline) > 1 and len(probline) > column and probline[column] == ":" and probline.count(':') > 1:
-        msg = msg + """
-This one looks easy to fix.  There seems to be an extra unquoted colon in the line
-and this is confusing the parser. It was only expecting to find one free
-colon. The solution is just add some quotes around the colon, or quote the
-entire line after the first colon.
-
-For instance, if the original line was:
-
-    copy: src=file.txt dest=/path/filename:with_colon.txt
-
-It can be written as:
-
-    copy: src=file.txt dest='/path/filename:with_colon.txt'
-
-Or:
-
-    copy: 'src=file.txt dest=/path/filename:with_colon.txt'
-
-
-"""
-        return msg
-    else:
-        parts = probline.split(":")
-        if len(parts) > 1:
-            middle = parts[1].strip()
-            match = False
-            unbalanced = False
-            if middle.startswith("'") and not middle.endswith("'"):
-                match = True
-            elif middle.startswith('"') and not middle.endswith('"'):
-                match = True
-            if len(middle) > 0 and middle[0] in [ '"', "'" ] and middle[-1] in [ '"', "'" ] and probline.count("'") > 2 or probline.count('"') > 2:
-                unbalanced = True
-            if match:
-                msg = msg + """
-This one looks easy to fix.  It seems that there is a value started
-with a quote, and the YAML parser is expecting to see the line ended
-with the same kind of quote.  For instance:
-
-    when: "ok" in result.stdout
-
-Could be written as:
-
-   when: '"ok" in result.stdout'
-
-or equivalently:
-
-   when: "'ok' in result.stdout"
-
-"""
-                return msg
-
-            if unbalanced:
-                msg = msg + """
-We could be wrong, but this one looks like it might be an issue with
-unbalanced quotes.  If starting a value with a quote, make sure the
-line ends with the same set of quotes.  For instance this arbitrary
-example:
-
-    foo: "bad" "wolf"
-
-Could be written as:
-
-    foo: '"bad" "wolf"'
-
-"""
-                return msg
-
-    return msg
-
-def process_yaml_error(exc, data, path=None, show_content=True):
-    if hasattr(exc, 'problem_mark'):
-        mark = exc.problem_mark
-        if show_content:
-            if mark.line -1 >= 0:
-                before_probline = data.split("\n")[mark.line-1]
-            else:
-                before_probline = ''
-            probline = data.split("\n")[mark.line]
-            arrow = " " * mark.column + "^"
-            msg = """Syntax Error while loading YAML script, %s
-Note: The error may actually appear before this position: line %s, column %s
-
-%s
-%s
-%s""" % (path, mark.line + 1, mark.column + 1, before_probline, probline, arrow)
-
-            unquoted_var = None
-            if '{{' in probline and '}}' in probline:
-                if '"{{' not in probline or "'{{" not in probline:
-                    unquoted_var = True
-
-            if not unquoted_var:
-                msg = process_common_errors(msg, probline, mark.column)
-            else:
-                msg = msg + """
-We could be wrong, but this one looks like it might be an issue with
-missing quotes.  Always quote template expression brackets when they
-start a value. For instance:
-
-    with_items:
-      - {{ foo }}
-
-Should be written as:
-
-    with_items:
-      - "{{ foo }}"
-
-"""
-        else:
-            # most likely displaying a file with sensitive content,
-            # so don't show any of the actual lines of yaml just the
-            # line number itself
-            msg = """Syntax error while loading YAML script, %s
-The error appears to have been on line %s, column %s, but may actually
-be before there depending on the exact syntax problem.
-""" % (path, mark.line + 1, mark.column + 1)
-
-    else:
-        # No problem markers means we have to throw a generic
-        # "stuff messed up" type message. Sry bud.
-        if path:
-            msg = "Could not parse YAML. Check over %s again." % path
-        else:
-            msg = "Could not parse YAML."
-    raise errors.AnsibleYAMLValidationFailed(msg)
-
-
-def load_data(data):
+def load(data):
 
     if isinstance(data, file):
         fd = open(f)
@@ -193,31 +43,3 @@ def load_data(data):
 
     raise AnsibleInternalError("expected file or string, got %s" % type(data))
 
-def load_data_from_file(path, vault_password=None):
-    '''
-    Convert a yaml file to a data structure. 
-    Was previously 'parse_yaml_from_file()'.
-    '''
-
-    data = None
-    show_content = True
-
-    try:
-        data = open(path).read()
-    except IOError:
-        raise errors.AnsibleError("file could not read: %s" % path)
-
-    vault = VaultLib(password=vault_password)
-    if vault.is_encrypted(data):
-        # if the file is encrypted and no password was specified,
-        # the decrypt call would throw an error, but we check first
-        # since the decrypt function doesn't know the file name
-        if vault_password is None:
-            raise errors.AnsibleError("A vault password must be specified to decrypt %s" % path)
-        data = vault.decrypt(data)
-        show_content = False
-
-    try:
-        return load_data(data)
-    except YAMLError as exc:
-        process_yaml_error(exc, data, path, show_content)

v2/ansible/parsing/mod_args.py

@@ -94,18 +94,13 @@ class ModuleArgsParser:
         if action not in ['shell', 'command']:
             return (action, args)
 
-        new_args = {}
-
         # the shell module really is the command module with an additional
         # parameter
         if action == 'shell':
             action = 'command'
-            new_args['_uses_shell'] = True
-
-        # make sure the non-key-value params hop in the data
-        new_args['_raw_params'] = args['_raw_params']
+            args['_uses_shell'] = True
 
-        return (action, new_args)
+        return (action, args)
 
     def _normalize_parameters(self, thing, action=None):
         '''

v2/ansible/playbook/base.py

@@ -25,7 +25,7 @@ from io import FileIO
 from six import iteritems, string_types
 
 from ansible.playbook.attribute import Attribute, FieldAttribute
-from ansible.parsing import load_data
+from ansible.parsing import load
 
 class Base:
 
@@ -64,7 +64,7 @@ class Base:
         assert ds is not None
 
         if isinstance(ds, string_types) or isinstance(ds, FileIO):
-            ds = load_data(ds)
+            ds = load(ds)
 
         # we currently don't do anything with private attributes but may
         # later decide to filter them out of 'ds' here.

v2/ansible/playbook/role.py

@@ -26,9 +26,12 @@ import os
 from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.block import Block
-from ansible.parsing import load_data_from_file
 from ansible.errors import AnsibleError
 
+# FIXME: this def was cruft from the old utils code, so we'll need
+#        to relocate it somewhere before we can use it
+#from ansible.parsing import load_data_from_file
+
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
 
 class Role(Base):

v2/test/parsing/test_general.py

@@ -20,8 +20,8 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from ansible.compat.tests import unittest
-from ansible.parsing import load_data
 from ansible.errors import AnsibleParserError
+from ansible.parsing import load
 
 import json
 

v2/test/parsing/vault/__init__.py

+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+

v2/test/parsing/vault/test_vault.py

+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import getpass
+import os
+import shutil
+import time
+import tempfile
+from binascii import unhexlify
+from binascii import hexlify
+from nose.plugins.skip import SkipTest
+
+from ansible.compat.tests import unittest
+
+from ansible import errors
+from ansible.parsing.vault import VaultLib
+
+# Counter import fails for 2.0.1, requires >= 2.6.1 from pip
+try:
+    from Crypto.Util import Counter
+    HAS_COUNTER = True
+except ImportError:
+    HAS_COUNTER = False
+
+# KDF import fails for 2.0.1, requires >= 2.6.1 from pip
+try:
+    from Crypto.Protocol.KDF import PBKDF2
+    HAS_PBKDF2 = True
+except ImportError:
+    HAS_PBKDF2 = False
+
+# AES IMPORTS
+try:
+    from Crypto.Cipher import AES as AES
+    HAS_AES = True
+except ImportError:
+    HAS_AES = False
+
+class TestVaultLib(unittest.TestCase):
+
+    def test_methods_exist(self):
+        v = VaultLib('ansible')
+        slots = ['is_encrypted',
+                 'encrypt',
+                 'decrypt',
+                 '_add_header',
+                 '_split_header',]
+        for slot in slots:         
+            assert hasattr(v, slot), "VaultLib is missing the %s method" % slot
+
+    def test_is_encrypted(self):
+        v = VaultLib(None)
+        assert not v.is_encrypted("foobar"), "encryption check on plaintext failed"
+        data = "$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify("ansible")
+        assert v.is_encrypted(data), "encryption check on headered text failed"
+
+    def test_add_header(self):
+        v = VaultLib('ansible')
+        v.cipher_name = "TEST"
+        sensitive_data = "ansible"
+        data = v._add_header(sensitive_data)
+        lines = data.split('\n')
+        assert len(lines) > 1, "failed to properly add header"
+        header = lines[0]
+        assert header.endswith(';TEST'), "header does end with cipher name"
+        header_parts = header.split(';')
+        assert len(header_parts) == 3, "header has the wrong number of parts"        
+        assert header_parts[0] == '$ANSIBLE_VAULT', "header does not start with $ANSIBLE_VAULT"
+        assert header_parts[1] == v.version, "header version is incorrect"
+        assert header_parts[2] == 'TEST', "header does end with cipher name"
+
+    def test_split_header(self):
+        v = VaultLib('ansible')
+        data = "$ANSIBLE_VAULT;9.9;TEST\nansible" 
+        rdata = v._split_header(data)        
+        lines = rdata.split('\n')
+        assert lines[0] == "ansible"
+        assert v.cipher_name == 'TEST', "cipher name was not set"
+        assert v.version == "9.9"
+
+    def test_encrypt_decrypt_aes(self):
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
+            raise SkipTest
+        v = VaultLib('ansible')
+        v.cipher_name = 'AES'
+        enc_data = v.encrypt("foobar")
+        dec_data = v.decrypt(enc_data)
+        assert enc_data != "foobar", "encryption failed"
+        assert dec_data == "foobar", "decryption failed"           
+
+    def test_encrypt_decrypt_aes256(self):
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
+            raise SkipTest
+        v = VaultLib('ansible')
+        v.cipher_name = 'AES256'
+        enc_data = v.encrypt("foobar")
+        dec_data = v.decrypt(enc_data)
+        assert enc_data != "foobar", "encryption failed"
+        assert dec_data == "foobar", "decryption failed"           
+
+    def test_encrypt_encrypted(self):
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
+            raise SkipTest
+        v = VaultLib('ansible')
+        v.cipher_name = 'AES'
+        data = "$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify("ansible")
+        error_hit = False
+        try:
+            enc_data = v.encrypt(data)
+        except errors.AnsibleError, e:
+            error_hit = True
+        assert error_hit, "No error was thrown when trying to encrypt data with a header"    
+
+    def test_decrypt_decrypted(self):
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
+            raise SkipTest
+        v = VaultLib('ansible')
+        data = "ansible"
+        error_hit = False
+        try:
+            dec_data = v.decrypt(data)
+        except errors.AnsibleError, e:
+            error_hit = True
+        assert error_hit, "No error was thrown when trying to decrypt data without a header"    
+
+    def test_cipher_not_set(self):
+        # not setting the cipher should default to AES256
+        if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
+            raise SkipTest
+        v = VaultLib('ansible')
+        data = "ansible"
+        error_hit = False
+        try:
+            enc_data = v.encrypt(data)
+        except errors.AnsibleError, e:
+            error_hit = True
+        assert not error_hit, "An error was thrown when trying to encrypt data without the cipher set"    
+        assert v.cipher_name == "AES256", "cipher name is not set to AES256: %s" % v.cipher_name