Skip to content

A
ansible
Commit 339a02c3 by James Cammarata
Options

Started reworking module_utils/basic unit tests (v2)

parent f1ab1c48
Showing with 227 additions and 287 deletions
lib/ansible/module_utils/basic.py
...@@ -930,7 +930,7 @@ class AnsibleModule(object): ...@@ -930,7 +930,7 @@ class AnsibleModule(object):
for check in spec: for check in spec:
count = self._count_terms(check) count = self._count_terms(check)
if count > 1: if count > 1:
self.fail_json(msg="parameters are mutually exclusive: %s" % check) self.fail_json(msg="parameters are mutually exclusive: %s" % (check,))
def _check_required_one_of(self, spec): def _check_required_one_of(self, spec):
if spec is None: if spec is None:
...@@ -948,7 +948,7 @@ class AnsibleModule(object): ...@@ -948,7 +948,7 @@ class AnsibleModule(object):
non_zero = [ c for c in counts if c > 0 ] non_zero = [ c for c in counts if c > 0 ]
if len(non_zero) > 0: if len(non_zero) > 0:
if 0 in counts: if 0 in counts:
self.fail_json(msg="parameters are required together: %s" % check) self.fail_json(msg="parameters are required together: %s" % (check,))
def _check_required_arguments(self): def _check_required_arguments(self):
''' ensure all required arguments are present ''' ''' ensure all required arguments are present '''
......
test/units/module_utils/test_basic.py
# -*- coding: utf-8 -*-
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> # (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# #
# This file is part of Ansible # This file is part of Ansible
...@@ -16,301 +17,167 @@ ...@@ -16,301 +17,167 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# Make coding more python3-ish # Make coding more python3-ish
#from __future__ import (absolute_import, division, print_function)
from __future__ import (absolute_import, division) from __future__ import (absolute_import, division)
__metaclass__ = type __metaclass__ = type
import os import __builtin__
import tempfile
from nose.tools import timed
from ansible.compat.tests import unittest from ansible.compat.tests import unittest
from ansible.compat.tests.mock import patch, MagicMock from ansible.compat.tests.mock import patch, MagicMock
from ansible.errors import *
from ansible.executor.module_common import modify_module
from ansible.module_utils.basic import heuristic_log_sanitize
from ansible.utils.hashing import checksum as utils_checksum
TEST_MODULE_DATA = """
from ansible.module_utils.basic import *
def get_module():
return AnsibleModule(
argument_spec = dict(),
supports_check_mode = True,
no_log = True,
)
get_module()
"""
class TestModuleUtilsBasic(unittest.TestCase): class TestModuleUtilsBasic(unittest.TestCase):
def cleanup_temp_file(self, fd, path):
try:
os.close(fd)
os.remove(path)
except:
pass
def cleanup_temp_dir(self, path):
try:
os.rmdir(path)
except:
pass
def setUp(self): def setUp(self):
# create a temporary file for the test module pass
# we're about to generate
self.tmp_fd, self.tmp_path = tempfile.mkstemp()
os.write(self.tmp_fd, TEST_MODULE_DATA)
# template the module code and eval it
module_data, module_style, shebang = modify_module(self.tmp_path, {})
d = {}
exec(module_data, d, d)
self.module = d['get_module']()
# module_utils/basic.py screws with CWD, let's save it and reset
self.cwd = os.getcwd()
def tearDown(self): def tearDown(self):
self.cleanup_temp_file(self.tmp_fd, self.tmp_path) pass
# Reset CWD back to what it was before basic.py changed it
os.chdir(self.cwd) def test_module_utils_basic_imports(self):
realimport = __builtin__.__import__
#################################################################################
# run_command() tests def _mock_import(name, *args, **kwargs):
if name == 'json':
# test run_command with a string command raise ImportError()
def test_run_command_string(self): realimport(name, *args, **kwargs)
(rc, out, err) = self.module.run_command("/bin/echo -n 'foo bar'")
self.assertEqual(rc, 0) with patch.object(__builtin__, '__import__', _mock_import, create=True) as m:
self.assertEqual(out, 'foo bar') m('ansible.module_utils.basic')
(rc, out, err) = self.module.run_command("/bin/echo -n 'foo bar'", use_unsafe_shell=True) __builtin__.__import__('ansible.module_utils.basic')
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar') def test_module_utils_basic_get_platform(self):
with patch('platform.system', return_value='foo'):
# test run_command with an array of args (with both use_unsafe_shell=True|False) from ansible.module_utils.basic import get_platform
def test_run_command_args(self): self.assertEqual(get_platform(), 'foo')
(rc, out, err) = self.module.run_command(['/bin/echo', '-n', "foo bar"])
self.assertEqual(rc, 0) def test_module_utils_basic_get_distribution(self):
self.assertEqual(out, 'foo bar') from ansible.module_utils.basic import get_distribution
(rc, out, err) = self.module.run_command(['/bin/echo', '-n', "foo bar"], use_unsafe_shell=True)
self.assertEqual(rc, 0) with patch('platform.system', return_value='Foo'):
self.assertEqual(out, 'foo bar') self.assertEqual(get_distribution(), None)
# test run_command with leading environment variables with patch('platform.system', return_value='Linux'):
#@raises(SystemExit) with patch('platform.linux_distribution', return_value=("foo", "1", "One")):
def test_run_command_string_with_env_variables(self): self.assertEqual(get_distribution(), "Foo")
self.assertRaises(SystemExit, self.module.run_command, 'FOO=bar /bin/echo -n "foo bar"')
with patch('os.path.isfile', return_value=True):
#@raises(SystemExit) def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
def test_run_command_args_with_env_variables(self): if supported_dists != ():
self.assertRaises(SystemExit, self.module.run_command, ['FOO=bar', '/bin/echo', '-n', 'foo bar']) return ("AmazonFooBar", "", "")
else:
def test_run_command_string_unsafe_with_env_variables(self): return ("", "", "")
(rc, out, err) = self.module.run_command('FOO=bar /bin/echo -n "foo bar"', use_unsafe_shell=True)
self.assertEqual(rc, 0) with patch('platform.linux_distribution', side_effect=_dist):
self.assertEqual(out, 'foo bar') self.assertEqual(get_distribution(), "Amazon")
# test run_command with a command pipe (with both use_unsafe_shell=True|False) def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
def test_run_command_string_unsafe_with_pipe(self): if supported_dists != ():
(rc, out, err) = self.module.run_command('echo "foo bar" | cat', use_unsafe_shell=True) return ("Bar", "2", "Two")
self.assertEqual(rc, 0) else:
self.assertEqual(out, 'foo bar\n') return ("", "", "")
# test run_command with a shell redirect in (with both use_unsafe_shell=True|False) with patch('platform.linux_distribution', side_effect=_dist):
def test_run_command_string_unsafe_with_redirect_in(self): self.assertEqual(get_distribution(), "OtherLinux")
(rc, out, err) = self.module.run_command('cat << EOF\nfoo bar\nEOF', use_unsafe_shell=True)
self.assertEqual(rc, 0) with patch('platform.linux_distribution', side_effect=Exception("boo")):
self.assertEqual(out, 'foo bar\n') with patch('platform.dist', return_value=("bar", "2", "Two")):
self.assertEqual(get_distribution(), "Bar")
# test run_command with a shell redirect out (with both use_unsafe_shell=True|False)
def test_run_command_string_unsafe_with_redirect_out(self): def test_module_utils_basic_get_distribution_version(self):
tmp_fd, tmp_path = tempfile.mkstemp() from ansible.module_utils.basic import get_distribution_version
try:
(rc, out, err) = self.module.run_command('echo "foo bar" > %s' % tmp_path, use_unsafe_shell=True) with patch('platform.system', return_value='Foo'):
self.assertEqual(rc, 0) self.assertEqual(get_distribution_version(), None)
self.assertTrue(os.path.exists(tmp_path))
checksum = utils_checksum(tmp_path) with patch('platform.system', return_value='Linux'):
self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec') with patch('platform.linux_distribution', return_value=("foo", "1", "One")):
except: self.assertEqual(get_distribution_version(), "1")
raise
finally: with patch('os.path.isfile', return_value=True):
self.cleanup_temp_file(tmp_fd, tmp_path) def _dist(distname='', version='', id='', supported_dists=(), full_distribution_name=1):
if supported_dists != ():
# test run_command with a double shell redirect out (append) (with both use_unsafe_shell=True|False) return ("AmazonFooBar", "2", "")
def test_run_command_string_unsafe_with_double_redirect_out(self): else:
tmp_fd, tmp_path = tempfile.mkstemp() return ("", "", "")
try:
(rc, out, err) = self.module.run_command('echo "foo bar" >> %s' % tmp_path, use_unsafe_shell=True) with patch('platform.linux_distribution', side_effect=_dist):
self.assertEqual(rc, 0) self.assertEqual(get_distribution_version(), "2")
self.assertTrue(os.path.exists(tmp_path))
checksum = utils_checksum(tmp_path) with patch('platform.linux_distribution', side_effect=Exception("boo")):
self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec') with patch('platform.dist', return_value=("bar", "3", "Three")):
except: self.assertEqual(get_distribution_version(), "3")
raise
finally: def test_module_utils_basic_load_platform_subclass(self):
self.cleanup_temp_file(tmp_fd, tmp_path) class LinuxTest:
# test run_command with data
def test_run_command_string_with_data(self):
(rc, out, err) = self.module.run_command('cat', data='foo bar')
self.assertEqual(rc, 0)
self.assertEqual(out, 'foo bar\n')
# test run_command with binary data
def test_run_command_string_with_binary_data(self):
(rc, out, err) = self.module.run_command('cat', data='\x41\x42\x43\x44', binary_data=True)
self.assertEqual(rc, 0)
self.assertEqual(out, 'ABCD')
# test run_command with a cwd set
def test_run_command_string_with_cwd(self):
tmp_path = tempfile.mkdtemp()
try:
(rc, out, err) = self.module.run_command('pwd', cwd=tmp_path)
self.assertEqual(rc, 0)
self.assertTrue(os.path.exists(tmp_path))
self.assertEqual(out.strip(), os.path.realpath(tmp_path))
except:
raise
finally:
self.cleanup_temp_dir(tmp_path)
class TestModuleUtilsBasicHelpers(unittest.TestCase):
''' Test some implementation details of AnsibleModule
Some pieces of AnsibleModule are implementation details but they have
potential cornercases that we need to check. Go ahead and test at
this level that the functions are behaving even though their API may
change and we'd have to rewrite these tests so that we know that we
need to check for those problems in any rewrite.
In the future we might want to restructure higher level code to be
friendlier to unittests so that we can test at the level that the public
is interacting with the APIs.
'''
MANY_RECORDS = 7000
URL_SECRET = 'http://username:pas:word@foo.com/data'
SSH_SECRET = 'username:pas:word@foo.com/data'
def cleanup_temp_file(self, fd, path):
try:
os.close(fd)
os.remove(path)
except:
pass pass
def cleanup_temp_dir(self, path): class Foo(LinuxTest):
try: platform = "Linux"
os.rmdir(path) distribution = None
except:
pass class Bar(LinuxTest):
platform = "Linux"
def _gen_data(self, records, per_rec, top_level, secret_text): distribution = "Bar"
hostvars = {'hostvars': {}}
for i in range(1, records, 1): from ansible.module_utils.basic import load_platform_subclass
host_facts = {'host%s' % i:
{'pstack': # match just the platform class, not a specific distribution
{'running': '875.1', with patch('ansible.module_utils.basic.get_platform', return_value="Linux"):
'symlinked': '880.0', with patch('ansible.module_utils.basic.get_distribution', return_value=None):
'tars': [], self.assertIs(type(load_platform_subclass(LinuxTest)), Foo)
'versions': ['885.0']},
}} # match both the distribution and platform class
with patch('ansible.module_utils.basic.get_platform', return_value="Linux"):
if per_rec: with patch('ansible.module_utils.basic.get_distribution', return_value="Bar"):
host_facts['host%s' % i]['secret'] = secret_text self.assertIs(type(load_platform_subclass(LinuxTest)), Bar)
hostvars['hostvars'].update(host_facts)
if top_level: # if neither match, the fallback should be the top-level class
hostvars['secret'] = secret_text with patch('ansible.module_utils.basic.get_platform', return_value="Foo"):
return hostvars with patch('ansible.module_utils.basic.get_distribution', return_value=None):
self.assertIs(type(load_platform_subclass(LinuxTest)), LinuxTest)
def setUp(self):
self.many_url = repr(self._gen_data(self.MANY_RECORDS, True, True, def test_module_utils_basic_json_dict_converters(self):
self.URL_SECRET)) from ansible.module_utils.basic import json_dict_unicode_to_bytes, json_dict_bytes_to_unicode
self.many_ssh = repr(self._gen_data(self.MANY_RECORDS, True, True,
self.SSH_SECRET)) test_data = dict(
self.one_url = repr(self._gen_data(self.MANY_RECORDS, False, True, item1 = u"Fóo",
self.URL_SECRET)) item2 = [u"Bár", u"Bam"],
self.one_ssh = repr(self._gen_data(self.MANY_RECORDS, False, True, item3 = dict(sub1=u"Súb"),
self.SSH_SECRET)) item4 = (u"föo", u"bär", u"©"),
self.zero_secrets = repr(self._gen_data(self.MANY_RECORDS, False, item5 = 42,
False, '')) )
self.few_url = repr(self._gen_data(2, True, True, self.URL_SECRET)) res = json_dict_unicode_to_bytes(test_data)
self.few_ssh = repr(self._gen_data(2, True, True, self.SSH_SECRET)) res2 = json_dict_bytes_to_unicode(res)
# create a temporary file for the test module self.assertEqual(test_data, res2)
# we're about to generate
self.tmp_fd, self.tmp_path = tempfile.mkstemp() def test_module_utils_basic_heuristic_log_sanitize(self):
os.write(self.tmp_fd, TEST_MODULE_DATA) from ansible.module_utils.basic import heuristic_log_sanitize
# template the module code and eval it URL_SECRET = 'http://username:pas:word@foo.com/data'
module_data, module_style, shebang = modify_module(self.tmp_path, {}) SSH_SECRET = 'username:pas:word@foo.com/data'
d = {} def _gen_data(records, per_rec, top_level, secret_text):
exec(module_data, d, d) hostvars = {'hostvars': {}}
self.module = d['get_module']() for i in range(1, records, 1):
host_facts = {'host%s' % i:
# module_utils/basic.py screws with CWD, let's save it and reset {'pstack':
self.cwd = os.getcwd() {'running': '875.1',
'symlinked': '880.0',
def tearDown(self): 'tars': [],
self.cleanup_temp_file(self.tmp_fd, self.tmp_path) 'versions': ['885.0']},
# Reset CWD back to what it was before basic.py changed it }}
os.chdir(self.cwd) if per_rec:
host_facts['host%s' % i]['secret'] = secret_text
hostvars['hostvars'].update(host_facts)
################################################################################# if top_level:
hostvars['secret'] = secret_text
# return hostvars
# Speed tests
# url_data = repr(_gen_data(3, True, True, URL_SECRET))
ssh_data = repr(_gen_data(3, True, True, SSH_SECRET))
# Previously, we used regexes which had some pathologically slow cases for
# parameters with large amounts of data with many ':' but no '@'. The
# present function gets slower when there are many replacements so we may
# want to explore regexes in the future (for the speed when substituting
# or flexibility). These speed tests will hopefully tell us if we're
# introducing code that has cases that are simply too slow.
#
# Some regex notes:
# * re.sub() is faster than re.match() + str.join().
# * We may be able to detect a large number of '@' symbols and then use
# a regex else use the present function.
#@timed(5)
#def test_log_sanitize_speed_many_url(self):
# heuristic_log_sanitize(self.many_url)
#@timed(5)
#def test_log_sanitize_speed_many_ssh(self):
# heuristic_log_sanitize(self.many_ssh)
#@timed(5)
#def test_log_sanitize_speed_one_url(self):
# heuristic_log_sanitize(self.one_url)
#@timed(5)
#def test_log_sanitize_speed_one_ssh(self):
# heuristic_log_sanitize(self.one_ssh)
#@timed(5)
#def test_log_sanitize_speed_zero_secrets(self):
# heuristic_log_sanitize(self.zero_secrets)
#
# Test that the password obfuscation sanitizes somewhat cleanly.
#
def test_log_sanitize_correctness(self):
url_data = repr(self._gen_data(3, True, True, self.URL_SECRET))
ssh_data = repr(self._gen_data(3, True, True, self.SSH_SECRET))
url_output = heuristic_log_sanitize(url_data) url_output = heuristic_log_sanitize(url_data)
ssh_output = heuristic_log_sanitize(ssh_data) ssh_output = heuristic_log_sanitize(ssh_data)
...@@ -349,7 +216,80 @@ class TestModuleUtilsBasicHelpers(unittest.TestCase): ...@@ -349,7 +216,80 @@ class TestModuleUtilsBasicHelpers(unittest.TestCase):
# python2.6 or less's unittest # python2.6 or less's unittest
self.assertTrue(":********@foo.com/data'" in ssh_output, '%s is not present in %s' % (":********@foo.com/data'", ssh_output)) self.assertTrue(":********@foo.com/data'" in ssh_output, '%s is not present in %s' % (":********@foo.com/data'", ssh_output))
# The overzealous-ness here may lead to us changing the algorithm in
# the future. We could make it consume less of the data (with the def test_module_utils_basic_ansible_module_creation(self):
# possibility of leaving partial passwords exposed) and encourage from ansible.module_utils import basic
# people to use no_log instead of relying on this obfuscation.
basic.MODULE_COMPLEX_ARGS = '{}'
am = basic.AnsibleModule(
argument_spec=dict(),
)
arg_spec = dict(
foo = dict(required=True),
bar = dict(),
bam = dict(),
baz = dict(),
)
mut_ex = (('bar', 'bam'),)
req_to = (('bam', 'baz'),)
# should test ok
basic.MODULE_COMPLEX_ARGS = '{"foo":"hello"}'
am = basic.AnsibleModule(
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
# fail, because a required param was not specified
basic.MODULE_COMPLEX_ARGS = '{}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
# fail because of mutually exclusive parameters
basic.MODULE_COMPLEX_ARGS = '{"foo":"hello", "bar": "bad", "bam": "bad"}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
# fail because a param required due to another param was not specified
basic.MODULE_COMPLEX_ARGS = '{"bam":"bad"}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
required_together = req_to,
no_log=True,
check_invalid_arguments=False,
add_file_common_args=True,
supports_check_mode=True,
)
def test_module_utils_basic_get_module_path(self):
from ansible.module_utils.basic import get_module_path
with patch('os.path.realpath', return_value='/path/to/foo/'):
self.assertEqual(get_module_path(), '/path/to/foo')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment