Commit 957867e0 by Michael DeHaan

Merge branch 'yaml-inventory' of https://github.com/jhoekx/ansible into jhoekx-yaml-inventory

Conflicts:
	lib/ansible/runner.py
parents d8f9d7c6 8c3206c9
......@@ -98,11 +98,11 @@ class Cli(object):
# ----------------------------------------------
def get_polling_runner(self, old_runner, hosts, jid):
def get_polling_runner(self, old_runner, jid):
return ansible.runner.Runner(
module_name='async_status', module_path=old_runner.module_path,
module_args="jid=%s" % jid, remote_user=old_runner.remote_user,
remote_pass=old_runner.remote_pass, host_list=hosts,
remote_pass=old_runner.remote_pass, inventory=old_runner.inventory,
timeout=old_runner.timeout, forks=old_runner.forks,
remote_port=old_runner.remote_port, pattern='*',
callbacks=self.silent_callbacks, verbose=True,
......@@ -138,8 +138,10 @@ class Cli(object):
clock = options.seconds
while (clock >= 0):
polling_runner = self.get_polling_runner(runner, poll_hosts, jid)
runner.inventory.restrict_to(poll_hosts)
polling_runner = self.get_polling_runner(runner, jid)
poll_results = polling_runner.run()
runner.inventory.lift_restrictions()
if poll_results is None:
break
for (host, host_result) in poll_results['contacted'].iteritems():
......
......@@ -17,6 +17,7 @@
#############################################
import ansible.inventory
import ansible.runner
import ansible.constants as C
from ansible import utils
......@@ -68,7 +69,6 @@ class PlayBook(object):
if playbook is None or callbacks is None or runner_callbacks is None or stats is None:
raise Exception('missing required arguments')
self.host_list = host_list
self.module_path = module_path
self.forks = forks
self.timeout = timeout
......@@ -88,9 +88,13 @@ class PlayBook(object):
self.basedir = os.path.dirname(playbook)
self.playbook = self._parse_playbook(playbook)
self.host_list, self.groups = ansible.runner.Runner.parse_hosts(
host_list, override_hosts=self.override_hosts, extra_vars=self.extra_vars)
if override_hosts is not None:
if type(override_hosts) != list:
raise errors.AnsibleError("override hosts must be a list")
self.inventory = ansible.inventory.Inventory(override_hosts)
else:
self.inventory = ansible.inventory.Inventory(host_list)
# *****************************************************
def _get_vars(self, play, dirname):
......@@ -233,7 +237,6 @@ class PlayBook(object):
def _async_poll(self, runner, hosts, async_seconds, async_poll_interval, only_if):
''' launch an async job, if poll_interval is set, wait for completion '''
runner.host_list = hosts
runner.background = async_seconds
results = runner.run()
self.stats.compute(results, poll=True)
......@@ -257,7 +260,7 @@ class PlayBook(object):
return results
clock = async_seconds
runner.host_list = self.hosts_to_poll(results)
host_list = self.hosts_to_poll(results)
poll_results = results
while (clock >= 0):
......@@ -267,11 +270,13 @@ class PlayBook(object):
runner.module_name = 'async_status'
runner.background = 0
runner.pattern = '*'
self.inventory.restrict_to(host_list)
poll_results = runner.run()
self.stats.compute(poll_results, poll=True)
runner.host_list = self.hosts_to_poll(poll_results)
host_list = self.hosts_to_poll(poll_results)
self.inventory.lift_restriction()
if len(runner.host_list) == 0:
if len(host_list) == 0:
break
if poll_results is None:
break
......@@ -298,15 +303,16 @@ class PlayBook(object):
# *****************************************************
def _run_module(self, pattern, host_list, module, args, vars, remote_user,
def _run_module(self, pattern, module, args, vars, remote_user,
async_seconds, async_poll_interval, only_if, sudo, transport):
''' run a particular module step in a playbook '''
hosts = [ h for h in host_list if (h not in self.stats.failures) and (h not in self.stats.dark)]
hosts = [ h for h in self.inventory.list_hosts() if (h not in self.stats.failures) and (h not in self.stats.dark)]
self.inventory.restrict_to(hosts)
runner = ansible.runner.Runner(
pattern=pattern, groups=self.groups, module_name=module,
module_args=args, host_list=hosts, forks=self.forks,
pattern=pattern, inventory=self.inventory, module_name=module,
module_args=args, forks=self.forks,
remote_pass=self.remote_pass, module_path=self.module_path,
timeout=self.timeout, remote_user=remote_user,
remote_port=self.remote_port, module_vars=vars,
......@@ -317,13 +323,16 @@ class PlayBook(object):
)
if async_seconds == 0:
return runner.run()
results = runner.run()
else:
return self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if)
results = self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if)
self.inventory.lift_restriction()
return results
# *****************************************************
def _run_task(self, pattern=None, host_list=None, task=None,
def _run_task(self, pattern=None, task=None,
remote_user=None, handlers=None, conditional=False, sudo=False, transport=None):
''' run a single task in the playbook and recursively run any subtasks. '''
......@@ -354,7 +363,7 @@ class PlayBook(object):
# load up an appropriate ansible runner to
# run the task in parallel
results = self._run_module(pattern, host_list, module_name,
results = self._run_module(pattern, module_name,
module_args, module_vars, remote_user, async_seconds,
async_poll_interval, only_if, sudo, transport)
......@@ -406,7 +415,7 @@ class PlayBook(object):
# *****************************************************
def _do_conditional_imports(self, vars_files, host_list):
def _do_conditional_imports(self, vars_files):
''' handle the vars_files section, which can contain variables '''
# FIXME: save parsed variable results in memory to avoid excessive re-reading/parsing
......@@ -417,7 +426,7 @@ class PlayBook(object):
if type(vars_files) != list:
raise errors.AnsibleError("vars_files must be a list")
for host in host_list:
for host in self.inventory.list_hosts():
cache_vars = SETUP_CACHE.get(host,{})
SETUP_CACHE[host] = cache_vars
for filename in vars_files:
......@@ -460,16 +469,18 @@ class PlayBook(object):
if vars_files is not None:
self.callbacks.on_setup_secondary()
self._do_conditional_imports(vars_files, self.host_list)
self._do_conditional_imports(vars_files)
else:
self.callbacks.on_setup_primary()
host_list = [ h for h in self.host_list if not (h in self.stats.failures or h in self.stats.dark) ]
host_list = [ h for h in self.inventory.list_hosts(pattern)
if not (h in self.stats.failures or h in self.stats.dark) ]
self.inventory.restrict_to(host_list)
# push any variables down to the system
setup_results = ansible.runner.Runner(
pattern=pattern, groups=self.groups, module_name='setup',
module_args=vars, host_list=host_list,
pattern=pattern, module_name='setup',
module_args=vars, inventory=self.inventory,
forks=self.forks, module_path=self.module_path,
timeout=self.timeout, remote_user=user,
remote_pass=self.remote_pass, remote_port=self.remote_port,
......@@ -479,6 +490,8 @@ class PlayBook(object):
).run()
self.stats.compute(setup_results, setup=True)
self.inventory.lift_restriction()
# now for each result, load into the setup cache so we can
# let runner template out future commands
setup_ok = setup_results.get('contacted', {})
......@@ -494,7 +507,6 @@ class PlayBook(object):
SETUP_CACHE[h].update(extra_vars)
except:
SETUP_CACHE[h] = extra_vars
return host_list
# *****************************************************
......@@ -530,7 +542,6 @@ class PlayBook(object):
for task in tasks:
self._run_task(
pattern=pattern,
host_list=self.host_list,
task=task,
handlers=handlers,
remote_user=user,
......@@ -547,16 +558,17 @@ class PlayBook(object):
for task in handlers:
triggered_by = task.get('run', None)
if type(triggered_by) == list:
self.inventory.restrict_to(triggered_by)
self._run_task(
pattern=pattern,
task=task,
handlers=[],
host_list=triggered_by,
conditional=True,
remote_user=user,
sudo=sudo,
transport=transport
)
self.inventory.lift_restriction()
# end of execution for this particular pattern. Multiple patterns
# can be in a single playbook file
......
......@@ -18,7 +18,6 @@
################################################
import fnmatch
import multiprocessing
import signal
import os
......@@ -32,6 +31,7 @@ import getpass
import ansible.constants as C
import ansible.connection
import ansible.inventory
from ansible import utils
from ansible import errors
from ansible import callbacks as ans_callbacks
......@@ -68,8 +68,6 @@ def _executor_hook(job_queue, result_queue):
class Runner(object):
_external_variable_script = None
def __init__(self, host_list=C.DEFAULT_HOST_LIST, module_path=C.DEFAULT_MODULE_PATH,
module_name=C.DEFAULT_MODULE_NAME, module_args=C.DEFAULT_MODULE_ARGS,
forks=C.DEFAULT_FORKS, timeout=C.DEFAULT_TIMEOUT, pattern=C.DEFAULT_PATTERN,
......@@ -77,7 +75,8 @@ class Runner(object):
sudo_pass=C.DEFAULT_SUDO_PASS, remote_port=C.DEFAULT_REMOTE_PORT, background=0,
basedir=None, setup_cache=None, transport=C.DEFAULT_TRANSPORT,
conditional='True', groups={}, callbacks=None, verbose=False,
debug=False, sudo=False, extra_vars=None, module_vars=None, is_playbook=False):
debug=False, sudo=False, extra_vars=None,
module_vars=None, is_playbook=False, inventory=None):
if setup_cache is None:
setup_cache = {}
......@@ -93,11 +92,10 @@ class Runner(object):
self.transport = transport
self.connector = ansible.connection.Connection(self, self.transport)
if type(host_list) == str:
self.host_list, self.groups = self.parse_hosts(host_list)
if inventory is None:
self.inventory = ansible.inventory.Inventory(host_list, extra_vars)
else:
self.host_list = host_list
self.groups = groups
self.inventory = inventory
self.setup_cache = setup_cache
self.conditional = conditional
......@@ -129,106 +127,17 @@ class Runner(object):
self._tmp_paths = {}
random.seed()
# *****************************************************
@classmethod
def parse_hosts_from_regular_file(cls, host_list):
''' parse a textual host file '''
results = []
groups = dict(ungrouped=[])
lines = file(host_list).read().split("\n")
group_name = 'ungrouped'
for item in lines:
item = item.lstrip().rstrip()
if item.startswith("#"):
# ignore commented out lines
pass
elif item.startswith("["):
# looks like a group
group_name = item.replace("[","").replace("]","").lstrip().rstrip()
groups[group_name] = []
elif item != "":
# looks like a regular host
groups[group_name].append(item)
if not item in results:
results.append(item)
return (results, groups)
# *****************************************************
@classmethod
def parse_hosts_from_script(cls, host_list, extra_vars):
''' evaluate a script that returns list of hosts by groups '''
results = []
groups = dict(ungrouped=[])
host_list = os.path.abspath(host_list)
cls._external_variable_script = host_list
cmd = [host_list, '--list']
if extra_vars:
cmd.extend(['--extra-vars', extra_vars])
cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
out, err = cmd.communicate()
rc = cmd.returncode
if rc:
raise errors.AnsibleError("%s: %s" % (host_list, err))
try:
groups = utils.json_loads(out)
except:
raise errors.AnsibleError("invalid JSON response from script: %s" % host_list)
for (groupname, hostlist) in groups.iteritems():
for host in hostlist:
if host not in results:
results.append(host)
return (results, groups)
# *****************************************************
@classmethod
def parse_hosts(cls, host_list, override_hosts=None, extra_vars=None):
''' parse the host inventory file, returns (hosts, groups) '''
if override_hosts is not None:
if type(override_hosts) != list:
raise errors.AnsibleError("override hosts must be a list")
return (override_hosts, dict(ungrouped=override_hosts))
if type(host_list) == list:
raise Exception("function can only be called on inventory files")
host_list = os.path.expanduser(host_list)
if not os.path.exists(host_list):
raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list)
if not os.access(host_list, os.X_OK):
return Runner.parse_hosts_from_regular_file(host_list)
if override_hosts is None:
inventory = ansible.inventory.Inventory(host_list, extra_vars)
else:
return Runner.parse_hosts_from_script(host_list, extra_vars)
# *****************************************************
inventory = ansible.inventory.Inventory(override_hosts)
def _matches(self, host_name, pattern):
''' returns if a hostname is matched by the pattern '''
# a pattern is in fnmatch format but more than one pattern
# can be strung together with semicolons. ex:
# atlanta-web*.example.com;dc-web*.example.com
if host_name == '':
return False
pattern = pattern.replace(";",":")
subpatterns = pattern.split(":")
for subpattern in subpatterns:
if subpattern == 'all':
return True
if fnmatch.fnmatch(host_name, subpattern):
return True
elif subpattern in self.groups:
if host_name in self.groups[subpattern]:
return True
return False
return inventory.host_list, inventory.groups
# *****************************************************
......@@ -298,34 +207,6 @@ class Runner(object):
# *****************************************************
def _add_variables_from_script(self, conn, inject):
''' support per system variabes from external variable scripts, see web docs '''
host = conn.host
cmd = [Runner._external_variable_script, '--host', host]
if self.extra_vars:
cmd.extend(['--extra-vars', self.extra_vars])
cmd = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False
)
out, err = cmd.communicate()
inject2 = {}
try:
inject2 = utils.json_loads(out)
except:
raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % (
Runner._external_variable_script,
host
))
# store injected variables in the templates
inject.update(inject2)
# *****************************************************
def _add_setup_vars(self, inject, args):
''' setup module variables need special handling '''
......@@ -379,8 +260,9 @@ class Runner(object):
if not eval(conditional):
return [ utils.smjson(dict(skipped=True)), None, 'skipped' ]
if Runner._external_variable_script is not None:
self._add_variables_from_script(conn, inject)
host_variables = self.inventory.get_variables(conn.host, self.extra_vars)
inject.update(host_variables)
if self.module_name == 'setup':
args = self._add_setup_vars(inject, args)
args = self._add_setup_metadata(args)
......@@ -714,13 +596,6 @@ class Runner(object):
# *****************************************************
def _match_hosts(self, pattern):
''' return all matched hosts fitting a pattern '''
return [ h for h in self.host_list if self._matches(h, pattern) ]
# *****************************************************
def _parallel_exec(self, hosts):
''' handles mulitprocessing when more than 1 fork is required '''
......@@ -767,7 +642,7 @@ class Runner(object):
results2["dark"][host] = result
# hosts which were contacted but never got a chance to return
for host in self._match_hosts(self.pattern):
for host in self.inventory.list_hosts(self.pattern):
if not (host in results2['dark'] or host in results2['contacted']):
results2["dark"][host] = {}
......@@ -779,7 +654,7 @@ class Runner(object):
''' xfer & run module on all matched hosts '''
# find hosts that match the pattern
hosts = self._match_hosts(self.pattern)
hosts = self.inventory.list_hosts(self.pattern)
if len(hosts) == 0:
self.callbacks.on_no_hosts()
return dict(contacted={}, dark={})
......
import os
import unittest
from ansible.inventory import Inventory
from ansible.runner import Runner
class TestInventory(unittest.TestCase):
def setUp(self):
self.cwd = os.getcwd()
self.test_dir = os.path.join(self.cwd, 'test')
self.inventory_file = os.path.join(self.test_dir, 'simple_hosts')
self.inventory_script = os.path.join(self.test_dir, 'inventory_api.py')
self.inventory_yaml = os.path.join(self.test_dir, 'yaml_hosts')
os.chmod(self.inventory_script, 0755)
def tearDown(self):
os.chmod(self.inventory_script, 0644)
### Simple inventory format tests
def simple_inventory(self):
return Inventory( self.inventory_file )
def script_inventory(self):
return Inventory( self.inventory_script )
def yaml_inventory(self):
return Inventory( self.inventory_yaml )
def test_simple(self):
inventory = self.simple_inventory()
hosts = inventory.list_hosts()
expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_simple_all(self):
inventory = self.simple_inventory()
hosts = inventory.list_hosts('all')
expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_simple_norse(self):
inventory = self.simple_inventory()
hosts = inventory.list_hosts("norse")
expected_hosts=['thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_simple_ungrouped(self):
inventory = self.simple_inventory()
hosts = inventory.list_hosts("ungrouped")
expected_hosts=['jupiter', 'saturn']
assert hosts == expected_hosts
def test_simple_combined(self):
inventory = self.simple_inventory()
hosts = inventory.list_hosts("norse:greek")
expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_simple_restrict(self):
inventory = self.simple_inventory()
restricted_hosts = ['hera', 'poseidon', 'thor']
expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
inventory.restrict_to(restricted_hosts)
hosts = inventory.list_hosts("norse:greek")
assert hosts == restricted_hosts
inventory.lift_restriction()
hosts = inventory.list_hosts("norse:greek")
assert hosts == expected_hosts
def test_simple_vars(self):
inventory = self.simple_inventory()
vars = inventory.get_variables('thor')
assert vars == {}
def test_simple_extra_vars(self):
inventory = self.simple_inventory()
vars = inventory.get_variables('thor', 'a=5')
assert vars == {}
def test_simple_port(self):
inventory = self.simple_inventory()
vars = inventory.get_variables('hera')
assert vars == {'ansible_ssh_port': 3000}
### Inventory API tests
def test_script(self):
inventory = self.script_inventory()
hosts = inventory.list_hosts()
expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
print "Expected: %s"%(expected_hosts)
print "Got : %s"%(hosts)
assert sorted(hosts) == sorted(expected_hosts)
def test_script_all(self):
inventory = self.script_inventory()
hosts = inventory.list_hosts('all')
expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert sorted(hosts) == sorted(expected_hosts)
def test_script_norse(self):
inventory = self.script_inventory()
hosts = inventory.list_hosts("norse")
expected_hosts=['thor', 'odin', 'loki']
assert sorted(hosts) == sorted(expected_hosts)
def test_script_combined(self):
inventory = self.script_inventory()
hosts = inventory.list_hosts("norse:greek")
expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert sorted(hosts) == sorted(expected_hosts)
def test_script_restrict(self):
inventory = self.script_inventory()
restricted_hosts = ['hera', 'poseidon', 'thor']
expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
inventory.restrict_to(restricted_hosts)
hosts = inventory.list_hosts("norse:greek")
assert sorted(hosts) == sorted(restricted_hosts)
inventory.lift_restriction()
hosts = inventory.list_hosts("norse:greek")
assert sorted(hosts) == sorted(expected_hosts)
def test_script_vars(self):
inventory = self.script_inventory()
vars = inventory.get_variables('thor')
assert vars == {"hammer":True}
def test_script_extra_vars(self):
inventory = self.script_inventory()
vars = inventory.get_variables('thor', 'simple=yes')
assert vars == {"hammer":True, "simple": "yes"}
### Tests for yaml inventory file
def test_yaml(self):
inventory = self.yaml_inventory()
hosts = inventory.list_hosts()
print hosts
expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_yaml_all(self):
inventory = self.yaml_inventory()
hosts = inventory.list_hosts('all')
expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_yaml_norse(self):
inventory = self.yaml_inventory()
hosts = inventory.list_hosts("norse")
expected_hosts=['thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_simple_ungrouped(self):
inventory = self.yaml_inventory()
hosts = inventory.list_hosts("ungrouped")
expected_hosts=['jupiter']
assert hosts == expected_hosts
def test_yaml_combined(self):
inventory = self.yaml_inventory()
hosts = inventory.list_hosts("norse:greek")
expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
def test_yaml_restrict(self):
inventory = self.yaml_inventory()
restricted_hosts = ['hera', 'poseidon', 'thor']
expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
inventory.restrict_to(restricted_hosts)
hosts = inventory.list_hosts("norse:greek")
assert hosts == restricted_hosts
inventory.lift_restriction()
hosts = inventory.list_hosts("norse:greek")
assert hosts == expected_hosts
def test_yaml_vars(self):
inventory = self.yaml_inventory()
vars = inventory.get_variables('thor')
assert vars == {"hammer":True}
def test_yaml_change_vars(self):
inventory = self.yaml_inventory()
vars = inventory.get_variables('thor')
vars["hammer"] = False
vars = inventory.get_variables('thor')
assert vars == {"hammer":True}
def test_yaml_host_vars(self):
inventory = self.yaml_inventory()
vars = inventory.get_variables('saturn')
assert vars == {"moon":"titan"}
def test_yaml_extra_vars(self):
inventory = self.yaml_inventory()
vars = inventory.get_variables('thor', 'a=5')
assert vars == {"hammer":True}
def test_yaml_port(self):
inventory = self.yaml_inventory()
vars = inventory.get_variables('hera')
assert vars == {'ansible_ssh_port': 3000}
### Test Runner class method
def test_class_method(self):
hosts, groups = Runner.parse_hosts(self.inventory_file)
expected_hosts = ['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
assert hosts == expected_hosts
expected_groups= {
'ungrouped': ['jupiter', 'saturn'],
'greek': ['zeus', 'hera', 'poseidon'],
'norse': ['thor', 'odin', 'loki']
}
assert groups == expected_groups
def test_class_override(self):
override_hosts = ['thor', 'odin']
hosts, groups = Runner.parse_hosts(self.inventory_file, override_hosts)
assert hosts == override_hosts
assert groups == { 'ungrouped': override_hosts }
#!/usr/bin/env python
import json
import sys
from optparse import OptionParser
parser = OptionParser()
parser.add_option('-l', '--list', default=False, dest="list_hosts", action="store_true")
parser.add_option('-H', '--host', default=None, dest="host")
parser.add_option('-e', '--extra-vars', default=None, dest="extra")
options, args = parser.parse_args()
systems = {
"ungouped": [ "jupiter", "saturn" ],
"greek": [ "zeus", "hera", "poseidon" ],
"norse": [ "thor", "odin", "loki" ]
}
variables = {
"thor": {
"hammer": True
}
}
if options.list_hosts == True:
print json.dumps(systems)
sys.exit(0)
if options.host is not None:
if options.extra:
k,v = options.extra.split("=")
variables[options.host][k] = v
print json.dumps(variables[options.host])
sys.exit(0)
parser.print_help()
sys.exit(1)
\ No newline at end of file
jupiter
saturn
[greek]
zeus
hera:3000
poseidon
[norse]
thor
odin
loki
---
- jupiter
- host: saturn
vars:
- moon: titan
- zeus
- group: greek
hosts:
- zeus
- hera
- poseidon
vars:
- ansible_ssh_port: 3000
- group: norse
hosts:
- host: thor
vars:
- hammer: True
- odin
- loki
- group: multiple
hosts:
- saturn
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment