Create a plugin loader system

lib/ansible/callbacks.py

@@ -24,14 +24,6 @@ import os.path
 from ansible.color import stringc
 import ansible.constants as C
 
-dirname = os.path.dirname(__file__)
-callbacks = utils.import_plugins(os.path.join(dirname, 'callback_plugins'))
-callbacks = [ c.CallbackModule() for c in callbacks.values() ]
-def load_more_callbacks(dirname):
-    callbacks.extend([c.CallbackModule() for c in utils.import_plugins(dirname).values()])
-for i in C.DEFAULT_CALLBACK_PLUGIN_PATH.split(os.pathsep):
-    load_more_callbacks(i)
-
 cowsay = None
 if os.getenv("ANSIBLE_NOCOWS") is not None:
     cowsay = None
@@ -45,7 +37,7 @@ elif os.path.exists("/usr/local/bin/cowsay"):
 
 def call_callback_module(method_name, *args, **kwargs):
 
-    for callback_plugin in callbacks:
+    for callback_plugin in utils.plugins.callback_loader.all():
         methods = [
             getattr(callback_plugin, method_name, None),
             getattr(callback_plugin, 'on_any', None)

lib/ansible/inventory/__init__.py

@@ -29,20 +29,13 @@ from ansible.inventory.host import Host
 from ansible import errors
 from ansible import utils
 
-# FIXME, adapt.
-dirname = os.path.dirname(__file__)
-vars_plugin_list = utils.import_plugins(os.path.join(dirname, 'vars_plugins'))
-for i in reversed(C.DEFAULT_VARS_PLUGIN_PATH.split(os.pathsep)):
-    vars_plugin_list.update(utils.import_plugins(i))
-
 class Inventory(object):
     """
     Host inventory for ansible.
     """
 
     __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', '_is_script',
-                  'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
-                   '_vars_plugins' ]
+                  'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list']
 
     def __init__(self, host_list=C.DEFAULT_HOST_LIST):
 
@@ -74,6 +67,9 @@ class Inventory(object):
                 host_list = host_list.split(",")
                 host_list = [ h for h in host_list if h and h.strip() ]
 
+        else:
+            utils.plugins.push_basedir(self.basedir())
+
         if type(host_list) == list:
             all = Group('all')
             self.groups = [ all ]
@@ -95,8 +91,6 @@ class Inventory(object):
             else:
                 raise errors.AnsibleError("YAML inventory support is deprecated in 0.6 and removed in 0.7, see the migration script in examples/scripts in the git checkout")
 
-        self._vars_plugins = [ i.VarsModule(self) for i in vars_plugin_list.values() ]
-
     def _match(self, str, pattern_str):
         return fnmatch.fnmatch(str, pattern_str)
 
@@ -280,8 +274,7 @@ class Inventory(object):
             raise errors.AnsibleError("host not found: %s" % hostname)
 
         vars = {}
-        for ip in self._vars_plugins:
-            updated = ip.run(host)
+        for updated in map(lambda x: x.run(host), utils.plugins.vars_loader.all(self)):
             if updated is not None:
                 vars.update(updated)
 

lib/ansible/playbook/__init__.py

@@ -27,8 +27,6 @@ from play import Play
 
 SETUP_CACHE = collections.defaultdict(dict)
 
-plugins_dir = os.path.join(os.path.dirname(__file__), '..', 'runner')
-
 class PlayBook(object):
     '''
     runs an ansible playbook, given as a datastructure or YAML filename.
@@ -112,7 +110,6 @@ class PlayBook(object):
         self.inventory.subset(subset)
 
         self.modules_list        = utils.get_available_modules(self.module_path)
-        self.lookup_plugins_list = ansible.runner.lookup_plugin_list
 
         if not self.inventory._is_script:
             self.global_vars.update(self.inventory.get_group_variables('all'))
@@ -120,9 +117,9 @@ class PlayBook(object):
         self.basedir     = os.path.dirname(playbook)
         (self.playbook, self.play_basedirs) = self._load_playbook_from_file(playbook)
         self.module_path = self.module_path + os.pathsep + os.path.join(self.basedir, "library")
-        ansible.callbacks.load_more_callbacks(os.path.join(self.basedir, "callback_plugins"))
 
-        self.lookup_plugins_list.update(utils.import_plugins(os.path.join(self.basedir, 'lookup_plugins')))
+        for i in self.play_basedirs:
+            utils.plugins.push_basedir(i)
 
     # *****************************************************
 

lib/ansible/playbook/play.py

@@ -109,10 +109,10 @@ class Play(object):
                     if not k.startswith("with_"):
                         continue
                     plugin_name = k[5:]
-                    if plugin_name not in self.playbook.lookup_plugins_list:
+                    if plugin_name not in utils.plugins.lookup_loader:
                         raise errors.AnsibleError("cannot find lookup plugin named %s for usage in with_%s" % (plugin_name, plugin_name))
                     terms = utils.varReplaceWithItems(self.basedir, x[k], task_vars)
-                    items = self.playbook.lookup_plugins_list[plugin_name].LookupModule(basedir=self.basedir, runner=None).run(terms, inject=task_vars)
+                    items = utils.plugins.lookup_loader.get(plugin_name, basedir=self.basedir, runner=None).run(terms, inject=task_vars)
 
                 for item in items:
                     mv = task_vars.copy()

lib/ansible/playbook/task.py

@@ -51,7 +51,7 @@ class Task(object):
             # code to allow "with_glob" and to reference a lookup plugin named glob
             elif x.startswith("with_"):
                 plugin_name = x.replace("with_","")
-                if plugin_name in play.playbook.lookup_plugins_list:
+                if plugin_name in utils.plugins.lookup_loader:
                     ds['items_lookup_plugin'] = plugin_name
                     ds['items_lookup_terms'] = ds[x]
                     ds.pop(x)

lib/ansible/runner/__init__.py

@@ -45,14 +45,6 @@ try:
 except ImportError:
     HAS_ATFORK=False
 
-dirname = os.path.dirname(__file__)
-action_plugin_list = utils.import_plugins(os.path.join(dirname, 'action_plugins'))
-for i in reversed(C.DEFAULT_ACTION_PLUGIN_PATH.split(os.pathsep)):
-    action_plugin_list.update(utils.import_plugins(i))
-lookup_plugin_list = utils.import_plugins(os.path.join(dirname, 'lookup_plugins'))
-for i in reversed(C.DEFAULT_LOOKUP_PLUGIN_PATH.split(os.pathsep)):
-    lookup_plugin_list.update(utils.import_plugins(i))
-
 multiprocessing_runner = None
 
 ################################################
@@ -164,19 +156,6 @@ class Runner(object):
         # ensure we are using unique tmp paths
         random.seed()
 
-        # instantiate plugin classes
-        self.action_plugins = {}
-        self.lookup_plugins = {}
-        for (k,v) in action_plugin_list.iteritems():
-            self.action_plugins[k] = v.ActionModule(self)
-        for (k,v) in lookup_plugin_list.iteritems():
-            self.lookup_plugins[k] = v.LookupModule(runner=self, basedir=self.basedir)
-
-        for (k,v) in utils.import_plugins(os.path.join(self.basedir, 'action_plugins')).iteritems():
-            self.action_plugins[k] = v.ActionModule(self)
-        for (k,v) in utils.import_plugins(os.path.join(self.basedir, 'lookup_plugins')).iteritems():
-            self.lookup_plugins[k] = v.LookupModule(runner=self, basedir=self.basedir)
-
     # *****************************************************
 
     def _transfer_str(self, conn, tmp, name, data):
@@ -292,10 +271,10 @@ class Runner(object):
         # allow with_foo to work in playbooks...
         items = None
         items_plugin = self.module_vars.get('items_lookup_plugin', None)
-        if items_plugin is not None and items_plugin in self.lookup_plugins:
+        if items_plugin is not None and items_plugin in utils.plugins.lookup_loader:
             items_terms = self.module_vars.get('items_lookup_terms', '')
             items_terms = utils.varReplaceWithItems(self.basedir, items_terms, inject)
-            items = self.lookup_plugins[items_plugin].run(items_terms, inject=inject)
+            items = utils.plugins.lookup_loader.get(items_plugin, runner=self, basedir=self.basedir).run(items_terms, inject=inject)
             if type(items) != list:
                 raise errors.AnsibleError("lookup plugins have to return a list: %r" % items)
 
@@ -417,16 +396,16 @@ class Runner(object):
             tmp = self._make_tmp_path(conn)
         result = None
 
-        handler = self.action_plugins.get(module_name, None)
-        if handler:
+        if module_name in utils.plugins.action_loader:
             if self.background != 0:
                 raise errors.AnsibleError("async mode is not supported with the %s module" % module_name)
+            handler = utils.plugins.action_loader.get(module_name, self)
             result = handler.run(conn, tmp, module_name, module_args, inject)
         else:
             if self.background == 0:
-                result = self.action_plugins['normal'].run(conn, tmp, module_name, module_args, inject)
+                result = utils.plugins.action_loader.get('normal', self).run(conn, tmp, module_name, module_args, inject)
             else:
-                result = self.action_plugins['async'].run(conn, tmp, module_name, module_args, inject)
+                result = utils.plugins.action_loader.get('async', self).run(conn, tmp, module_name, module_args, inject)
 
         conn.close()
 
@@ -648,7 +627,7 @@ class Runner(object):
         # to be ran once per group of hosts. Example module: pause,
         # run once per hostgroup, rather than pausing once per each
         # host.
-        p = self.action_plugins.get(self.module_name, None)
+        p = utils.plugins.action_loader.get(self.module_name, self)
         if p and getattr(p, 'BYPASS_HOST_LOOP', None):
             # Expose the current hostgroup to the bypassing plugins
             self.host_set = hosts

lib/ansible/runner/connection.py

@@ -24,32 +24,19 @@ import ansible.constants as C
 
 import os
 import os.path
-dirname = os.path.dirname(__file__)
-modules = utils.import_plugins(os.path.join(dirname, 'connection_plugins'))
-for i in reversed(C.DEFAULT_CONNECTION_PLUGIN_PATH.split(os.pathsep)):
-    modules.update(utils.import_plugins(i))
-
-# rename this module
-modules['paramiko'] = modules['paramiko_ssh']
-del modules['paramiko_ssh']
 
 class Connection(object):
     ''' Handles abstract connections to remote hosts '''
 
     def __init__(self, runner):
         self.runner = runner
-        self.modules = None
 
     def connect(self, host, port):
-        if self.modules is None:
-            self.modules = modules.copy()
-            self.modules.update(utils.import_plugins(os.path.join(self.runner.basedir, 'connection_plugins')))
         conn = None
         transport = self.runner.transport
-        module = self.modules.get(transport, None)
-        if module is None:
+        conn = utils.plugins.connection_loader.get(transport, self.runner, host, port)
+        if conn is None:
             raise AnsibleError("unsupported connection type: %s" % transport)
-        conn = module.Connection(self.runner, host, port)
         self.active = conn.connect()
         return self.active
 

lib/ansible/utils/__init__.py

@@ -24,11 +24,10 @@ import operator
 from ansible import errors
 from ansible import __version__
 from ansible.utils.template import *
+from ansible.utils.plugins import *
 import ansible.constants as C
 import time
 import StringIO
-import imp
-import glob
 import stat
 import termios
 import tty
@@ -457,17 +456,6 @@ def filter_leading_non_json_lines(buf):
             filtered_lines.write(line + '\n')
     return filtered_lines.getvalue()
 
-def import_plugins(directory):
-    modules = {}
-    python_files = os.path.join(directory, '*.py')
-    for path in glob.glob(python_files):
-        if path.startswith("_"):
-            continue
-        name, ext = os.path.splitext(os.path.basename(path))
-        if not name.startswith("_"):
-            modules[name] = imp.load_source(name, path)
-    return modules
-
 def get_available_modules(dirname=None):
     """
     returns a list of modules available based on current directory

lib/ansible/utils/plugins.py

+# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import glob
+import imp
+import ansible.constants as C
+from ansible import errors
+
+_basedirs = []
+
+def push_basedir(basedir):
+    _basedirs.insert(0, basedir)
+
+class PluginLoader(object):
+    """PluginLoader loads plugins from the best source
+
+    It searches for plugins by iterating through the combined list of
+    play basedirs, configured paths, and the installed package directory.
+    The first match is used.
+    """
+    def __init__(self, class_name, package, config, subdir, aliases={}):
+        """Create a new PluginLoader"""
+        self.class_name = class_name
+        self.package = package
+        self.config = config
+        self.subdir = subdir
+        self.aliases = aliases
+        self._module_cache = {}
+
+    def _get_package_path(self):
+        """Gets the path of a Python package"""
+        if not hasattr(self, 'package_path'):
+            m = __import__(self.package)
+            parts = self.package.split('.')[1:]
+            self.package_path = os.path.join(os.path.dirname(m.__file__), *parts)
+        return self.package_path
+
+    def _get_paths(self):
+        """Return a list of paths to search for plugins in
+
+        The list is searched in order."""
+        return [os.path.join(basedir, self.subdir) for basedir in _basedirs] + self.config.split(os.pathsep) + [self._get_package_path()]
+
+    def find_plugin(self, name):
+        """Find a plugin named name"""
+        for i in self._get_paths():
+            path = os.path.join(i, "%s.py" % name)
+            if os.path.exists(path):
+                return path
+        return None
+
+    def has_plugin(self, name):
+        """Checks if a plugin named name exists"""
+        return self.find_plugin(name) is not None
+    __contains__ = has_plugin
+
+    def get(self, name, *args, **kwargs):
+        if name in self.aliases:
+            name = self.aliases[name]
+        path = self.find_plugin(name)
+        if path is None:
+            return None
+        if path not in self._module_cache:
+            self._module_cache[path] = imp.load_source('.'.join([self.package, name]), path)
+        return getattr(self._module_cache[path], self.class_name)(*args, **kwargs)
+
+    def all(self, *args, **kwargs):
+        for i in self._get_paths():
+            for path in glob.glob(os.path.join(i, "*.py")):
+                name, ext = os.path.splitext(os.path.basename(path))
+                if name.startswith("_"):
+                    continue
+                if path not in self._module_cache:
+                    self._module_cache[path] = imp.load_source('.'.join([self.package, name]), path)
+                yield getattr(self._module_cache[path], self.class_name)(*args, **kwargs)
+
+action_loader     = PluginLoader('ActionModule',   'ansible.runner.action_plugins',     C.DEFAULT_ACTION_PLUGIN_PATH,     'action_plugins')
+callback_loader   = PluginLoader('CallbackModule', 'ansible.callback_plugins',          C.DEFAULT_CALLBACK_PLUGIN_PATH,   'callback_plugins')
+connection_loader = PluginLoader('Connection',     'ansible.runner.connection_plugins', C.DEFAULT_CONNECTION_PLUGIN_PATH, 'connection_plugins', aliases={'paramiko': 'paramiko_ssh'})
+lookup_loader     = PluginLoader('LookupModule',   'ansible.runner.lookup_plugins',     C.DEFAULT_LOOKUP_PLUGIN_PATH,     'lookup_plugins')
+vars_loader       = PluginLoader('VarsModule',     'ansible.inventory.vars_plugins',    C.DEFAULT_VARS_PLUGIN_PATH,       'vars_plugins')