WIP on data structure processing patterns.

Makefile

@@ -94,7 +94,7 @@ tests:
 	PYTHONPATH=./lib $(NOSETESTS) -d -w test/units -v
 
 newtests:
-	PYTHONPATH=./v2 $(NOSETESTS) -d -w test/v2 -v
+	PYTHONPATH=./v2:./lib $(NOSETESTS) -d -w test/v2 -v
 
 
 authors:

test/v2/playbook/test_task.py

@@ -16,24 +16,39 @@ class TestTask(unittest.TestCase):
     def tearDown(self):
         pass
 
-    def test_can_construct_empty_task(self):
+    def test_construct_empty_task(self):
         t = Task()
     
-    def test_can_construct_task_with_role(self):
+    def test_construct_task_with_role(self):
         pass
 
-    def test_can_construct_task_with_block(self):
+    def test_construct_task_with_block(self):
         pass
 
-    def test_can_construct_task_with_role_and_block(self):
+    def test_construct_task_with_role_and_block(self):
         pass
 
-    def test_can_load_simple_task(self):
-       t = Task.load(basic_shell_task)
-       assert t is not None
-       print "NAME=%s" % t.name
-       assert t.name == basic_shell_task['name']
-       #assert t.module == 'shell'
-       #assert t.args == 'echo hi'
+    def test_load_simple_task(self):
+        t = Task.load(basic_shell_task)
+        assert t is not None
+        assert t.name == basic_shell_task['name']
+        assert t.module == 'shell'
+        assert t.args == 'echo hi'
 
+    def test_can_load_action_kv_form(self):
+        pass
+
+    def test_can_load_action_complex_form(self):
+        pass
+
+    def test_can_load_module_complex_form(self):
+        pass 
+
+    def test_local_action_implies_delegate(self):
+        pass 
  
+    def test_local_action_conflicts_with_delegate(self):
+        pass 
+
+    def test_delegate_to_parses(self):
+        pass 

v2/ansible/playbook/base.py

@@ -15,44 +15,39 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-#from ansible.cmmon.errors import AnsibleError
-#from playbook.tag import Tag
 from ansible.playbook.attribute import Attribute, FieldAttribute
 
-
-# general concept
-#  FooObject.load(datastructure) -> Foo
-#  FooObject._load_field # optional
-#  FooObject._validate_field # optional
-#  FooObject._post_validate_field # optional
-# FooObject.evaluate(host_context) -> FooObject ?  (calls post_validators, templates all members)
-# question - are there some things that need to be evaluated *before* host context, i.e. globally?
-# most things should be templated but want to provide as much early checking as possible
-# TODO: also check for fields in datastructure that are not valid
-# TODO: PluginAttribute(type) allows all the valid plugins as valid types of names
-#   lookupPlugins start with "with_", ModulePluginAttribute allows any key
-
 class Base(object):
 
     def __init__(self):
-        self._data = dict()
+     
+        # each class knows attributes set upon it, see Task.py for example 
         self._attributes = dict()
-        
         for name in self.__class__.__dict__:
             aname = name[1:]
             if isinstance(aname, Attribute) and not isinstance(aname, FieldAttribute):
                 self._attributes[aname] = None
 
+    def munge(self, ds):
+        ''' infrequently used method to do some pre-processing of legacy terms '''
+
+        return ds
+
     def load_data(self, ds):
         ''' walk the input datastructure and assign any values '''
 
         assert ds is not None
+        ds = self.munge(ds)
 
+        # walk all attributes in the class
         for (name, attribute) in self.__class__.__dict__.iteritems():
             aname = name[1:]
 
-            # process Fields
+            # process Field attributes which get loaded from the YAML
+
             if isinstance(attribute, FieldAttribute):
+
+                # copy the value over unless a _load_field method is defined
                 method = getattr(self, '_load_%s' % aname, None)
                 if method:
                     self._attributes[aname] = method(self, attribute)
@@ -60,38 +55,45 @@ class Base(object):
                     if aname in ds:
                         self._attributes[aname] = ds[aname]
 
-             # TODO: implement PluginAtrribute which allows "with_" and "action" aliases.
-
+        # return the constructed object
+        self.validate()
         return self
 
 
     def validate(self):
-        # TODO: finish
-        for name in self.__dict__:
-            aname = name[1:]
-            attribute = self.__dict__[aname]
-            if instanceof(attribute, FieldAttribute):
+        ''' validation that is done at parse time, not load time ''' 
+
+        # walk all fields in the object
+        for (name, attribute) in self.__dict__:
+
+            # find any field attributes
+            if isinstance(attribute, FieldAttribute):
+
+                if not name.startswith("_"):
+                    raise AnsibleError("FieldAttribute %s must start with _" % name)
+                
+                aname = name[1:]
+                
+                # run validator only if present
                 method = getattr(self, '_validate_%s' % (prefix, aname), None)
                 if method:
                     method(self, attribute)
 
     def post_validate(self, runner_context):
-        # TODO: finish
+        '''
+        we can't tell that everything is of the right type until we have
+        all the variables.  Run basic types (from isa) as well as 
+        any _post_validate_<foo> functions.
+        ''' 
+
         raise exception.NotImplementedError
 
     def __getattr__(self, needle):
+
+        # return any attribute names as if they were real.
+        # access them like obj.attrname()
         if needle in self._attributes:
             return self._attributes[needle]
-        if needle in self.__dict__:
-            return self.__dict__[needle]
-        raise AttributeError
-
-    #def __setattr__(self, needle, value):
-    #    if needle in self._attributes:
-    #        self._attributes[needle] = value
-    #    if needle in self.__dict__:
-    #        super(Base, self).__setattr__(needle, value)
-    #        # self.__dict__[needle] = value
-    #    raise AttributeError
 
+        raise AttributeError
 

v2/ansible/playbook/task.py

@@ -17,13 +17,15 @@
 
 from ansible.playbook.base import Base
 from ansible.playbook.attribute import Attribute, FieldAttribute
-from ansible.playbook.conditional import Conditional
-#from ansible.common.errors import AnsibleError
-#from ansible import utils
+
+# from ansible.playbook.conditional import Conditional
+# from ansible.common.errors import AnsibleError
 
 # TODO: it would be fantastic (if possible) if a task new where in the YAML it was defined for describing
 # it in error conditions
 
+from ansible.plugins import module_finder, lookup_finder
+
 class Task(Base):
 
     """
@@ -44,6 +46,7 @@ class Task(Base):
     # might be possible to define others 
 
     _action               = FieldAttribute(isa='string')
+    
     _always_run           = FieldAttribute(isa='bool')
     _any_errors_fatal     = FieldAttribute(isa='bool')
     _async                = FieldAttribute(isa='int') 
@@ -55,12 +58,14 @@ class Task(Base):
     _ignore_errors        = FieldAttribute(isa='bool')
 
     # FIXME: this should not be a Task
-    # include            = FieldAttribute(isa='string')
+    # include             = FieldAttribute(isa='string')
 
+    _loop                 = Attribute()
     _local_action         = FieldAttribute(isa='string')
   
     # FIXME: this should not be a Task
-    _meta                 = FieldAttribute(isa='string')
+    _module_args          = Attribute(isa='dict')
+    _meta                 = FieldAttribute(isa='string')    
 
     _name                 = FieldAttribute(isa='string')
 
@@ -106,6 +111,44 @@ class Task(Base):
         ''' returns a human readable representation of the task '''
         return "TASK: %s" % self.get_name()
 
+    def munge(self, ds):
+        ''' 
+        tasks are especially complex arguments so need pre-processing.
+        keep it short.
+        '''
+
+
+        assert isinstance(ds, dict)
+
+        new_ds = dict()
+        for (k,v) in ds.iteritems():
+        
+            # if any attributes of the datastructure match a module name
+            # convert it to "module + args"
+
+            if k in module_finder:
+                if _module.value is not None or 'action' in ds or 'local_action' in ds:
+                    raise AnsibleError("duplicate action in task: %s" % k)
+                _module.value      = k
+                _module_args.value = v
+
+            # handle any loops, there can be only one kind of loop
+
+            elif "with_%s" % k in lookup_finder:
+                if _loop.value is not None:
+                    raise AnsibleError("duplicate loop in task: %s" % k)
+                _loop.value      = k
+                _loop_args.value = v
+
+            # otherwise send it through straight
+
+            else:
+                # nothing we need to filter
+                new_ds[k] = v
+
+        return new_ds
+
+
     # ==================================================================================
     # BELOW THIS LINE
     # info below this line is "old" and is before the attempt to build Attributes
@@ -119,7 +162,7 @@ LEGACY = """
 
         results = dict()
         module_name, params = v.strip().split(' ', 1)
-        if module_name not in utils.plugins.module_finder:
+        if module_name not in module_finder:
             raise AnsibleError("the specified module '%s' could not be found, check your module path" % module_name)
         results['_module_name'] = module_name
         results['_parameters'] = utils.parse_kv(params)

v2/ansible/plugins/__init__.py

-# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com>
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others
 #
 # This file is part of Ansible
 #
@@ -15,3 +16,271 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+import os.path
+import sys
+import glob
+import imp
+from ansible import constants as C
+from ansible import errors
+
+MODULE_CACHE = {}
+PATH_CACHE = {}
+PLUGIN_PATH_CACHE = {}
+_basedirs = []
+
+def push_basedir(basedir):
+    # avoid pushing the same absolute dir more than once
+    basedir = os.path.realpath(basedir)
+    if basedir not in _basedirs:
+        _basedirs.insert(0, basedir)
+
+class PluginLoader(object):
+
+    '''
+    PluginLoader loads plugins from the configured plugin directories.
+
+    It searches for plugins by iterating through the combined list of
+    play basedirs, configured paths, and the python path.
+    The first match is used.
+    '''
+
+    def __init__(self, class_name, package, config, subdir, aliases={}):
+
+        self.class_name         = class_name
+        self.package            = package
+        self.config             = config
+        self.subdir             = subdir
+        self.aliases            = aliases
+
+        if not class_name in MODULE_CACHE:
+            MODULE_CACHE[class_name] = {}
+        if not class_name in PATH_CACHE:
+            PATH_CACHE[class_name] = None
+        if not class_name in PLUGIN_PATH_CACHE:
+            PLUGIN_PATH_CACHE[class_name] = {}
+
+        self._module_cache      = MODULE_CACHE[class_name]
+        self._paths             = PATH_CACHE[class_name]
+        self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name]
+
+        self._extra_dirs = []
+
+    def print_paths(self):
+        ''' Returns a string suitable for printing of the search path '''
+
+        # Uses a list to get the order right
+        ret = []
+        for i in self._get_paths():
+            if i not in ret:
+                ret.append(i)
+        return os.pathsep.join(ret)
+
+    def _all_directories(self, dir):
+        results = []
+        results.append(dir)
+        for root, subdirs, files in os.walk(dir):
+           if '__init__.py' in files:
+               for x in subdirs:
+                   results.append(os.path.join(root,x))
+        return results
+
+    def _get_package_paths(self):
+        ''' Gets the path of a Python package '''
+
+        paths = []
+        if not self.package:
+            return []
+        if not hasattr(self, 'package_path'):
+            m = __import__(self.package)
+            parts = self.package.split('.')[1:]
+            self.package_path = os.path.join(os.path.dirname(m.__file__), *parts)
+        paths.extend(self._all_directories(self.package_path))
+        return paths
+
+    def _get_paths(self):
+        ''' Return a list of paths to search for plugins in '''
+
+        if self._paths is not None:
+            return self._paths
+
+        ret = self._extra_dirs[:]
+        for basedir in _basedirs:
+            fullpath = os.path.realpath(os.path.join(basedir, self.subdir))
+            if os.path.isdir(fullpath):
+
+                files = glob.glob("%s/*" % fullpath)
+
+                # allow directories to be two levels deep
+                files2 = glob.glob("%s/*/*" % fullpath)
+
+                if files2 is not None:
+                    files.extend(files2)
+
+                for file in files:
+                    if os.path.isdir(file) and file not in ret:
+                        ret.append(file)
+                if fullpath not in ret:
+                    ret.append(fullpath)
+
+        # look in any configured plugin paths, allow one level deep for subcategories
+        if self.config is not None:
+            configured_paths = self.config.split(os.pathsep)
+            for path in configured_paths:
+                path = os.path.realpath(os.path.expanduser(path))
+                contents = glob.glob("%s/*" % path)
+                for c in contents:
+                    if os.path.isdir(c) and c not in ret:
+                        ret.append(c)
+                if path not in ret:
+                    ret.append(path)
+
+        # look for any plugins installed in the package subtree
+        ret.extend(self._get_package_paths())
+
+        # cache and return the result
+        self._paths = ret
+        return ret
+
+
+    def add_directory(self, directory, with_subdir=False):
+        ''' Adds an additional directory to the search path '''
+
+        directory = os.path.realpath(directory)
+
+        if directory is not None:
+            if with_subdir:
+                directory = os.path.join(directory, self.subdir)
+            if directory not in self._extra_dirs:
+                # append the directory and invalidate the path cache
+                self._extra_dirs.append(directory)
+                self._paths = None
+
+    def find_plugin(self, name, suffixes=None, transport=''):
+        ''' Find a plugin named name '''
+
+        if not suffixes:
+            if self.class_name:
+                suffixes = ['.py']
+            else:
+                if transport == 'winrm':
+                    suffixes = ['.ps1', '']
+                else:
+                    suffixes = ['.py', '']
+
+        for suffix in suffixes:
+            full_name = '%s%s' % (name, suffix)
+            if full_name in self._plugin_path_cache:
+                return self._plugin_path_cache[full_name]
+
+            for i in self._get_paths():
+                path = os.path.join(i, full_name)
+                if os.path.isfile(path):
+                    self._plugin_path_cache[full_name] = path
+                    return path
+
+        return None
+
+    def has_plugin(self, name):
+        ''' Checks if a plugin named name exists '''
+
+        return self.find_plugin(name) is not None
+
+    __contains__ = has_plugin
+
+    def get(self, name, *args, **kwargs):
+        ''' instantiates a plugin of the given name using arguments '''
+
+        if name in self.aliases:
+            name = self.aliases[name]
+        path = self.find_plugin(name)
+        if path is None:
+            return None
+        if path not in self._module_cache:
+            self._module_cache[path] = imp.load_source('.'.join([self.package, name]), path)
+        return getattr(self._module_cache[path], self.class_name)(*args, **kwargs)
+
+    def all(self, *args, **kwargs):
+        ''' instantiates all plugins with the same arguments '''
+
+        for i in self._get_paths():
+            matches = glob.glob(os.path.join(i, "*.py"))
+            matches.sort()
+            for path in matches:
+                name, ext = os.path.splitext(os.path.basename(path))
+                if name.startswith("_"):
+                    continue
+                if path not in self._module_cache:
+                    self._module_cache[path] = imp.load_source('.'.join([self.package, name]), path)
+                yield getattr(self._module_cache[path], self.class_name)(*args, **kwargs)
+
+action_loader = PluginLoader(
+    'ActionModule',
+    'ansible.runner.action_plugins',
+    C.DEFAULT_ACTION_PLUGIN_PATH,
+    'action_plugins'
+)
+
+cache_loader = PluginLoader(
+    'CacheModule',
+    'ansible.cache',
+    C.DEFAULT_CACHE_PLUGIN_PATH,
+    'cache_plugins'
+)
+
+callback_loader = PluginLoader(
+    'CallbackModule',
+    'ansible.callback_plugins',
+    C.DEFAULT_CALLBACK_PLUGIN_PATH,
+    'callback_plugins'
+)
+
+connection_loader = PluginLoader(
+    'Connection',
+    'ansible.runner.connection_plugins',
+    C.DEFAULT_CONNECTION_PLUGIN_PATH,
+    'connection_plugins',
+    aliases={'paramiko': 'paramiko_ssh'}
+)
+
+shell_loader = PluginLoader(
+    'ShellModule',
+    'ansible.runner.shell_plugins',
+    'shell_plugins',
+    'shell_plugins',
+)
+
+module_finder = PluginLoader(
+    '',
+    'ansible.modules',
+    C.DEFAULT_MODULE_PATH,
+    'library'
+)
+
+lookup_finder = PluginLoader(
+    'LookupModule',
+    'ansible.runner.lookup_plugins',
+    C.DEFAULT_LOOKUP_PLUGIN_PATH,
+    'lookup_plugins'
+)
+
+vars_finder = PluginLoader(
+    'VarsModule',
+    'ansible.inventory.vars_plugins',
+    C.DEFAULT_VARS_PLUGIN_PATH,
+    'vars_plugins'
+)
+
+filter_finder = PluginLoader(
+    'FilterModule',
+    'ansible.runner.filter_plugins',
+    C.DEFAULT_FILTER_PLUGIN_PATH,
+    'filter_plugins'
+)
+
+fragment_finder = PluginLoader(
+    'ModuleDocFragment',
+    'ansible.utils.module_docs_fragments',
+    os.path.join(os.path.dirname(__file__), 'module_docs_fragments'),
+    '',
+)