Merge pull request #12636 from bcoca/galaxy

Galaxy

lib/ansible/cli/__init__.py

@@ -512,3 +512,16 @@ class CLI(object):
 
         return vault_pass
 
+    def get_opt(self, k, defval=""):
+        """
+        Returns an option from an Optparse values instance.
+        """
+        try:
+            data = getattr(self.options, k)
+        except:
+            return defval
+        if k == "roles_path":
+            if os.pathsep in data:
+                data = data.split(os.pathsep)[0]
+        return data
+

lib/ansible/cli/galaxy.py

@@ -29,8 +29,6 @@ from distutils.version import LooseVersion
 from jinja2 import Environment
 
 import ansible.constants as C
-import ansible.utils
-import ansible.galaxy
 from ansible.cli import CLI
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.galaxy import Galaxy
@@ -126,19 +124,6 @@ class GalaxyCLI(CLI):
 
         self.execute()
 
-    def get_opt(self, k, defval=""):
-        """
-        Returns an option from an Optparse values instance.
-        """
-        try:
-            data = getattr(self.options, k)
-        except:
-            return defval
-        if k == "roles_path":
-            if os.pathsep in data:
-                data = data.split(os.pathsep)[0]
-        return data
-
     def exit_without_ignore(self, rc=1):
         """
         Exits with the specified return code unless the
@@ -147,40 +132,6 @@ class GalaxyCLI(CLI):
         if not self.get_opt("ignore_errors", False):
             raise AnsibleError('- you can use --ignore-errors to skip failed roles and finish processing the list.')
 
-    def parse_requirements_files(self, role):
-        if 'role' in role:
-            # Old style: {role: "galaxy.role,version,name", other_vars: "here" }
-            role_info = role_spec_parse(role['role'])
-            if isinstance(role_info, dict):
-                # Warning: Slight change in behaviour here.  name may be being
-                # overloaded.  Previously, name was only a parameter to the role.
-                # Now it is both a parameter to the role and the name that
-                # ansible-galaxy will install under on the local system.
-                if 'name' in role and 'name' in role_info:
-                    del role_info['name']
-                role.update(role_info)
-        else:
-            # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
-            if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'):
-                role["src"] = "git+" + role["src"]
-
-            if '+' in role["src"]:
-                (scm, src) = role["src"].split('+')
-                role["scm"] = scm
-                role["src"] = src
-
-            if 'name' not in role:
-                role["name"] = GalaxyRole.url_to_spec(role["src"])
-
-            if 'version' not in role:
-                role['version'] = ''
-
-            if 'scm' not in role:
-                role['scm'] = None
-
-        return role
-
-
     def _display_role_info(self, role_info):
 
         text = "\nRole: %s \n" % role_info['name']
@@ -298,9 +249,8 @@ class GalaxyCLI(CLI):
         data = ''
         for role in self.args:
 
-            role_info = {}
+            role_info = {'role_path': roles_path}
             gr = GalaxyRole(self.galaxy, role)
-            #self.galaxy.add_role(gr)
 
             install_info = gr.install_info
             if install_info:
@@ -351,98 +301,52 @@ class GalaxyCLI(CLI):
 
         no_deps    = self.get_opt("no_deps", False)
         force      = self.get_opt('force', False)
-        roles_path = self.get_opt("roles_path")
 
-        roles_done = []
         roles_left = []
         if role_file:
-            self.display.debug('Getting roles from %s' % role_file)
             try:
-                self.display.debug('Processing role file: %s' % role_file)
                 f = open(role_file, 'r')
                 if role_file.endswith('.yaml') or role_file.endswith('.yml'):
-                    try:
-                        rolesparsed = map(self.parse_requirements_files, yaml.safe_load(f))
-                    except Exception as e:
-                       raise AnsibleError("%s does not seem like a valid yaml file: %s" % (role_file, str(e)))
-                    roles_left = [GalaxyRole(self.galaxy, **r) for r in rolesparsed]
+                    for role in yaml.safe_load(f.read()):
+                        self.display.debug('found role %s in yaml file' % str(role))
+                        if 'name' not in role:
+                            if 'src' in role:
+                                role['name'] = RoleRequirement.repo_url_to_role_name(role['src'])
                             else:
+                                raise AnsibleError("Must specify name or src for role")
+                        roles_left.append(GalaxyRole(self.galaxy, **role))
+                else:
+                    self.display.deprecated("going forward only the yaml format will be supported")
                     # roles listed in a file, one per line
-                    self.display.deprecated("Non yaml files for role requirements")
-                    for rname in f.readlines():
-                        if rname.startswith("#") or rname.strip() == '':
-                            continue
-                        roles_left.append(GalaxyRole(self.galaxy, rname.strip()))
+                    for rline in f.readlines():
+                        self.display.debug('found role %s in text file' % str(rline))
+                        roles_left.append(GalaxyRole(self.galaxy, **RoleRequirement.role_spec_parse(rline)))
                 f.close()
-            except (IOError,OSError) as e:
-                raise AnsibleError("Unable to read requirements file (%s): %s" % (role_file, str(e)))
+            except (IOError, OSError) as e:
+                self.display.error('Unable to open %s: %s' % (role_file, str(e)))
         else:
             # roles were specified directly, so we'll just go out grab them
             # (and their dependencies, unless the user doesn't want us to).
             for rname in self.args:
                 roles_left.append(GalaxyRole(self.galaxy, rname.strip()))
 
-        while len(roles_left) > 0:
+        for role in roles_left:
+            self.display.debug('Installing role %s ' % role.name)
             # query the galaxy API for the role data
             role_data = None
             role = roles_left.pop(0)
-            role_path = role.path
 
             if role.install_info is not None and not force:
                 self.display.display('- %s is already installed, skipping.' % role.name)
                 continue
 
-            if role_path:
-                self.options.roles_path = role_path
-            else:
-                self.options.roles_path = roles_path
-
-            self.display.debug('Installing role %s from %s' % (role.name, self.options.roles_path))
-
-            tmp_file = None
-            installed = False
-            if role.src and os.path.isfile(role.src):
-                # installing a local tar.gz
-                tmp_file = role.src
-            else:
-                if role.scm:
-                    # create tar file from scm url
-                    tmp_file = GalaxyRole.scm_archive_role(role.scm, role.src, role.version, role.name)
-                if role.src:
-                    if '://' not in role.src:
-                        role_data = self.api.lookup_role_by_name(role.src)
-                        if not role_data:
-                            self.display.warning("- sorry, %s was not found on %s." % (role.src, self.options.api_server))
-                            self.exit_without_ignore()
-                            continue
-
-                        role_versions = self.api.fetch_role_related('versions', role_data['id'])
-                        if not role.version:
-                            # convert the version names to LooseVersion objects
-                            # and sort them to get the latest version. If there
-                            # are no versions in the list, we'll grab the head
-                            # of the master branch
-                            if len(role_versions) > 0:
-                                loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
-                                loose_versions.sort()
-                                role.version = str(loose_versions[-1])
-                            else:
-                                role.version = 'master'
-                        elif role.version != 'master':
-                            if role_versions and role.version not in [a.get('name', None) for a in role_versions]:
-                                self.display.warning('role is %s' % role)
-                                self.display.warning("- the specified version (%s) was not found in the list of available versions (%s)." % (role.version, role_versions))
+            try:
+                installed = role.install()
+            except AnsibleError as e:
+                self.display.warning("- %s was NOT installed successfully: %s " % (role.name, str(e)))
                 self.exit_without_ignore()
                 continue
 
-                    # download the role. if --no-deps was specified, we stop here,
-                    # otherwise we recursively grab roles and all of their deps.
-                    tmp_file = role.fetch(role_data)
-            if tmp_file:
-                installed = role.install(tmp_file)
-                # we're done with the temp file, clean it up
-                if tmp_file != role.src:
-                    os.unlink(tmp_file)
             # install dependencies, if we want them
             if not no_deps and installed:
                 role_dependencies = role.metadata.get('dependencies', [])
@@ -460,9 +364,10 @@ class GalaxyCLI(CLI):
                     else:
                         self.display.display('- dependency %s is already installed, skipping.' % dep_name)
 
-            if not tmp_file or not installed:
+            if not installed:
                 self.display.warning("- %s was NOT installed successfully." % role.name)
                 self.exit_without_ignore()
+
         return 0
 
     def execute_remove(self):

lib/ansible/playbook/role/__init__.py

@@ -21,19 +21,14 @@ __metaclass__ = type
 
 from six import iteritems, string_types
 
-import inspect
 import os
 
-from hashlib import sha1
-
 from ansible.errors import AnsibleError, AnsibleParserError
-from ansible.parsing import DataLoader
 from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.become import Become
 from ansible.playbook.conditional import Conditional
 from ansible.playbook.helpers import load_list_of_blocks
-from ansible.playbook.role.include import RoleInclude
 from ansible.playbook.role.metadata import RoleMetadata
 from ansible.playbook.taggable import Taggable
 from ansible.plugins import get_all_plugin_loaders

lib/ansible/playbook/role/requirement.py

@@ -19,11 +19,14 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from six import iteritems, string_types
+from six import string_types
 
 import os
+import shutil
+import subprocess
+import tempfile
 
-from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.errors import AnsibleError
 from ansible.playbook.role.definition import RoleDefinition
 
 __all__ = ['RoleRequirement']
@@ -73,7 +76,7 @@ class RoleRequirement(RoleDefinition):
     def _preprocess_role_spec(self, ds):
         if 'role' in ds:
             # Old style: {role: "galaxy.role,version,name", other_vars: "here" }
-            role_info = role_spec_parse(ds['role'])
+            role_info = RoleRequirement.role_spec_parse(ds['role'])
             if isinstance(role_info, dict):
                 # Warning: Slight change in behaviour here.  name may be being
                 # overloaded.  Previously, name was only a parameter to the role.
@@ -96,7 +99,7 @@ class RoleRequirement(RoleDefinition):
                 ds["role"] = ds["name"]
                 del ds["name"]
             else:
-                ds["role"] = repo_url_to_role_name(ds["src"])
+                ds["role"] = RoleRequirement.repo_url_to_role_name(ds["src"])
 
             # set some values to a default value, if none were specified
             ds.setdefault('version', '')
@@ -104,7 +107,8 @@ class RoleRequirement(RoleDefinition):
 
         return ds
 
-def repo_url_to_role_name(repo_url):
+    @staticmethod
+    def repo_url_to_role_name(repo_url):
         # gets the role name out of a repo like
         # http://git.example.com/repos/repo.git" => "repo"
 
@@ -119,7 +123,8 @@ def repo_url_to_role_name(repo_url):
             trailing_path = trailing_path.split(',')[0]
         return trailing_path
 
-def role_spec_parse(role_spec):
+    @staticmethod
+    def role_spec_parse(role_spec):
         # takes a repo and a version like
         # git+http://git.example.com/repos/repo.git,v1.0
         # and returns a list of properties such as:
@@ -156,50 +161,83 @@ def role_spec_parse(role_spec):
         if len(tokens) == 3:
             role_name = tokens[2]
         else:
-        role_name = repo_url_to_role_name(tokens[0])
+            role_name = RoleRequirement.repo_url_to_role_name(tokens[0])
 
         if scm and not role_version:
             role_version = default_role_versions.get(scm, '')
 
-    return dict(scm=scm, src=role_url, version=role_version, role_name=role_name)
+        return dict(scm=scm, src=role_url, version=role_version, name=role_name)
 
-# FIXME: all of these methods need to be cleaned up/reorganized below this
-def get_opt(options, k, defval=""):
-    """
-    Returns an option from an Optparse values instance.
-    """
-    try:
-        data = getattr(options, k)
-    except:
-        return defval
-    if k == "roles_path":
-        if os.pathsep in data:
-            data = data.split(os.pathsep)[0]
-    return data
+    @staticmethod
+    def role_yaml_parse(role):
 
-def get_role_path(role_name, options):
-    """
-    Returns the role path based on the roles_path option
-    and the role name.
-    """
-    roles_path = get_opt(options,'roles_path')
-    roles_path = os.path.join(roles_path, role_name)
-    roles_path = os.path.expanduser(roles_path)
-    return roles_path
+        if 'role' in role:
+            # Old style: {role: "galaxy.role,version,name", other_vars: "here" }
+            role_info = RoleRequirement.role_spec_parse(role['role'])
+            if isinstance(role_info, dict):
+                # Warning: Slight change in behaviour here.  name may be being
+                # overloaded.  Previously, name was only a parameter to the role.
+                # Now it is both a parameter to the role and the name that
+                # ansible-galaxy will install under on the local system.
+                if 'name' in role and 'name' in role_info:
+                    del role_info['name']
+                role.update(role_info)
+        else:
+            # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
+            if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'):
+                role["src"] = "git+" + role["src"]
 
-def get_role_metadata(role_name, options):
-    """
-    Returns the metadata as YAML, if the file 'meta/main.yml'
-    exists in the specified role_path
-    """
-    role_path = os.path.join(get_role_path(role_name, options), 'meta/main.yml')
+            if '+' in role["src"]:
+                (scm, src) = role["src"].split('+')
+                role["scm"] = scm
+                role["src"] = src
+
+            if 'name' not in role:
+                role["name"] = RoleRequirement.repo_url_to_role_name(role["src"])
+
+            if 'version' not in role:
+                role['version'] = ''
+
+            if 'scm' not in role:
+                role['scm'] = None
+
+        return role
+
+    @staticmethod
+    def scm_archive_role(src, scm='git', name=None, version='HEAD'):
+        if scm not in ['hg', 'git']:
+            raise AnsibleError("- scm %s is not currently supported" % scm)
+        tempdir = tempfile.mkdtemp()
+        clone_cmd = [scm, 'clone', src, name]
+        with open('/dev/null', 'w') as devnull:
             try:
-        if os.path.isfile(role_path):
-            f = open(role_path, 'r')
-            meta_data = yaml.safe_load(f)
-            f.close()
-            return meta_data
-        else:
-            return None
+                popen = subprocess.Popen(clone_cmd, cwd=tempdir, stdout=devnull, stderr=devnull)
             except:
-        return None    
+                raise AnsibleError("error executing: %s" % " ".join(clone_cmd))
+            rc = popen.wait()
+        if rc != 0:
+            raise AnsibleError ("- command %s failed in directory %s" % (' '.join(clone_cmd), tempdir))
+
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.tar')
+        if scm == 'hg':
+            archive_cmd = ['hg', 'archive', '--prefix', "%s/" % name]
+            if version:
+                archive_cmd.extend(['-r', version])
+            archive_cmd.append(temp_file.name)
+        if scm == 'git':
+            archive_cmd = ['git', 'archive', '--prefix=%s/' % name, '--output=%s' % temp_file.name]
+            if version:
+                archive_cmd.append(version)
+            else:
+                archive_cmd.append('HEAD')
+
+        with open('/dev/null', 'w') as devnull:
+            popen = subprocess.Popen(archive_cmd, cwd=os.path.join(tempdir, name),
+                                     stderr=devnull, stdout=devnull)
+            rc = popen.wait()
+        if rc != 0:
+            raise AnsibleError("- command %s failed in directory %s" % (' '.join(archive_cmd), tempdir))
+
+        shutil.rmtree(tempdir, ignore_errors=True)
+        return temp_file.name
+

lib/ansible/utils/module_docs.py

@@ -63,6 +63,7 @@ def get_docstring(filename, verbose=False):
                         theid = t.id
                     except AttributeError as e:
                         # skip errors can happen when trying to use the normal code
+                        display.warning("Failed to assign id for %t on %s, skipping" % (t, filename))
                         continue
 
                     if 'DOCUMENTATION' in theid:
@@ -119,6 +120,7 @@ def get_docstring(filename, verbose=False):
     except:
         display.error("unable to parse %s" % filename)
         if verbose == True:
+            display.display("unable to parse %s" % filename)
             raise
     return doc, plainexamples, returndocs
 

test/integration/Makefile

@@ -172,7 +172,7 @@ test_galaxy: test_galaxy_spec test_galaxy_yaml
 
 test_galaxy_spec:
 	mytmpdir=$(MYTMPDIR) ; \
-	ansible-galaxy install -r galaxy_rolesfile -p $$mytmpdir/roles ; \
+	ansible-galaxy install -r galaxy_rolesfile -p $$mytmpdir/roles -vvvv ; \
     cp galaxy_playbook.yml $$mytmpdir ; \
     ansible-playbook -i $(INVENTORY) $$mytmpdir/galaxy_playbook.yml -v $(TEST_FLAGS) ; \
     RC=$$? ; \
@@ -181,7 +181,7 @@ test_galaxy_spec:
 
 test_galaxy_yaml:
 	mytmpdir=$(MYTMPDIR) ; \
-	ansible-galaxy install -r galaxy_roles.yml -p $$mytmpdir/roles ; \
+	ansible-galaxy install -r galaxy_roles.yml -p $$mytmpdir/roles -vvvv; \
     cp galaxy_playbook.yml $$mytmpdir ; \
     ansible-playbook -i $(INVENTORY) $$mytmpdir/galaxy_playbook.yml -v $(TEST_FLAGS) ; \
     RC=$$? ; \