Merge pull request #11168 from sivel/v1-sync

Resync the v1 directory with v1_last. Fixes #11162

v1/ansible/constants.py

@@ -134,7 +134,10 @@ DEFAULT_SU_FLAGS          = get_config(p, DEFAULTS, 'su_flags', 'ANSIBLE_SU_FLAG
 DEFAULT_SU_USER           = get_config(p, DEFAULTS, 'su_user', 'ANSIBLE_SU_USER', 'root')
 DEFAULT_ASK_SU_PASS       = get_config(p, DEFAULTS, 'ask_su_pass', 'ANSIBLE_ASK_SU_PASS', False, boolean=True)
 DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
-DEFAULT_LOG_PATH               = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))
+DEFAULT_LOG_PATH          = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))
+
+# selinux
+DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf', islist=True)
 
 #TODO: get rid of ternary chain mess
 BECOME_METHODS            = ['sudo','su','pbrun','pfexec','runas']
@@ -176,6 +179,9 @@ DEFAULT_LOAD_CALLBACK_PLUGINS  = get_config(p, DEFAULTS, 'bin_ansible_callbacks'
 DEFAULT_FORCE_HANDLERS         = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
 
 
+RETRY_FILES_ENABLED            = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True)
+RETRY_FILES_SAVE_PATH          = get_config(p, DEFAULTS, 'retry_files_save_path', 'ANSIBLE_RETRY_FILES_SAVE_PATH', '~/')
+
 # CONNECTION RELATED
 ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None)
 ANSIBLE_SSH_CONTROL_PATH       = get_config(p, 'ssh_connection', 'control_path', 'ANSIBLE_SSH_CONTROL_PATH', "%(directory)s/ansible-ssh-%%h-%%p-%%r")

v1/ansible/inventory/__init__.py

@@ -36,7 +36,7 @@ class Inventory(object):
     Host inventory for ansible.
     """
 
-    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 
+    __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset',
                   'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
                   '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir']
 
@@ -53,7 +53,7 @@ class Inventory(object):
         self._vars_per_host  = {}
         self._vars_per_group = {}
         self._hosts_cache    = {}
-        self._groups_list    = {} 
+        self._groups_list    = {}
         self._pattern_cache  = {}
 
         # to be set by calling set_playbook_basedir by playbook code

v1/ansible/module_utils/cloudstack.py

@@ -64,14 +64,12 @@ class AnsibleCloudStack:
         api_secret = self.module.params.get('secret_key')
         api_url = self.module.params.get('api_url')
         api_http_method = self.module.params.get('api_http_method')
-        api_timeout = self.module.params.get('api_timeout')
 
         if api_key and api_secret and api_url:
             self.cs = CloudStack(
                 endpoint=api_url,
                 key=api_key,
                 secret=api_secret,
-                timeout=api_timeout,
                 method=api_http_method
                 )
         else:

v1/ansible/module_utils/facts.py

@@ -99,8 +99,9 @@ class Facts(object):
                     ('/etc/os-release', 'SuSE'),
                     ('/etc/gentoo-release', 'Gentoo'),
                     ('/etc/os-release', 'Debian'),
+                    ('/etc/lsb-release', 'Mandriva'),
                     ('/etc/os-release', 'NA'),
-                    ('/etc/lsb-release', 'Mandriva'))
+                )
     SELINUX_MODE_DICT = { 1: 'enforcing', 0: 'permissive', -1: 'disabled' }
 
     # A list of dicts.  If there is a platform with more than one
@@ -416,11 +417,13 @@ class Facts(object):
                                                 self.facts['distribution_version'] = self.facts['distribution_version'] + '.' + release.group(1)
                         elif name == 'Debian':
                             data = get_file_content(path)
-                            if 'Debian' in data or 'Raspbian' in data:
+                            if 'Ubuntu' in data:
+                                break # Ubuntu gets correct info from python functions
+                            elif 'Debian' in data or 'Raspbian' in data:
                                 release = re.search("PRETTY_NAME=[^(]+ \(?([^)]+?)\)", data)
                                 if release:
                                     self.facts['distribution_release'] = release.groups()[0]
-                                break
+                                    break
                         elif name == 'Mandriva':
                             data = get_file_content(path)
                             if 'Mandriva' in data:
@@ -2160,7 +2163,7 @@ class DarwinNetwork(GenericBsdIfconfigNetwork, Network):
         current_if['media'] = 'Unknown' # Mac does not give us this
         current_if['media_select'] = words[1]
         if len(words) > 2:
-            current_if['media_type'] = words[2][1:]
+            current_if['media_type'] = words[2][1:-1]
         if len(words) > 3:
             current_if['media_options'] = self.get_options(words[3])
 
@@ -2545,6 +2548,43 @@ class LinuxVirtual(Virtual):
         self.facts['virtualization_role'] = 'NA'
         return
 
+class FreeBSDVirtual(Virtual):
+    """
+    This is a FreeBSD-specific subclass of Virtual.  It defines
+    - virtualization_type
+    - virtualization_role
+    """
+    platform = 'FreeBSD'
+
+    def __init__(self):
+        Virtual.__init__(self)
+
+    def populate(self):
+        self.get_virtual_facts()
+        return self.facts
+
+    def get_virtual_facts(self):
+        self.facts['virtualization_type'] = ''
+        self.facts['virtualization_role'] = ''
+
+class OpenBSDVirtual(Virtual):
+    """
+    This is a OpenBSD-specific subclass of Virtual.  It defines
+    - virtualization_type
+    - virtualization_role
+    """
+    platform = 'OpenBSD'
+
+    def __init__(self):
+        Virtual.__init__(self)
+
+    def populate(self):
+        self.get_virtual_facts()
+        return self.facts
+
+    def get_virtual_facts(self):
+        self.facts['virtualization_type'] = ''
+        self.facts['virtualization_role'] = ''
 
 class HPUXVirtual(Virtual):
     """

v1/ansible/module_utils/powershell.ps1

@@ -65,7 +65,7 @@ Function Exit-Json($obj)
         $obj = New-Object psobject
     }
 
-    echo $obj | ConvertTo-Json -Depth 99
+    echo $obj | ConvertTo-Json -Compress -Depth 99
     Exit
 }
 
@@ -89,7 +89,7 @@ Function Fail-Json($obj, $message = $null)
 
     Set-Attr $obj "msg" $message
     Set-Attr $obj "failed" $true
-    echo $obj | ConvertTo-Json -Depth 99
+    echo $obj | ConvertTo-Json -Compress -Depth 99
     Exit 1
 }
 

v1/ansible/module_utils/urls.py

@@ -50,6 +50,15 @@ try:
 except:
     HAS_SSL=False
 
+HAS_MATCH_HOSTNAME = True
+try:
+    from ssl import match_hostname, CertificateError
+except ImportError:
+    try:
+        from backports.ssl_match_hostname import match_hostname, CertificateError
+    except ImportError:
+        HAS_MATCH_HOSTNAME = False
+
 import httplib
 import os
 import re
@@ -293,11 +302,13 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     connect_result = s.recv(4096)
                     self.validate_proxy_response(connect_result)
                     ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
+                    match_hostname(ssl_s.getpeercert(), self.hostname)
                 else:
                     self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
             else:
                 s.connect((self.hostname, self.port))
                 ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
+                match_hostname(ssl_s.getpeercert(), self.hostname)
             # close the ssl connection
             #ssl_s.unwrap()
             s.close()
@@ -311,6 +322,9 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     'Use validate_certs=no or make sure your managed systems have a valid CA certificate installed. ' + \
                     'Paths checked for this platform: %s' % ", ".join(paths_checked)
                 )
+        except CertificateError:
+            self.module.fail_json(msg="SSL Certificate does not belong to %s.  Make sure the url has a certificate that belongs to it or use validate_certs=no (insecure)" % self.hostname)
+
         try:
             # cleanup the temp file created, don't worry
             # if it fails for some reason
@@ -363,28 +377,29 @@ def fetch_url(module, url, data=None, headers=None, method=None,
     # FIXME: change the following to use the generic_urlparse function
     #        to remove the indexed references for 'parsed'
     parsed = urlparse.urlparse(url)
-    if parsed[0] == 'https':
-        if not HAS_SSL and validate_certs:
+    if parsed[0] == 'https' and validate_certs:
+        if not HAS_SSL:
             if distribution == 'Redhat':
                 module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended. You can also install python-ssl from EPEL')
             else:
                 module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended')
-
-        elif validate_certs:
-            # do the cert validation
-            netloc = parsed[1]
-            if '@' in netloc:
-                netloc = netloc.split('@', 1)[1]
-            if ':' in netloc:
-                hostname, port = netloc.split(':', 1)
-                port = int(port)
-            else:
-                hostname = netloc
-                port = 443
-            # create the SSL validation handler and
-            # add it to the list of handlers
-            ssl_handler = SSLValidationHandler(module, hostname, port)
-            handlers.append(ssl_handler)
+        if not HAS_MATCH_HOSTNAME:
+            module.fail_json(msg='Available SSL validation does not check that the certificate matches the hostname.  You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer.  You could also use validate_certs=no, however this is unsafe and not recommended')
+
+        # do the cert validation
+        netloc = parsed[1]
+        if '@' in netloc:
+            netloc = netloc.split('@', 1)[1]
+        if ':' in netloc:
+            hostname, port = netloc.split(':', 1)
+            port = int(port)
+        else:
+            hostname = netloc
+            port = 443
+        # create the SSL validation handler and
+        # add it to the list of handlers
+        ssl_handler = SSLValidationHandler(module, hostname, port)
+        handlers.append(ssl_handler)
 
     if parsed[0] != 'ftp':
         username = module.params.get('url_username', '')

v1/ansible/runner/connection_plugins/ssh.py

@@ -16,22 +16,21 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import fcntl
-import gettext
-import hmac
 import os
+import re
+import subprocess
+import shlex
 import pipes
-import pty
-import pwd
 import random
-import re
 import select
-import shlex
-import subprocess
-import time
+import fcntl
+import hmac
+import pwd
+import gettext
+import pty
 from hashlib import sha1
 import ansible.constants as C
-from ansible.callbacks import vvv, vv
+from ansible.callbacks import vvv
 from ansible import errors
 from ansible import utils
 
@@ -257,51 +256,7 @@ class Connection(object):
             vvv("EXEC previous known host file not found for %s" % host)
         return True
 
-    def exec_command(self, *args, **kwargs):
-        """ Wrapper around _exec_command to retry in the case of an ssh
-            failure
-
-            Will retry if:
-            * an exception is caught
-            * ssh returns 255
-
-            Will not retry if
-            * remaining_tries is <2
-            * retries limit reached
-            """
-        remaining_tries = C.get_config(
-            C.p, 'ssh_connection', 'retries',
-            'ANSIBLE_SSH_RETRIES', 3, integer=True) + 1
-        cmd_summary = "%s %s..." % (args[0], str(kwargs)[:200])
-        for attempt in xrange(remaining_tries):
-            pause = 2 ** attempt - 1
-            if pause > 30:
-                pause = 30
-            time.sleep(pause)
-            try:
-                return_tuple = self._exec_command(*args, **kwargs)
-            except Exception as e:
-                msg = ("ssh_retry: attempt: %d, caught exception(%s) from cmd "
-                       "(%s).") % (attempt, e, cmd_summary)
-                vv(msg)
-                if attempt == remaining_tries - 1:
-                    raise e
-                else:
-                    continue
-            # 0 = success
-            # 1-254 = remote command return code
-            # 255 = failure from the ssh command itself
-            if return_tuple[0] != 255:
-                break
-            else:
-                msg = ('ssh_retry: attempt: %d, ssh return code is 255. cmd '
-                       '(%s).') % (attempt, cmd_summary)
-                vv(msg)
-
-        return return_tuple
-
-
-    def _exec_command(self, cmd, tmp_path, become_user=None, sudoable=False, executable='/bin/sh', in_data=None):
+    def exec_command(self, cmd, tmp_path, become_user=None, sudoable=False, executable='/bin/sh', in_data=None):
         ''' run a command on the remote host '''
 
         if sudoable and self.runner.become and self.runner.become_method not in self.become_methods_supported:

v1/ansible/utils/__init__.py

@@ -1024,9 +1024,9 @@ def base_parser(constants=C, usage="", output_opts=False, runas_opts=False,
 
     if runas_opts:
         # priv user defaults to root later on to enable detecting when this option was given here
-        parser.add_option('-K', '--ask-sudo-pass', default=False, dest='ask_sudo_pass', action='store_true',
+        parser.add_option('-K', '--ask-sudo-pass', default=constants.DEFAULT_ASK_SUDO_PASS, dest='ask_sudo_pass', action='store_true',
             help='ask for sudo password (deprecated, use become)')
-        parser.add_option('--ask-su-pass', default=False, dest='ask_su_pass', action='store_true',
+        parser.add_option('--ask-su-pass', default=constants.DEFAULT_ASK_SU_PASS, dest='ask_su_pass', action='store_true',
             help='ask for su password (deprecated, use become)')
         parser.add_option("-s", "--sudo", default=constants.DEFAULT_SUDO, action="store_true", dest='sudo',
             help="run operations with sudo (nopasswd) (deprecated, use become)")
@@ -1617,7 +1617,9 @@ def _load_vars_from_folder(folder_path, results, vault_password=None):
     names.sort()
 
     # do not parse hidden files or dirs, e.g. .svn/
-    paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
+    paths = [os.path.join(folder_path, name) for name in names
+            if not name.startswith('.')
+            and os.path.splitext(name)[1] in C.YAML_FILENAME_EXTENSIONS]
     for path in paths:
         _found, results = _load_vars_from_path(path, results, vault_password=vault_password)
     return results

v1/ansible/utils/module_docs.py

@@ -23,6 +23,8 @@ import ast
 import yaml
 import traceback
 
+from collections import MutableMapping, MutableSet, MutableSequence
+
 from ansible import utils
 
 # modules that are ok that they do not have documentation strings
@@ -86,7 +88,14 @@ def get_docstring(filename, verbose=False):
                             if not doc.has_key(key):
                                 doc[key] = value
                             else:
-                                doc[key].update(value)
+                                if isinstance(doc[key], MutableMapping):
+                                    doc[key].update(value)
+                                elif isinstance(doc[key], MutableSet):
+                                    doc[key].add(value)
+                                elif isinstance(doc[key], MutableSequence):
+                                    doc[key] = sorted(frozenset(doc[key] + value))
+                                else:
+                                    raise Exception("Attempt to extend a documentation fragement of unknown type")
 
                 if 'EXAMPLES' in (t.id for t in child.targets):
                     plainexamples = child.value.s[1:]  # Skip first empty line