Move from md5 to sha1 to work on fips-140 enabled systems

CHANGELOG.md

@@ -58,7 +58,14 @@ Some other notable changes:
 * ec2_ami_search: support for SSD and IOPS provisioned EBS images
 * can set ansible_sudo_exe as an inventory variable which allows specifying
   a different sudo (or equivalent) command
-* git module: Submodule handling has changed.  Previously if you used the ``recursive`` parameter to handle submodules, ansible would track the submodule upstream's head revision.  This has been changed to checkout the version of the submodule specified in the superproject's git repository.  This is inline with what git submodule update does.  If you want the old behaviour use the new module parameter track_submodules=yes
+* git module: Submodule handling has changed.  Previously if you used the
+  ``recursive`` parameter to handle submodules, ansible would track the
+  submodule upstream's head revision.  This has been changed to checkout the
+  version of the submodule specified in the superproject's git repository.
+  This is inline with what git submodule update does.  If you want the old
+  behaviour use the new module parameter track_submodules=yes
+* Checksumming of transferred files has been made more portable and now uses
+  the sha1 algorithm instead of md5 to be compatible with FIPS-140.
 
 And various other bug fixes and improvements ...
 

docsite/rst/developing_modules.rst

@@ -262,7 +262,7 @@ And failures are just as simple (where 'msg' is a required parameter to explain
 
     module.fail_json(msg="Something fatal happened")
 
-There are also other useful functions in the module class, such as module.md5(path).  See
+There are also other useful functions in the module class, such as module.sha1(path).  See
 lib/ansible/module_common.py in the source checkout for implementation details.
 
 Again, modules developed this way are best tested with the hacking/test-module script in the git

docsite/rst/playbooks_prompts.rst

@@ -55,7 +55,7 @@ entered value so you can use it, for instance, with the user module to define a
      - name: "my_password2"
        prompt: "Enter password2"
        private: yes
-       encrypt: "md5_crypt"
+       encrypt: "sha512_crypt"
        confirm: yes
        salt_size: 7
 

docsite/rst/playbooks_variables.rst

@@ -327,9 +327,9 @@ To work with Base64 encoded strings::
     {{ encoded | b64decode }}
     {{ decoded | b64encode }}
 
-To take an md5sum of a filename::
+To take a sha1sum of a filename::
 
-    {{ filename | md5 }}
+    {{ filename | sha1 }}
 
 To cast values as certain types, such as when you input a string as "True" from a vars_prompt and the system
 doesn't know it is a boolean value::

lib/ansible/module_utils/basic.py

@@ -87,9 +87,14 @@ except ImportError:
 
 HAVE_HASHLIB=False
 try:
-    from hashlib import md5 as _md5
+    from hashlib import sha1 as _sha1
     HAVE_HASHLIB=True
 except ImportError:
+    from sha import sha as _sha1
+
+try:
+    from hashlib import md5 as _md5
+except ImportError:
     from md5 import md5 as _md5
 
 try:
@@ -1236,6 +1241,10 @@ class AnsibleModule(object):
         ''' Return MD5 hex digest of local file using digest_from_file(). '''
         return self.digest_from_file(filename, _md5())
 
+    def sha1(self, filename):
+        ''' Return SHA1 hex digest of local file using digest_from_file(). '''
+        return self.digest_from_file(filename, _sha1())
+
     def sha256(self, filename):
         ''' Return SHA-256 hex digest of local file using digest_from_file(). '''
         if not HAVE_HASHLIB:

core @ 6317d3a9

-Subproject commit 2970b339eb8ea6031e6153cabe45459bc2bd5754
+Subproject commit 6317d3a988f7269340cb7a0d105d2c671ca1cd1e

extras @ 5a514ccd

-Subproject commit ad181b7aa949848e3085065e09195cb28c34fdf7
+Subproject commit 5a514ccddae85ccc5802eea8751401600e45c32f

lib/ansible/runner/__init__.py

@@ -53,9 +53,9 @@ from ansible.utils import update_hash
 module_replacer = ModuleReplacer(strip_comments=False)
 
 try:
-    from hashlib import md5 as _md5
+    from hashlib import sha1
 except ImportError:
-    from md5 import md5 as _md5
+    from sha import sha as sha1
 
 HAS_ATFORK=True
 try:
@@ -209,7 +209,7 @@ class Runner(object):
         self.su_user_var      = su_user
         self.su_user          = None
         self.su_pass          = su_pass
-        self.omit_token       = '__omit_place_holder__%s' % _md5(os.urandom(64)).hexdigest()
+        self.omit_token       = '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest()
         self.vault_pass       = vault_pass
         self.no_log           = no_log
         self.run_once         = run_once
@@ -1159,26 +1159,29 @@ class Runner(object):
 
     # *****************************************************
 
-    def _remote_md5(self, conn, tmp, path):
-        ''' takes a remote md5sum without requiring python, and returns 1 if no file '''
-        cmd = conn.shell.md5(path)
+    def _remote_checksum(self, conn, tmp, path):
+        ''' takes a remote checksum and returns 1 if no file '''
+        inject = self.get_inject_vars(conn.host)
+        hostvars = HostVars(inject['combined_cache'], self.inventory, vault_password=self.vault_pass)
+        python_interp = hostvars[conn.host].get('ansible_python_interpreter', 'python')
+        cmd = conn.shell.checksum(path, python_interp)
         data = self._low_level_exec_command(conn, cmd, tmp, sudoable=True)
         data2 = utils.last_non_blank_line(data['stdout'])
         try:
             if data2 == '':
                 # this may happen if the connection to the remote server
-                # failed, so just return "INVALIDMD5SUM" to avoid errors
-                return "INVALIDMD5SUM"
+                # failed, so just return "INVALIDCHECKSUM" to avoid errors
+                return "INVALIDCHECKSUM"
             else:
                 return data2.split()[0]
         except IndexError:
-            sys.stderr.write("warning: md5sum command failed unusually, please report this to the list so it can be fixed\n")
-            sys.stderr.write("command: %s\n" % md5s)
+            sys.stderr.write("warning: Calculating checksum failed unusually, please report this to the list so it can be fixed\n")
+            sys.stderr.write("command: %s\n" % cmd)
             sys.stderr.write("----\n")
             sys.stderr.write("output: %s\n" % data)
             sys.stderr.write("----\n")
             # this will signal that it changed and allow things to keep going
-            return "INVALIDMD5SUM"
+            return "INVALIDCHECKSUM"
 
     # *****************************************************
 

lib/ansible/runner/action_plugins/assemble.py

@@ -108,10 +108,10 @@ class ActionModule(object):
         # Does all work assembling the file
         path = self._assemble_from_fragments(src, delimiter, _re)
 
-        pathmd5 = utils.md5s(path)
-        remote_md5 = self.runner._remote_md5(conn, tmp, dest)
+        path_checksum = utils.checksum_s(path)
+        remote_checksum = self.runner._remote_checksum(conn, tmp, dest)
 
-        if pathmd5 != remote_md5:
+        if path_checksum != remote_checksum:
             resultant = file(path).read()
             if self.runner.diff:
                 dest_result = self.runner._execute_module(conn, tmp, 'slurp', "path=%s" % dest, inject=inject, persist_files=True)

lib/ansible/runner/action_plugins/copy.py

@@ -158,11 +158,11 @@ class ActionModule(object):
                 tmp_path = self.runner._make_tmp_path(conn)
 
         for source_full, source_rel in source_files:
-            # Generate the MD5 hash of the local file.
-            local_md5 = utils.md5(source_full)
+            # Generate a hash of the local file.
+            local_checksum = utils.checksum(source_full)
 
-            # If local_md5 is not defined we can't find the file so we should fail out.
-            if local_md5 is None:
+            # If local_checksum is not defined we can't find the file so we should fail out.
+            if local_checksum is None:
                 result = dict(failed=True, msg="could not find src=%s" % source_full)
                 return ReturnData(conn=conn, result=result)
 
@@ -174,27 +174,27 @@ class ActionModule(object):
             else:
                 dest_file = conn.shell.join_path(dest)
 
-            # Attempt to get the remote MD5 Hash.
-            remote_md5 = self.runner._remote_md5(conn, tmp_path, dest_file)
+            # Attempt to get the remote checksum
+            remote_checksum = self.runner._remote_checksum(conn, tmp_path, dest_file)
 
-            if remote_md5 == '3':
-                # The remote_md5 was executed on a directory.
+            if remote_checksum == '3':
+                # The remote_checksum was executed on a directory.
                 if content is not None:
                     # If source was defined as content remove the temporary file and fail out.
                     self._remove_tempfile_if_content_defined(content, content_tempfile)
                     result = dict(failed=True, msg="can not use content with a dir as dest")
                     return ReturnData(conn=conn, result=result)
                 else:
-                    # Append the relative source location to the destination and retry remote_md5.
+                    # Append the relative source location to the destination and retry remote_checksum
                     dest_file = conn.shell.join_path(dest, source_rel)
-                    remote_md5 = self.runner._remote_md5(conn, tmp_path, dest_file)
+                    remote_checksum = self.runner._remote_checksum(conn, tmp_path, dest_file)
 
-            if remote_md5 != '1' and not force:
+            if remote_checksum != '1' and not force:
                 # remote_file does not exist so continue to next iteration.
                 continue
 
-            if local_md5 != remote_md5:
-                # The MD5 hashes don't match and we will change or error out.
+            if local_checksum != remote_checksum:
+                # The checksums don't match and we will change or error out.
                 changed = True
 
                 # Create a tmp_path if missing only if this is not recursive.
@@ -254,7 +254,7 @@ class ActionModule(object):
                 module_executed = True
 
             else:
-                # no need to transfer the file, already correct md5, but still need to call
+                # no need to transfer the file, already correct hash, but still need to call
                 # the file module in case we want to change attributes
                 self._remove_tempfile_if_content_defined(content, content_tempfile)
 
@@ -283,8 +283,8 @@ class ActionModule(object):
                 module_executed = True
 
             module_result = module_return.result
-            if not module_result.get('md5sum'):
-                module_result['md5sum'] = local_md5
+            if not module_result.get('checksum'):
+                module_result['checksum'] = local_checksum
             if module_result.get('failed') == True:
                 return module_return
             if module_result.get('changed') == True:

lib/ansible/runner/action_plugins/fetch.py

@@ -50,26 +50,40 @@ class ActionModule(object):
         flat = utils.boolean(flat)
         fail_on_missing = options.get('fail_on_missing', False)
         fail_on_missing = utils.boolean(fail_on_missing)
-        validate_md5 = options.get('validate_md5', True)
-        validate_md5 = utils.boolean(validate_md5)
+        validate_checksum = options.get('validate_checksum', None)
+        if validate_checksum is not None:
+            validate_checksum = utils.boolean(validate_checksum)
+        # Alias for validate_checksum (old way of specifying it)
+        validate_md5 = options.get('validate_md5', None)
+        if validate_md5 is not None:
+            validate_md5 = utils.boolean(validate_md5)
+        if validate_md5 is None and validate_checksum is None:
+            # Default
+            validate_checksum = True
+        elif validate_checksum is None:
+            validate_checksum = validate_md5
+        elif validate_md5 is not None and validate_checksum is not None:
+            results = dict(failed=True, msg="validate_checksum and validate_md5 cannot both be specified")
+            return ReturnData(conn, result=results)
+
         if source is None or dest is None:
             results = dict(failed=True, msg="src and dest are required")
             return ReturnData(conn=conn, result=results)
 
         source = conn.shell.join_path(source)
 
-        # calculate md5 sum for the remote file
-        remote_md5 = self.runner._remote_md5(conn, tmp, source)
+        # calculate checksum for the remote file
+        remote_checksum = self.runner._remote_checksum(conn, tmp, source)
 
         # use slurp if sudo and permissions are lacking
         remote_data = None
-        if remote_md5 in ('1', '2') or self.runner.sudo:
+        if remote_checksum in ('1', '2') or self.runner.sudo:
             slurpres = self.runner._execute_module(conn, tmp, 'slurp', 'src=%s' % source, inject=inject)
             if slurpres.is_successful():
                 if slurpres.result['encoding'] == 'base64':
                     remote_data = base64.b64decode(slurpres.result['content'])
                 if remote_data is not None:
-                    remote_md5 = utils.md5s(remote_data)
+                    remote_checksum = utils.checksum_s(remote_data)
                 # the source path may have been expanded on the
                 # target system, so we compare it here and use the
                 # expanded version if it's different
@@ -101,23 +115,23 @@ class ActionModule(object):
 
         # these don't fail because you may want to transfer a log file that possibly MAY exist
         # but keep going to fetch other log files
-        if remote_md5 == '0':
+        if remote_checksum == '0':
             result = dict(msg="unable to calculate the md5 sum of the remote file", file=source, changed=False)
             return ReturnData(conn=conn, result=result)
-        if remote_md5 == '1':
+        if remote_checksum == '1':
             if fail_on_missing:
                 result = dict(failed=True, msg="the remote file does not exist", file=source)
             else:
                 result = dict(msg="the remote file does not exist, not transferring, ignored", file=source, changed=False)
             return ReturnData(conn=conn, result=result)
-        if remote_md5 == '2':
+        if remote_checksum == '2':
             result = dict(msg="no read permission on remote file, not transferring, ignored", file=source, changed=False)
             return ReturnData(conn=conn, result=result)
 
-        # calculate md5 sum for the local file
-        local_md5 = utils.md5(dest)
+        # calculate checksum for the local file
+        local_checksum = utils.checksum(dest)
 
-        if remote_md5 != local_md5:
+        if remote_checksum != local_checksum:
             # create the containing directories, if needed
             if not os.path.isdir(os.path.dirname(dest)):
                 os.makedirs(os.path.dirname(dest))
@@ -129,13 +143,27 @@ class ActionModule(object):
                 f = open(dest, 'w')
                 f.write(remote_data)
                 f.close()
-            new_md5 = utils.md5(dest)
-            if validate_md5 and new_md5 != remote_md5:
-                result = dict(failed=True, md5sum=new_md5, msg="md5 mismatch", file=source, dest=dest, remote_md5sum=remote_md5)
+            new_checksum = utils.secure_hash(dest)
+            # For backwards compatibility.  We'll return None on FIPS enabled
+            # systems
+            try:
+                new_md5 = utils.md5(dest)
+            except ValueError:
+                new_md5 = None
+
+            if validate_checksum and new_checksum != remote_checksum:
+                result = dict(failed=True, md5sum=new_md5, msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum)
                 return ReturnData(conn=conn, result=result)
-            result = dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=remote_md5)
+            result = dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum)
             return ReturnData(conn=conn, result=result)
         else:
-            result = dict(changed=False, md5sum=local_md5, file=source, dest=dest)
+            # For backwards compatibility.  We'll return None on FIPS enabled
+            # systems
+            try:
+                local_md5 = utils.md5(dest)
+            except ValueError:
+                local_md5 = None
+
+            result = dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum)
             return ReturnData(conn=conn, result=result)
 

lib/ansible/runner/action_plugins/template.py

@@ -87,10 +87,10 @@ class ActionModule(object):
             result = dict(failed=True, msg=type(e).__name__ + ": " + str(e))
             return ReturnData(conn=conn, comm_ok=False, result=result)
 
-        local_md5 = utils.md5s(resultant)
-        remote_md5 = self.runner._remote_md5(conn, tmp, dest)
+        local_checksum = utils.checksum_s(resultant)
+        remote_checksum = self.runner._remote_checksum(conn, tmp, dest)
 
-        if local_md5 != remote_md5:
+        if local_checksum != remote_checksum:
 
             # template is different from the remote value
 

lib/ansible/runner/action_plugins/unarchive.py

@@ -62,8 +62,8 @@ class ActionModule(object):
             else:
                 source = utils.path_dwim(self.runner.basedir, source)
 
-        remote_md5 = self.runner._remote_md5(conn, tmp, dest)
-        if remote_md5 != '3':
+        remote_checksum = self.runner._remote_checksum(conn, tmp, dest)
+        if remote_checksum != '3':
             result = dict(failed=True, msg="dest '%s' must be an existing dir" % dest)
             return ReturnData(conn=conn, result=result)
 

lib/ansible/runner/filter_plugins/core.py

@@ -26,7 +26,7 @@ import re
 import collections
 import operator as py_operator
 from ansible import errors
-from ansible.utils import md5s
+from ansible.utils import md5s, checksum_s
 from distutils.version import LooseVersion, StrictVersion
 from random import SystemRandom
 from jinja2.filters import environmentfilter
@@ -281,8 +281,13 @@ class FilterModule(object):
             # quote string for shell usage
             'quote': quote,
 
+            # hash filters
             # md5 hex digest of string
             'md5': md5s,
+            # sha1 hex digeset of string
+            'sha1': checksum_s,
+            # checksum of string as used by ansible for checksuming files
+            'checksum': checksum_s,
 
             # file glob
             'fileglob': fileglob,

lib/ansible/runner/shell_plugins/sh.py

@@ -59,23 +59,17 @@ class ShellModule(object):
         cmd += ' && echo %s' % basetmp
         return cmd
 
-    def md5(self, path):
+    def checksum(self, path, python_interp):
         path = pipes.quote(path)
         # The following test needs to be SH-compliant.  BASH-isms will
         # not work if /bin/sh points to a non-BASH shell.
         test = "rc=0; [ -r \"%s\" ] || rc=2; [ -f \"%s\" ] || rc=1; [ -d \"%s\" ] && echo 3 && exit 0" % ((path,) * 3)
-        md5s = [
-            "(/usr/bin/md5sum %s 2>/dev/null)" % path,          # Linux
-            "(/sbin/md5sum -q %s 2>/dev/null)" % path,          # ?
-            "(/usr/bin/digest -a md5 %s 2>/dev/null)" % path,   # Solaris 10+
-            "(/sbin/md5 -q %s 2>/dev/null)" % path,             # Freebsd
-            "(/usr/bin/md5 -n %s 2>/dev/null)" % path,          # Netbsd
-            "(/bin/md5 -q %s 2>/dev/null)" % path,              # Openbsd
-            "(/usr/bin/csum -h MD5 %s 2>/dev/null)" % path,     # AIX
-            "(/bin/csum -h MD5 %s 2>/dev/null)" % path          # AIX also
+        csums = [
+            "(%s -c 'import hashlib; print(hashlib.sha1(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path),      # Python > 2.4 (including python3)
+            "(%s -c 'import sha; print(sha.sha(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path),        # Python == 2.4
         ]
 
-        cmd = " || ".join(md5s)
+        cmd = " || ".join(csums)
         cmd = "%s; %s || (echo \"${rc}  %s\")" % (test, cmd, path)
         return cmd
 

lib/ansible/utils/__init__.py

@@ -68,6 +68,14 @@ try:
 except ImportError:
     import simplejson as json
 
+# Note, sha1 is the only hash algorithm compatible with python2.4 and with
+# FIPS-140 mode (as of 11-2014)
+try:
+    from hashlib import sha1 as sha1
+except ImportError:
+    from sha import sha as sha1
+
+# Backwards compat only
 try:
     from hashlib import md5 as _md5
 except ImportError:
@@ -821,22 +829,22 @@ def merge_hash(a, b):
 
     return result
 
-def md5s(data):
-    ''' Return MD5 hex digest of data. '''
+def secure_hash_s(data, hash_func=sha1):
+    ''' Return a secure hash hex digest of data. '''
 
-    digest = _md5()
+    digest = hash_func()
     try:
         digest.update(data)
     except UnicodeEncodeError:
         digest.update(data.encode('utf-8'))
     return digest.hexdigest()
 
-def md5(filename):
-    ''' Return MD5 hex digest of local file, None if file is not present or a directory. '''
+def secure_hash(filename, hash_func=sha1):
+    ''' Return a secure hash hex digest of local file, None if file is not present or a directory. '''
 
     if not os.path.exists(filename) or os.path.isdir(filename):
         return None
-    digest = _md5()
+    digest = hash_func()
     blocksize = 64 * 1024
     try:
         infile = open(filename, 'rb')
@@ -849,6 +857,19 @@ def md5(filename):
         raise errors.AnsibleError("error while accessing the file %s, error was: %s" % (filename, e))
     return digest.hexdigest()
 
+# The checksum algorithm must match with the algorithm in ShellModule.checksum() method
+checksum = secure_hash
+checksum_s = secure_hash_s
+
+# Backwards compat.  Some modules include md5s in their return values
+# Continue to support that for now.  As of ansible-1.8, all of those modules
+# should also return "checksum" (sha1 for now)
+def md5s(data):
+    return secure_hash_s(data, _md5)
+
+def md5(filename):
+    return secure_hash(filename, _md5)
+
 def default(value, function):
     ''' syntactic sugar around lazy evaluation of defaults '''
     if value is None:

lib/ansible/utils/vault.py

@@ -26,6 +26,8 @@ from io import BytesIO
 from subprocess import call
 from ansible import errors
 from hashlib import sha256
+# Note: Only used for loading obsolete VaultAES files.  All files are written
+# using the newer VaultAES256 which does not require md5
 from hashlib import md5
 from binascii import hexlify
 from binascii import unhexlify

test/integration/roles/test_assemble/tasks/main.yml

@@ -37,7 +37,19 @@
   assert:
     that:
     - "result.state == 'file'"
-    - "result.md5sum == '96905702a2ece40de6bf3a94b5062513'"
+    - "result.changed == True"
+    - "result.checksum == '048a1bd1951aa5ccc427eeb4ca19aee45e9c68b3'"
+
+- name: test assemble with all fragments
+  assemble: src="{{output_dir}}/src" dest="{{output_dir}}/assembled1"
+  register: result
+
+- name: assert that the same assemble made no changes
+  assert:
+    that:
+    - "result.state == 'file'"
+    - "result.changed == False"
+    - "result.checksum == '048a1bd1951aa5ccc427eeb4ca19aee45e9c68b3'"
 
 - name: test assemble with fragments matching a regex
   assemble: src="{{output_dir}}/src" dest="{{output_dir}}/assembled2" regexp="^fragment[1-3]$"
@@ -47,7 +59,7 @@
   assert:
     that:
     - "result.state == 'file'"
-    - "result.md5sum == 'eb9e3486a9cd6943b5242e573b9b9349'"
+    - "result.checksum == 'edfe2d7487ef8f5ebc0f1c4dc57ba7b70a7b8e2b'"
 
 - name: test assemble with a delimiter
   assemble: src="{{output_dir}}/src" dest="{{output_dir}}/assembled3" delimiter="#--- delimiter ---#"
@@ -57,7 +69,7 @@
   assert:
     that:
     - "result.state == 'file'"
-    - "result.md5sum == '4773eac67aba3f0be745876331c8a450'"
+    - "result.checksum == '505359f48c65b3904127cf62b912991d4da7ed6d'"
 
 - name: test assemble with remote_src=False
   assemble: src="./" dest="{{output_dir}}/assembled4" remote_src=no
@@ -67,7 +79,7 @@
   assert:
     that:
     - "result.state == 'file'"
-    - "result.md5sum == '96905702a2ece40de6bf3a94b5062513'"
+    - "result.checksum == '048a1bd1951aa5ccc427eeb4ca19aee45e9c68b3'"
 
 - name: test assemble with remote_src=False and a delimiter
   assemble: src="./" dest="{{output_dir}}/assembled5" remote_src=no delimiter="#--- delimiter ---#"
@@ -77,5 +89,5 @@
   assert:
     that:
     - "result.state == 'file'"
-    - "result.md5sum == '4773eac67aba3f0be745876331c8a450'"
+    - "result.checksum == '505359f48c65b3904127cf62b912991d4da7ed6d'"
 

test/integration/roles/test_command_shell/tasks/main.yml

@@ -185,7 +185,7 @@
     "multiline echo" \
     "with a new line
     in quotes" \
-    | md5sum \
+    | sha1sum \
     | tr -s ' ' \
     | cut -f1 -d ' '
     echo "this is a second line"
@@ -197,7 +197,7 @@
   assert:
     that:
       - "shell_result6.changed"
-      - "shell_result6.stdout == '32f3cc201b69ed8afa3902b80f554ca8\nthis is a second line'"
+      - "shell_result6.stdout == '5575bb6b71c9558db0b6fbbf2f19909eeb4e3b98\nthis is a second line'"
 
 - name: execute a shell command using a literal multiline block with arguments in it
   shell: |

test/integration/roles/test_copy/tasks/main.yml

@@ -40,6 +40,7 @@
       - "'group' in copy_result"
       - "'gid' in copy_result"
       - "'md5sum' in copy_result"
+      - "'checksum' in copy_result"
       - "'owner' in copy_result"
       - "'size' in copy_result"
       - "'src' in copy_result"
@@ -51,10 +52,11 @@
     that:
       - "copy_result.changed == true"
 
-- name: verify that the file md5sum is correct
-  assert: 
-    that: 
+- name: verify that the file checksums are correct
+  assert:
+    that:
       - "copy_result.md5sum == 'c47397529fe81ab62ba3f85e9f4c71f2'"
+      - "copy_result.checksum == 'c79a6506c1c948be0d456ab5104d5e753ab2f3e6'"
 
 - name: check the stat results of the file
   stat: path={{output_file}}
@@ -71,6 +73,7 @@
       - "stat_results.stat.isreg == true"
       - "stat_results.stat.issock == false"
       - "stat_results.stat.md5 == 'c47397529fe81ab62ba3f85e9f4c71f2'"
+      - "stat_results.stat.checksum == 'c79a6506c1c948be0d456ab5104d5e753ab2f3e6'"
 
 - name: overwrite the file via same means
   copy: src=foo.txt dest={{output_file}}
@@ -180,7 +183,7 @@
     that:
       - "copy_result6.changed"
       - "copy_result6.dest == '{{output_dir|expanduser}}/multiline.txt'"
-      - "copy_result6.md5sum == '1627d51e7e607c92cf1a502bf0c6cce3'"
+      - "copy_result6.checksum == '9cd0697c6a9ff6689f0afb9136fa62e0b3fee903'"
 
 # test overwriting a file as an unprivileged user (pull request #8624)
 # this can't be relative to {{output_dir}} as ~root usually has mode 700
@@ -202,7 +205,7 @@
     that:
       - "copy_result7.changed"
       - "copy_result7.dest == '/tmp/worldwritable/file.txt'"
-      - "copy_result7.md5sum == '73feffa4b7f6bb68e44cf984c85f6e88'"
+      - "copy_result7.checksum == 'bbe960a25ea311d21d40669e93df2003ba9b90a2'"
 
 - name: clean up
   file: dest=/tmp/worldwritable state=absent
@@ -230,10 +233,10 @@
     - stat_link_result.stat.islnk
 
 - name: get the md5 of the link target
-  shell: md5sum {{output_dir}}/follow_test | cut -f1 -sd ' '
+  shell: sha1sum {{output_dir}}/follow_test | cut -f1 -sd ' '
   register: target_file_result
 
 - name: assert that the link target was updated
   assert:
     that:
-    - replace_follow_result.md5sum == target_file_result.stdout
+    - replace_follow_result.checksum == target_file_result.stdout

test/integration/roles/test_lineinfile/tasks/main.yml

@@ -24,7 +24,7 @@
   assert:
     that:
     - "result.changed == true"
-    - "result.md5sum == '6be7fb7fa7fb758c80a6dc0722979c40'"
+    - "result.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'"
     - "result.state == 'file'"
 
 - name: insert a line at the beginning of the file, and back it up
@@ -42,19 +42,19 @@
   stat: path={{result.backup}}
   register: result
 
-- name: assert the backup file matches the previous md5
+- name: assert the backup file matches the previous hash
   assert:
     that:
-    - "result.stat.md5 == '6be7fb7fa7fb758c80a6dc0722979c40'"
+    - "result.stat.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'"
 
 - name: stat the test after the insert at the head
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after the insert at the head
+- name: assert test hash is what we expect for the file with the insert at the head
   assert:
     that:
-    - "result.stat.md5 == '07c16434644a2a3cc1807c685917443a'"
+    - "result.stat.checksum == '7eade4042b23b800958fe807b5bfc29f8541ec09'"
 
 - name: insert a line at the end of the file
   lineinfile: dest={{output_dir}}/test.txt state=present line="New line at the end" insertafter="EOF"
@@ -70,10 +70,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after the insert at the end
+- name: assert test checksum matches after the insert at the end
   assert:
     that:
-    - "result.stat.md5 == 'da4c2150e5782fcede1840280ab87eff'"
+    - "result.stat.checksum == 'fb57af7dc10a1006061b000f1f04c38e4bef50a9'"
 
 - name: insert a line after the first line
   lineinfile: dest={{output_dir}}/test.txt state=present line="New line after line 1" insertafter="^This is line 1$"
@@ -89,10 +89,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after the insert after the first line
+- name: assert test checksum matches after the insert after the first line
   assert:
     that:
-    - "result.stat.md5 == '196722c8faaa28b960bee66fa4cce58c'"
+    - "result.stat.checksum == '5348da605b1bc93dbadf3a16474cdf22ef975bec'"
 
 - name: insert a line before the last line
   lineinfile: dest={{output_dir}}/test.txt state=present line="New line after line 5" insertbefore="^This is line 5$"
@@ -108,10 +108,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after the insert before the last line
+- name: assert test checksum matches after the insert before the last line
   assert:
     that:
-    - "result.stat.md5 == 'd5955ee042139dfef16dbe3a7334475f'"
+    - "result.stat.checksum == 'e1cae425403507feea4b55bb30a74decfdd4a23e'"
 
 - name: replace a line with backrefs
   lineinfile: dest={{output_dir}}/test.txt state=present line="This is line 3" backrefs=yes regexp="^(REF) .* \\1$"
@@ -127,16 +127,16 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after backref line was replaced
+- name: assert test checksum matches after backref line was replaced
   assert:
     that:
-    - "result.stat.md5 == '0f585270054e17be242743dd31c6f593'"
+    - "result.stat.checksum == '2ccdf45d20298f9eaece73b713648e5489a52444'"
 
 - name: remove the middle line
   lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 3$"
   register: result
 
-- name: assert that the line was inserted at the head of the file
+- name: assert that the line was removed
   assert:
     that:
     - "result.changed == true"
@@ -146,10 +146,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after the middle line was removed
+- name: assert test checksum matches after the middle line was removed
   assert:
     that:
-    - "result.stat.md5 == '661603660051991b79429c2dc68d9a67'"
+    - "result.stat.checksum == 'a6ba6865547c19d4c203c38a35e728d6d1942c75'"
 
 - name: run a validation script that succeeds
   lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 5$" validate="true %s"
@@ -165,10 +165,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after the validation succeeded
+- name: assert test checksum matches after the validation succeeded
   assert:
     that:
-    - "result.stat.md5 == '9af984939bd859f7794661e501b4f1a4'"
+    - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'"
 
 - name: run a validation script that fails
   lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 1$" validate="/bin/false %s"
@@ -184,10 +184,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches the previous after the validation failed
+- name: assert test checksum matches the previous after the validation failed
   assert:
     that:
-    - "result.stat.md5 == '9af984939bd859f7794661e501b4f1a4'"
+    - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'"
 
 - name: use create=yes
   lineinfile: dest={{output_dir}}/new_test.txt create=yes insertbefore=BOF state=present line="This is a new file"
@@ -204,10 +204,10 @@
   register: result
   ignore_errors: yes
 
-- name: assert the newly created test md5 matches
+- name: assert the newly created test checksum matches
   assert:
     that:
-    - "result.stat.md5 == 'fef1d487711facfd7aa2c87d788c19d9'"
+    - "result.stat.checksum == '038f10f9e31202451b093163e81e06fbac0c6f3a'"
 
 # Test EOF in cases where file has no newline at EOF
 - name: testnoeof deploy the file for lineinfile
@@ -238,10 +238,10 @@
   stat: path={{output_dir}}/testnoeof.txt
   register: result
 
-- name: testnoeof assert test md5 matches after the insert at the end
+- name: testnoeof assert test checksum matches after the insert at the end
   assert:
     that:
-    - "result.stat.md5 == 'f75c9d51f45afd7295000e63ce655220'"
+    - "result.stat.checksum == 'f9af7008e3cb67575ce653d094c79cabebf6e523'"
 
 # Test EOF with empty file to make sure no unneccessary newline is added
 - name: testempty deploy the testempty file for lineinfile
@@ -262,18 +262,18 @@
   stat: path={{output_dir}}/testempty.txt
   register: result
 
-- name: testempty assert test md5 matches after the insert at the end
+- name: testempty assert test checksum matches after the insert at the end
   assert:
     that:
-    - "result.stat.md5 == '357dcbee8dfb4436f63bab00a235c45a'"
+    - "result.stat.checksum == 'f440dc65ea9cec3fd496c1479ddf937e1b949412'"
 
 - stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after insert the multiple lines
+- name: assert test checksum matches after inserting multiple lines
   assert:
     that:
-    - "result.stat.md5 == 'c2510d5bc8fdef8e752b8f8e74c784c2'"
+    - "result.stat.checksum == 'bf5b711f8f0509355aaeb9d0d61e3e82337c1365'"
 
 - name: replace a line with backrefs included in the line
   lineinfile: dest={{output_dir}}/test.txt state=present line="New \\1 created with the backref" backrefs=yes regexp="^This is (line 4)$"
@@ -289,10 +289,10 @@
   stat: path={{output_dir}}/test.txt
   register: result
 
-- name: assert test md5 matches after backref line was replaced
+- name: assert test checksum matches after backref line was replaced
   assert:
     that:
-    - "result.stat.md5 == '65f955c2a9722fd43d07103d7756ff9b'"
+    - "result.stat.checksum == '04b7a54d0fb233a4e26c9e625325bb4874841b3c'"
 
 ###################################################################
 # issue 8535
@@ -332,10 +332,10 @@
   stat: path={{output_dir}}/test_quoting.txt
   register: result
 
-- name: assert test md5 matches after backref line was replaced
+- name: assert test checksum matches after backref line was replaced
   assert:
     that:
-    - "result.stat.md5 == '29f349baf1b9c6703beeb346fe8dc669'"
+    - "result.stat.checksum == '7dc3cb033c3971e73af0eaed6623d4e71e5743f1'"
 
 - name: insert a line into the quoted file with a single quote
   lineinfile: dest={{output_dir}}/test_quoting.txt line="import g'"
@@ -350,9 +350,9 @@
   stat: path={{output_dir}}/test_quoting.txt
   register: result
 
-- name: assert test md5 matches after backref line was replaced
+- name: assert test checksum matches after backref line was replaced
   assert:
     that:
-    - "result.stat.md5 == 'fbe9c4ba2490f70eb1974ce31ec4a39f'"
+    - "result.stat.checksum == '73b271c2cc1cef5663713bc0f00444b4bf9f4543'"
 
 ###################################################################

test/integration/roles/test_service/tasks/main.yml

@@ -6,7 +6,7 @@
   assert:
     that:
     - "install_result.dest == '/usr/sbin/ansible_test_service'"
-    - "install_result.md5sum == '9ad49eaf390b30b1206b793ec71200ed'"
+    - "install_result.checksum == 'baaa79448a976922c080f1971321d203c6df0961'"
     - "install_result.state == 'file'"
     - "install_result.mode == '0755'"
 

test/integration/roles/test_service/tasks/systemd_setup.yml

@@ -12,7 +12,7 @@
     - "install_systemd_result.dest == '/usr/lib/systemd/system/ansible_test.service'"
     - "install_systemd_result.state == 'file'"
     - "install_systemd_result.mode == '0644'"
-    - "install_systemd_result.md5sum == '6be64a1e44e9e72a467e70a0b562444f'"
+    - "install_systemd_result.checksum == 'ca4b413fdf3cb2002f51893b9e42d2e449ec5afb'"
     - "install_broken_systemd_result.dest == '/usr/lib/systemd/system/ansible_test_broken.service'"
     - "install_broken_systemd_result.state == 'link'"
 

test/integration/roles/test_service/tasks/sysv_setup.yml

@@ -8,5 +8,5 @@
     - "install_sysv_result.dest == '/etc/init.d/ansible_test'"
     - "install_sysv_result.state == 'file'"
     - "install_sysv_result.mode == '0755'"
-    - "install_sysv_result.md5sum == 'ebf6a9064ca8628187f3a6caf8e2a279'"
+    - "install_sysv_result.md5sum == '174fa255735064b420600e4c8637ea0eff28d0c1'"
 

test/integration/roles/test_service/tasks/upstart_setup.yml

@@ -12,8 +12,8 @@
     - "install_upstart_result.dest == '/etc/init/ansible_test.conf'"
     - "install_upstart_result.state == 'file'"
     - "install_upstart_result.mode == '0644'"
-    - "install_upstart_result.md5sum == 'ab3900ea4de8423add764c12aeb90c01'"
+    - "install_upstart_result.checksum == '5c314837b6c4dd6c68d1809653a2974e9078e02a'"
     - "install_upstart_broken_result.dest == '/etc/init/ansible_broken_test.conf'"
     - "install_upstart_broken_result.state == 'file'"
     - "install_upstart_broken_result.mode == '0644'"
-    - "install_upstart_broken_result.md5sum == '015e183d10c311276c3e269cbeb309b7'"
+    - "install_upstart_broken_result.checksum == 'e66497894f2b2bf71e1380a196cc26089cc24a10'"

test/integration/roles/test_stat/tasks/main.yml

@@ -46,6 +46,8 @@
         - "'isuid' in stat_result.stat" 
         - "'md5' in stat_result.stat"
         - "stat_result.stat.md5 == '5eb63bbbe01eeed093cb22bb8f5acdc3'" 
+        - "'checksum' in stat_result.stat"
+        - "stat_result.stat.checksum == '2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'"
         - "'mode' in stat_result.stat" # why is this 420?
         - "'mtime' in stat_result.stat" 
         - "'nlink' in stat_result.stat" 

test/integration/roles/test_template/tasks/main.yml

@@ -27,6 +27,7 @@
         - "'group' in template_result" 
         - "'gid' in template_result" 
         - "'md5sum' in template_result" 
+        - "'checksum' in template_result"
         - "'owner' in template_result" 
         - "'size' in template_result" 
         - "'src' in template_result" 

test/units/TestModuleUtilsBasic.py

@@ -7,7 +7,7 @@ from nose.tools import timed
 
 from ansible import errors
 from ansible.module_common import ModuleReplacer
-from ansible.utils import md5 as utils_md5
+from ansible.utils import checksum as utils_checksum
 
 TEST_MODULE_DATA = """
 from ansible.module_utils.basic import *
@@ -113,8 +113,8 @@ class TestModuleUtilsBasic(unittest.TestCase):
             (rc, out, err) = self.module.run_command('echo "foo bar" > %s' % tmp_path, use_unsafe_shell=True)
             self.assertEqual(rc, 0)
             self.assertTrue(os.path.exists(tmp_path))
-            md5sum = utils_md5(tmp_path)
-            self.assertEqual(md5sum, '5ceaa7ed396ccb8e959c02753cb4bd18')
+            checksum = utils_checksum(tmp_path)
+            self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec')
         except:
             raise
         finally:
@@ -127,8 +127,8 @@ class TestModuleUtilsBasic(unittest.TestCase):
             (rc, out, err) = self.module.run_command('echo "foo bar" >> %s' % tmp_path, use_unsafe_shell=True)
             self.assertEqual(rc, 0)
             self.assertTrue(os.path.exists(tmp_path))
-            md5sum = utils_md5(tmp_path)
-            self.assertEqual(md5sum, '5ceaa7ed396ccb8e959c02753cb4bd18')
+            checksum = utils_checksum(tmp_path)
+            self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec')
         except:
             raise
         finally:

test/units/TestUtils.py

@@ -366,6 +366,16 @@ class TestUtils(unittest.TestCase):
         self.assertEqual(ansible.utils.md5(os.path.join(os.path.dirname(__file__), 'ansible.cf')),
                          None)
 
+    def test_checksum_s(self):
+        self.assertEqual(ansible.utils.checksum_s('ansible'), 'bef45157a43c9e5f469d188810814a4a8ab9f2ed')
+        # Need a test that causes UnicodeEncodeError See 4221
+
+    def test_checksum(self):
+        self.assertEqual(ansible.utils.checksum(os.path.join(os.path.dirname(__file__), 'ansible.cfg')),
+                         '658b67c8ac7595adde7048425ff1f9aba270721a')
+        self.assertEqual(ansible.utils.md5(os.path.join(os.path.dirname(__file__), 'ansible.cf')),
+                         None)
+
     def test_default(self):
         self.assertEqual(ansible.utils.default(None, lambda: {}), {})
         self.assertEqual(ansible.utils.default(dict(foo='bar'), lambda: {}), dict(foo='bar'))

v2/ansible/parsing/vault/__init__.py

@@ -30,6 +30,8 @@ from io import BytesIO
 from subprocess import call
 from ansible import errors
 from hashlib import sha256
+# Note: Only used for loading obsolete VaultAES files.  All files are written
+# using the newer VaultAES256 which does not require md5
 from hashlib import md5
 from binascii import hexlify
 from binascii import unhexlify

v2/ansible/playbook/role/__init__.py

@@ -23,7 +23,7 @@ from six import iteritems, string_types
 
 import os
 
-from hashlib import md5
+from hashlib import sha1
 from types import NoneType
 
 from ansible.errors import AnsibleError, AnsibleParserError
@@ -39,7 +39,7 @@ __all__ = ['Role', 'ROLE_CACHE']
 
 
 # The role cache is used to prevent re-loading roles, which
-# may already exist. Keys into this cache are the MD5 hash
+# may already exist. Keys into this cache are the SHA1 hash
 # of the role definition (for dictionary definitions, this
 # will be based on the repr() of the dictionary object)
 ROLE_CACHE = dict()
@@ -60,7 +60,7 @@ class Role:
         self._handler_blocks   = []
         self._default_vars     = dict()
         self._role_vars        = dict()
-        
+
     def __repr__(self):
         return self.get_name()