Add integration tests for fetch/slurp, make powershell fetch/slurp work as close…

Add integration tests for fetch/slurp, make powershell fetch/slurp work as close as possible to existing fetch/slurp modules.

lib/ansible/module_utils/powershell.ps1

@@ -62,5 +62,5 @@ Function Fail-Json($obj, $message)
     Set-Attr $obj "msg" $message
     Set-Attr $obj "failed" $true
     echo $obj | ConvertTo-Json
-    Exit
+    Exit 1
 }

lib/ansible/runner/action_plugins/fetch.py

@@ -57,19 +57,24 @@ class ActionModule(object):
             return ReturnData(conn=conn, result=results)
 
         source = os.path.expanduser(source)
+        source = conn.shell.join_path(source)
+        if os.path.sep not in conn.shell.join_path('a', ''):
+            source_local = source.replace('\\', '/')
+        else:
+            source_local = source
 
         if flat:
-            if dest.endswith("/"): # CCTODO: Fix path for Windows hosts.
+            if dest.endswith("/"):
                 # if the path ends with "/", we'll use the source filename as the
                 # destination filename
-                base = os.path.basename(source)
+                base = os.path.basename(source_local)
                 dest = os.path.join(dest, base)
             if not dest.startswith("/"):
                 # if dest does not start with "/", we'll assume a relative path
                 dest = utils.path_dwim(self.runner.basedir, dest)
         else:
             # files are saved in dest dir, with a subdir for each host, then the filename
-            dest = "%s/%s/%s" % (utils.path_dwim(self.runner.basedir, dest), conn.host, source)
+            dest = "%s/%s/%s" % (utils.path_dwim(self.runner.basedir, dest), conn.host, source_local)
 
         dest = os.path.expanduser(dest.replace("//","/"))
 

lib/ansible/runner/connection_plugins/winrm.py

@@ -178,8 +178,8 @@ class Connection(object):
                     cmd_parts = powershell._encode_script(script, as_list=True)
                     result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
                     if result.status_code != 0:
-                        raise RuntimeError(result.std_err.encode('utf-8'))
-                except Exception: # IOError?
+                        raise IOError(result.std_err.encode('utf-8'))
+                except Exception:
                     traceback.print_exc()
                     raise errors.AnsibleError("failed to transfer file to %s" % out_path)
 
@@ -189,31 +189,61 @@ class Connection(object):
         buffer_size = 2**20 # 1MB chunks
         if not os.path.exists(os.path.dirname(out_path)):
             os.makedirs(os.path.dirname(out_path))
-        with open(out_path, 'wb') as out_file:
+        out_file = None
+        try:
             offset = 0
             while True:
                 try:
                     script = '''
-                        $bufferSize = %d;
-                        $stream = [System.IO.File]::OpenRead("%s");
-                        $stream.Seek(%d, [System.IO.SeekOrigin]::Begin) | Out-Null;
-                        $buffer = New-Object Byte[] $bufferSize;
-                        $bytesRead = $stream.Read($buffer, 0, $bufferSize);
-                        $bytes = $buffer[0..($bytesRead-1)];
-                        [System.Convert]::ToBase64String($bytes);
-                        $stream.Close() | Out-Null;
-                    ''' % (buffer_size, powershell._escape(in_path), offset)
+                        If (Test-Path -PathType Leaf "%(path)s")
+                        {
+                            $stream = [System.IO.File]::OpenRead("%(path)s");
+                            $stream.Seek(%(offset)d, [System.IO.SeekOrigin]::Begin) | Out-Null;
+                            $buffer = New-Object Byte[] %(buffer_size)d;
+                            $bytesRead = $stream.Read($buffer, 0, %(buffer_size)d);
+                            $bytes = $buffer[0..($bytesRead-1)];
+                            [System.Convert]::ToBase64String($bytes);
+                            $stream.Close() | Out-Null;
+                        }
+                        ElseIf (Test-Path -PathType Container "%(path)s")
+                        {
+                            Write-Host "[DIR]";
+                        }
+                        Else
+                        {
+                            Write-Error "%(path)s does not exist";
+                            Exit 1;
+                        }
+                    ''' % dict(buffer_size=buffer_size, path=powershell._escape(in_path), offset=offset)
                     vvvv("WINRM FETCH %s to %s (offset=%d)" % (in_path, out_path, offset), host=self.host)
                     cmd_parts = powershell._encode_script(script, as_list=True)
                     result = self._winrm_exec(cmd_parts[0], cmd_parts[1:])
-                    data = base64.b64decode(result.std_out.strip())
-                    out_file.write(data)
-                    if len(data) < buffer_size:
+                    if result.status_code != 0:
+                        raise IOError(result.std_err.encode('utf-8'))
+                    if result.std_out.strip() == '[DIR]':
+                        data = None
+                    else:
+                        data = base64.b64decode(result.std_out.strip())
+                    if data is None:
+                        if not os.path.exists(out_path):
+                            os.makedirs(out_path)
                         break
-                    offset += len(data)
-                except Exception: # IOError?
+                    else:
+                        if not out_file:
+                            # If out_path is a directory and we're expecting a file, bail out now.
+                            if os.path.isdir(out_path):
+                                break
+                            out_file = open(out_path, 'wb')
+                        out_file.write(data)
+                        if len(data) < buffer_size:
+                            break
+                        offset += len(data)
+                except Exception:
                     traceback.print_exc()
                     raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+        finally:
+            if out_file:
+                out_file.close()
 
     def close(self):
         if self.protocol and self.shell_id:

lib/ansible/runner/shell_plugins/powershell.py

@@ -85,7 +85,21 @@ class ShellModule(object):
 
     def md5(self, path):
         path = _escape(path)
-        return _encode_script('''(Get-FileHash -Path "%s" -Algorithm MD5).Hash.ToLower();''' % path)
+        script = '''
+            If (Test-Path -PathType Leaf "%(path)s")
+            {
+                (Get-FileHash -Path "%(path)s" -Algorithm MD5).Hash.ToLower();
+            }
+            ElseIf (Test-Path -PathType Container "%(path)s")
+            {
+                Write-Host "3";
+            }
+            Else
+            {
+                Write-Host "1";
+            }
+        ''' % dict(path=path)
+        return _encode_script(script)
 
     def build_module_command(self, env_string, shebang, cmd, rm_tmp=None):
         cmd_parts = shlex.split(cmd, posix=False)

lib/ansible/utils/__init__.py

@@ -608,9 +608,9 @@ def md5s(data):
     return digest.hexdigest()
 
 def md5(filename):
-    ''' Return MD5 hex digest of local file, or None if file is not present. '''
+    ''' Return MD5 hex digest of local file, None if file is not present or a directory. '''
 
-    if not os.path.exists(filename):
+    if not os.path.exists(filename) or os.path.isdir(filename):
         return None
     digest = _md5()
     blocksize = 64 * 1024

library/windows/slurp.ps1

@@ -22,26 +22,43 @@ $params = Parse-Args $args;
 $src = '';
 If ($params.src.GetType)
 {
-   $src = $params.src;
+    $src = $params.src;
 }
 Else
 {
-   If ($params.path.GetType)
-   {
-      $src = $params.path;
-   }
+    If ($params.path.GetType)
+    {
+        $src = $params.path;
+    }
 }
 If (-not $src)
 {
-
+    $result = New-Object psobject @{};
+    Fail-Json $result "missing required argument: src";
 }
 
-$bytes = [System.IO.File]::ReadAllBytes($src);
-$content = [System.Convert]::ToBase64String($bytes);
+If (Test-Path $src)
+{
+    If ((Get-Item $src).Directory) # Only files have the .Directory attribute.
+    {
+        $bytes = [System.IO.File]::ReadAllBytes($src);
+        $content = [System.Convert]::ToBase64String($bytes);
 
-$result = New-Object psobject @{
-    changed = $false
-    encoding = "base64"
-};
-Set-Attr $result "content" $content;
-Exit-Json $result;
+        $result = New-Object psobject @{
+            changed = $false
+            encoding = "base64"
+        };
+        Set-Attr $result "content" $content;
+        Exit-Json $result;
+    }
+    Else
+    {
+        $result = New-Object psobject @{};
+        Fail-Json $result ("is a directory: " + $src);
+    }
+}
+Else
+{
+    $result = New-Object psobject @{};
+    Fail-Json $result ("file not found: " + $src);
+}

test/integration/roles/test_win_fetch/tasks/main.yml

+# test code for the fetch module when using winrm connection
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+- name: clean out the test directory
+  local_action: file name={{ output_dir|mandatory }} state=absent
+  tags: me
+
+- name: create the test directory
+  local_action: file name={{ output_dir }} state=directory
+  tags: me
+
+- name: fetch a small file
+  fetch: src="C:/Windows/win.ini" dest={{ output_dir }}
+  register: fetch_small
+
+- name: check fetch small result
+  assert:
+    that:
+      - "fetch_small.changed"
+
+- name: check file created by fetch small
+  local_action: stat path={{ fetch_small.dest }}
+  register: fetch_small_stat
+
+- name: verify fetched small file exists locally
+  assert:
+    that:
+      - "fetch_small_stat.stat.exists"
+      - "fetch_small_stat.stat.isreg"
+      - "fetch_small_stat.stat.md5 == fetch_small.md5sum"
+
+- name: fetch the same small file
+  fetch: src="C:/Windows/win.ini" dest={{ output_dir }}
+  register: fetch_small_again
+
+- name: check fetch small result again
+  assert:
+    that:
+      - "not fetch_small_again.changed"
+
+- name: fetch a small file to flat namespace
+  fetch: src="C:/Windows/win.ini" dest="{{ output_dir }}/" flat=yes
+  register: fetch_flat
+
+- name: check fetch flat result
+  assert:
+    that:
+      - "fetch_flat.changed"
+
+- name: check file created by fetch flat
+  local_action: stat path="{{ output_dir }}/win.ini"
+  register: fetch_flat_stat
+
+- name: verify fetched file exists locally in output_dir
+  assert:
+    that:
+      - "fetch_flat_stat.stat.exists"
+      - "fetch_flat_stat.stat.isreg"
+      - "fetch_flat_stat.stat.md5 == fetch_flat.md5sum"
+
+- name: fetch a small file to flat directory (without trailing slash)
+  fetch: src="C:/Windows/win.ini" dest="{{ output_dir }}" flat=yes
+  register: fetch_flat_dir
+  ignore_errors: true
+
+- name: check fetch flat to directory result
+  assert:
+    that:
+      - "fetch_flat_dir|failed"
+      - "fetch_flat_dir.msg"
+
+- name: fetch a large binary file
+  fetch: src="C:/Windows/explorer.exe" dest={{ output_dir }}
+  register: fetch_large
+
+- name: check fetch large binary file result
+  assert:
+    that:
+      - "fetch_large.changed"
+
+- name: check file created by fetch large binary
+  local_action: stat path={{ fetch_large.dest }}
+  register: fetch_large_stat
+
+- name: verify fetched large file exists locally
+  assert:
+    that:
+      - "fetch_large_stat.stat.exists"
+      - "fetch_large_stat.stat.isreg"
+      - "fetch_large_stat.stat.md5 == fetch_large.md5sum"
+
+- name: fetch a large binary file again
+  fetch: src="C:/Windows/explorer.exe" dest={{ output_dir }}
+  register: fetch_large_again
+
+- name: check fetch large binary file result again
+  assert:
+    that:
+      - "not fetch_large_again.changed"
+
+- name: fetch a small file using backslashes in src path
+  fetch: src="C:\Windows\system.ini" dest={{ output_dir }}
+  register: fetch_small_bs
+
+- name: check fetch small result with backslashes
+  assert:
+    that:
+      - "fetch_small_bs.changed"
+
+- name: check file created by fetch small with backslashes
+  local_action: stat path={{ fetch_small_bs.dest }}
+  register: fetch_small_bs_stat
+
+- name: verify fetched small file with backslashes exists locally
+  assert:
+    that:
+      - "fetch_small_bs_stat.stat.exists"
+      - "fetch_small_bs_stat.stat.isreg"
+      - "fetch_small_bs_stat.stat.md5 == fetch_small_bs.md5sum"
+
+- name: attempt to fetch a non-existent file - do not fail on missing
+  fetch: src="C:/this_file_should_not_exist.txt" dest={{ output_dir }}
+  register: fetch_missing_nofail
+
+- name: check fetch missing no fail result
+  assert:
+    that:
+      - "not fetch_missing_nofail|failed"
+      - "fetch_missing_nofail.msg"
+      - "not fetch_missing_nofail|changed"
+
+- name: attempt to fetch a non-existent file - fail on missing
+  fetch: src="C:/this_file_should_not_exist.txt" dest={{ output_dir }} fail_on_missing=yes
+  register: fetch_missing
+  ignore_errors: true
+
+- name: check fetch missing with failure
+  assert:
+    that:
+      - "fetch_missing|failed"
+      - "fetch_missing.msg"
+      - "not fetch_missing|changed"
+
+- name: attempt to fetch a directory
+  fetch: src="C:\Windows" dest={{ output_dir }}
+  register: fetch_dir
+  ignore_errors: true
+
+- name: check fetch directory result
+  assert:
+    that:
+      - "fetch_dir|failed"
+      - "fetch_dir.msg"

test/integration/roles/test_win_ping/tasks/main.yml

----
+# test code for the win_ping module
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 - name: test win_ping
   action: win_ping

test/integration/roles/test_win_raw/tasks/main.yml

----
+# test code for the raw module when using winrm connection
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 - name: run getmac
   raw: getmac

test/integration/roles/test_win_script/files/test_script_with_errors.ps1

-# http://stackoverflow.com/questions/9948517/how-to-stop-a-powershell-script-on-the-first-error
-#$ErrorActionPreference = "Stop";
-# http://stackoverflow.com/questions/15777492/why-are-my-powershell-exit-codes-always-0
+# Test script to make sure we handle non-zero exit codes.
 
 trap
 {

test/integration/roles/test_win_script/tasks/main.yml

----
+# test code for the script module when using winrm connection
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 - name: run simple test script
   script: test_script.ps1

test/integration/roles/test_win_slurp/tasks/main.yml

+# test code for the slurp module when using winrm connection
+# (c) 2014, Chris Church <chris@ninemoreminutes.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+- name: test slurping an existing file
+  slurp: src="C:/Windows/win.ini"
+  register: slurp_existing
+
+- name: check slurp existing result
+  assert:
+    that:
+      - "slurp_existing.content"
+      - "slurp_existing.encoding == 'base64'"
+      - "not slurp_existing|changed"
+      - "not slurp_existing|failed"
+
+- name: test slurping a large binary file with path param and backslashes
+  slurp: path="C:\Windows\explorer.exe"
+  register: slurp_path_backslashes
+
+- name: check slurp result with path param and backslashes
+  assert:
+    that:
+      - "slurp_path_backslashes.content"
+      - "slurp_path_backslashes.encoding == 'base64'"
+      - "not slurp_path_backslashes|changed"
+      - "not slurp_path_backslashes|failed"
+
+- name: test slurping a non-existent file
+  slurp: src="C:/this_file_should_not_exist.txt"
+  register: slurp_missing
+  ignore_errors: true
+
+- name: check slurp missing result
+  assert:
+    that:
+      - "slurp_missing|failed"
+      - "slurp_missing.msg"
+      - "not slurp_missing|changed"
+
+- name: test slurping a directory
+  slurp: src="C:/Windows"
+  register: slurp_dir
+  ignore_errors: true
+
+- name: check slurp directory result
+  assert:
+    that:
+      - "slurp_dir|failed"
+      - "slurp_dir.msg"
+      - "not slurp_dir|changed"
+
+- name: test slurp with missing argument
+  action: slurp
+  register: slurp_no_args
+  ignore_errors: true
+
+- name: check slurp with missing argument result
+  assert:
+    that:
+      - "slurp_no_args|failed"
+      - "slurp_no_args.msg"
+      - "not slurp_no_args|changed"

test/integration/test_winrm.yml

@@ -6,3 +6,5 @@
     - { role: test_win_raw, tags: test_win_raw }
     - { role: test_win_script, tags: test_win_script }
     - { role: test_win_ping, tags: test_win_ping }
+    - { role: test_win_slurp, tags: test_win_slurp }
+    - { role: test_win_fetch, tags: test_win_fetch }