Download logs from the playbook

This broke when we stopped using /tmp to store logs etc. Pushed the logic into ansible.

Change-Id: Ib32c236159bbb6654af9c69e98b5540050a3df07

edx/analytics/tasks/launchers/remote.py

@@ -25,8 +25,6 @@ def main():
     uid = arguments.remote_name or str(uuid.uuid4())
 
     return_code = run_task_playbook(arguments, uid)
-    if arguments.log_path:
-        download_logs(arguments, uid)
 
     sys.exit(return_code)
 
@@ -65,6 +63,8 @@ def convert_args_to_extra_vars(arguments, uid):
         extra_vars['repo'] = arguments.repo
     if arguments.wait:
         extra_vars['wait_for_task'] = True
+    if arguments.log_path:
+        extra_vars['local_log_dir'] = arguments.log_path
     return ' '.join(["{}='{}'".format(k, extra_vars[k]) for k in extra_vars])
 
 
@@ -102,30 +102,3 @@ def run_ansible(args, verbose, executable='ansible'):
         proc.wait()
 
     return proc.returncode
-
-
-def download_logs(arguments, uid):
-    """
-    Connect to the remote machine and download the logs produced by luigi.
-
-    Args:
-        arguments (argparse.Namespace): The arguments that were passed in on the command line.
-        uid (str): A unique identifier for this task execution.
-    """
-    for extension in ['out', 'err']:
-        args = [
-            'mr_{job_flow}_master'.format(job_flow=arguments.job_flow_id),
-            '-m', 'fetch',
-            '-a', 'src=/tmp/{uid}.{ext} dest={dest} flat=yes'.format(
-                uid=uid,
-                ext=extension,
-                dest=arguments.log_path
-            )
-        ]
-        if arguments.user:
-            args.extend(['-u', arguments.user])
-
-        run_ansible(
-            tuple(args),
-            arguments.verbose
-        )

share/task.yml

@@ -25,6 +25,7 @@
     - git_server_ip_address: 207.97.227.239
     - git_server_public_key: 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=='
     - wait_for_task: False
+    - local_log_dir: build/logs
 
   tasks:
     - name: known_hosts file exists
@@ -113,3 +114,10 @@
       sudo: True
       sudo_user: hadoop
       when: wait_for_task
+
+    - name: fetch logs
+      fetch: src={{ log_dir }}/{{ item }} dest={{ local_log_dir }} flat=yes
+      with_items:
+        - stdout
+        - stderr
+        - edx_analytics.log