run tasks using ansible

edx/analytics/tasks/remote.py

+"""Remote Task
+
+Usage: 
+  remote-task [--job-flow-id=ID] [--branch=BRANCH] [--repo=REPO] [--name=NAME] [--verbose] [<launch-task-args>]
+
+Options:
+  --job-flow-id=ID      EMR job flow to run the task
+  --branch=BRANCH       git branch to checkout before running the task [default: release]
+  --repo=REPO           git repository to clone
+  --name=NAME           an identifier for this task
+  --verbose             display very verbose output
+  <launch-task-args>    a single-quoted string of arguments to pass-through to launch-task on the remote machine
+"""
+
+
+import os
+from subprocess import Popen
+import sys
+import uuid
+
+from docopt import docopt
+
+
+def main():
+    arguments = docopt(__doc__)
+
+    change_directory_to_ansible_script_home()
+
+    extra_vars = convert_cli_arguments_to_ansible_extra_vars(arguments)
+    
+    run_ansible_playbook(arguments.get('--verbose', False), extra_vars)
+
+
+def change_directory_to_ansible_script_home():
+    os.chdir(os.path.join(sys.prefix, 'share', 'edx.analytics.tasks'))
+
+
+def convert_cli_arguments_to_ansible_extra_vars(arguments):
+    uid = arguments.get('--name') or str(uuid.uuid4())
+    extra_vars = {
+        'name': arguments['--job-flow-id'],
+        'branch': arguments.get('--branch', 'release'),
+        'task_arguments': arguments.get('<launch-task-args>', '') + ' >/tmp/{0}.out 2>/tmp/{0}.err'.format(uid),
+        'uuid': uid,
+    }
+    if arguments['--repo']:
+        extra_vars['repo'] = arguments['--repo']
+    return ' '.join(["{}='{}'".format(k, extra_vars[k]) for k in extra_vars])
+
+
+def run_ansible_playbook(verbose, extra_vars):
+    ansible_playbook_path = os.path.join(sys.prefix, 'bin', 'ansible-playbook')
+    command = [
+         ansible_playbook_path, '-i', 'ec2.py', 'task.yml', '-e', extra_vars
+    ]
+    if verbose:
+        command.append('-vvvv')
+
+    env = dict(os.environ)
+    env.update({
+        'ANSIBLE_SSH_ARGS': '-o ForwardAgent=yes'
+    })
+
+    with open('/dev/null', 'rw') as devnull:
+        proc = Popen(command, stdin=devnull, env=env)
+        proc.communicate()

requirements/default.txt

@@ -3,5 +3,7 @@ boto==2.22.1
 pbr==0.5.23
 stevedore==0.13
 tornado==3.1.1
+ansible==1.4.4
+docopt==0.6.1
 
 -e git+https://github.com/spotify/luigi.git@a33756c781b9bf7e51384f0eb19d6a25050ef136#egg=luigi

setup.cfg

@@ -15,10 +15,13 @@ packages =
 namespace_packages =
     edx
     edx.analytics
+data_files =
+    share/edx.analytics.tasks = share/*
 
 [entry_points]
 console_scripts =
     launch-task = edx.analytics.tasks.main:main
+    remote-task = edx.analytics.tasks.remote:main
 edx.analytics.tasks =
     s3-copy = edx.analytics.tasks.s3:S3Copy
     s3-sync = edx.analytics.tasks.s3:S3Sync

share/README

+This directory contains ansible playbooks and their supporting files.  It is intended to be unpacked and ansible will be executed from the root of this directory.

share/ec2.ini

+[ec2]
+regions=us-east-1
+destination_variable=route53_domain_name
+backup_destination_variable=public_dns_name
+cache_path=/tmp
+cache_max_age=300
+regions_exclude=
+route53=True
\ No newline at end of file

share/roles/luigi/defaults/main.yml

+---
+
+luigi_hadoop_version: apache1
+luigi_hadoop_command: /home/hadoop/bin/hadoop
+luigi_hadoop_streaming_jar: /home/hadoop/.versions/1.0.3/share/hadoop/contrib/streaming/hadoop-streaming.jar

share/roles/luigi/tasks/main.yml

+---
+
+- name: configuration directory created
+  file: path=/etc/luigi state=directory mode=755
+
+- name: configuration file written
+  template: src=client.cfg.j2 dest=/etc/luigi/client.cfg mode=644

share/roles/luigi/templates/client.cfg.j2

+[hadoop]
+version: {{ luigi_hadoop_version }}
+command: {{ luigi_hadoop_command }}
+streaming-jar: {{ luigi_hadoop_streaming_jar }}

share/task.yml

+---
+
+- name: Configure luigi
+  hosts: "mr_{{ name }}_master"
+  gather_facts: False
+  sudo: True
+  roles:
+    - luigi
+
+- name: Run a task
+  hosts: "mr_{{ name }}_master"
+  gather_facts: False
+  sudo: False
+
+  vars:
+    - repo: git@github.com:edx/analytics-tasks.git
+    - branch: master
+    - working_dir: "/tmp/task-work-{{ uuid }}"
+    - working_repo_dir: "{{ working_dir }}/repo"
+    - working_venv_dir: "{{ working_dir }}/venv"
+    - task_arguments: ''
+    - git_server_hostname: github.com
+    - git_server_ip_address: 207.97.227.239
+    - git_server_public_key: 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==''
+
+  tasks:
+    - name: known_hosts file exists
+      command: touch /home/{{ ansible_ssh_user }}/.ssh/known_hosts creates=/home/{{ ansible_ssh_user }}/.ssh/known_hosts
+
+    - name: git server in known_hosts file
+      lineinfile: >
+        dest=/home/{{ ansible_ssh_user }}/.ssh/known_hosts
+        regexp=^github.com
+        line="{{ git_server_hostname }},{{ git_server_ip_address }} {{ git_server_public_key }}"
+
+    - name: working directories created
+      file: path={{ item }} state=directory
+      with_items:
+        - "{{ working_dir }}"
+        - "{{ working_repo_dir }}"
+        - "{{ working_venv_dir }}"
+
+    - name: analytics tasks repository checked out
+      git: repo={{ repo }} dest={{ working_repo_dir }}
+
+    - name: branch fetched
+      command: git fetch origin {{ branch }} chdir={{ working_repo_dir }}
+
+    - name: branch checked out
+      command: git checkout FETCH_HEAD chdir={{ working_repo_dir }}
+
+    - name: bootstrap pip
+      command: sudo apt-get install -q -y python-pip
+      sudo: True
+
+    - name: virtualenv installed
+      pip: name=virtualenv version=1.10.1
+      sudo: True
+
+    - name: virtualenv created
+      command: >
+        virtualenv {{ working_venv_dir }}
+
+    - name: virtualenv initialized
+      shell: >
+        . {{ working_venv_dir }}/bin/activate && make install
+        chdir={{ working_repo_dir }}
+
+    - name: task run
+      shell: >
+        {{ working_venv_dir }}/bin/launch-task {{ task_arguments }} chdir={{ working_repo_dir }}
+      async: 10000000000
+      poll: 0