Merge pull request #3824 from edx/pwnage101/loadtest-worker-with-venv

bake pip requirements into the loadtest worker AMI

playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml

+---
+# Create a virtualenv for edx-load-tests by installing the requirements and
+# packaging the virtualenv.
+
+- name: Create shallow clone of edx-load-tests
+  git:
+    repo: https://github.com/edx/edx-load-tests.git
+    dest: "{{ jenkins_home }}/shallow-clone"
+    version: "master"
+    depth: 1
+  become_user: "{{ jenkins_user }}"
+
+- name: Install most edx-load-tests requirements using pip
+  pip:
+    requirements: "{{ jenkins_home }}/shallow-clone/requirements/{{ item }}"
+    extra_args: "--exists-action=w"
+    virtualenv: "{{ jenkins_home }}/edx-venv"
+    virtualenv_command: virtualenv
+  with_items:
+    - base.txt
+  become_user: "{{ jenkins_user }}"
+
+# Archive the current state of the virtualenv as a starting point for new
+# builds.  The edx-venv directory is deleted and then recreated cleanly from
+# the archive by the jenkins build scripts.
+#
+# TODO: after we migrate to ansible 2.3+, change this task to use the archive
+# module.  http://docs.ansible.com/ansible/archive_module.html
+- name: Create a clean virtualenv archive
+  command: "tar -cpzf edx-venv_clean.tar.gz edx-venv"
+  args:
+    chdir: "{{ jenkins_home }}"
+  become_user: "{{ jenkins_user }}"
+
+- name: Remove the shallow-clone
+  file: path={{ jenkins_home }}/shallow-clone state=absent

playbooks/roles/jenkins_worker/tasks/main.yml

@@ -14,6 +14,10 @@
 - include: python_platform_worker.yml
   when: platform_worker is defined
 
+# only loadtest driver workers
+- include: loadtest_driver_worker.yml
+  when: loadtest_driver_worker is defined
+
 # Run appropriate tests
 - include: test.yml
 - include: test_platform_worker.yml

util/packer/jenkins_worker_loadtest.json

@@ -2,6 +2,7 @@
   "variables": {
     "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
     "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
+    "new_relic_key": "{{env `NEW_RELIC_KEY`}}",
     "playbook_remote_dir": "/tmp/packer-edx-playbooks",
     "venv_dir": "/edx/app/edx_ansible/venvs/edx_ansible",
     "ami": "{{env `JENKINS_WORKER_AMI`}}",
@@ -48,6 +49,9 @@
     "playbook_dir": "../../playbooks",
     "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook",
     "inventory_groups": "jenkins_worker",
-    "extra_arguments": [ "-vvv" ]
+    "extra_arguments": [
+      "-e \"NEWRELIC_LICENSE_KEY={{user `new_relic_key`}}\"",
+      "-vvv"
+    ]
   }]
 }