Merge pull request #1716 from edx/zoldak/pip-accel

[WIP] Use pip-accel for building jenkins workers

playbooks/roles/jenkins_worker/defaults/main.yml

@@ -32,3 +32,21 @@ jscover_version: "1.0.2"
 
 # packer direct download URL
 packer_url: "https://dl.bintray.com/mitchellh/packer/0.6.1_linux_amd64.zip"
+
+# Pip-accel itself and other workarounds that need to be installed with pip
+pip_accel_reqs:
+  # Install Shapely with pip as it does not install cleanly
+  # with pip-accel because it has a weird setup.py
+  - "Shapely==1.2.16"
+  # Install unittest2 which is needed by lettuce
+  # but also pip-accel has trouble with determining that.
+  # unittest2>=0.8.0 (from testtools>=0.9.34->python-subunit->lettuce==0.2.20)
+  - "unittest2>=0.8.0"
+  # There is a bug in pip 1.4.1 by which --exists-action is broken.
+  # This is fixed in pip 1.5.x, but alas pip-accel is not yet compatible with pip 1.5.x.
+  # Remove when we can upgrade to a version of pip-accel that supports pip 1.5.x.
+  - "git+https://github.com/jzoldak/pip.git@v1.4.1patch772#egg=pip"
+  # Install pip-accel itself (using pip)
+  - "pip-accel==0.21.1"
+  # pip-accel only makes the s3 functionality available if boto is installed
+  - "boto==2.34.0"

playbooks/roles/jenkins_worker/tasks/packer.yml

 ---
 - name: Download packer
   shell: "curl -L {{ packer_url }} -o /var/tmp/packer.zip"
+  args:
+      creates: /var/tmp/packer.zip
 
 - name: Unzip packer
   unarchive: src=/var/tmp/packer.zip dest=/usr/local/bin copy=no

playbooks/roles/jenkins_worker/tasks/python.yml

@@ -30,12 +30,24 @@
     depth=1
   sudo_user: "{{ jenkins_user }}"
 
-- name: Install edx-platform requirements
+- name: Pip installs that are needed for pip-accel to work for us
+  pip: >
+    name="{{ item }}"
+    virtualenv={{ jenkins_home }}/edx-venv
+    virtualenv_command=virtualenv-2.7
+  sudo_user: "{{ jenkins_user }}"
+  with_items: pip_accel_reqs
+
+# Install the platform requirements using pip-accel.
+# This will allow the binary distributions to be downloaded from S3
+# rather than compiled each time.
+- name: Install edx-platform requirements using pip-accel
   pip: >
     requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
-    extra_args="--exists-action w"
+    extra_args="--exists-action=w"
     virtualenv={{ jenkins_home }}/edx-venv
     virtualenv_command=virtualenv-2.7
+    executable=pip-accel
   with_items:
     - pre.txt
     - github.txt

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -57,3 +57,13 @@
   template:
     src=.npmrc.j2 dest={{ jenkins_home }}/.npmrc
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0664
+
+# Set up configuration for pip-accel for caching python requirements
+- name: Create directory for pip-accel config file
+  file: path={{ jenkins_home }}/.pip-accel state=directory
+    owner={{ jenkins_user }} group={{ jenkins_group }} mode=0777 recurse=yes
+
+- name: Create pip-accel config file
+  template:
+    src=pip-accel.conf.j2 dest={{ jenkins_home }}/.pip-accel/pip-accel.conf
+    owner={{ jenkins_user }} group={{ jenkins_group }} mode=0664

playbooks/roles/jenkins_worker/templates/pip-accel.conf.j2

+[pip-accel]
+auto-install = no
+data-directory = ~/.pip-accel
+download-cache = ~/.pip/download-cache
+s3-bucket = edx-platform.pip-accel-cache
+s3-prefix = precise64
+s3-readonly = no

util/packer/README.rst

+Packer
+=======
+
+``jenkins_worker.json`` is the `packer configuration template`_ that tells packer how to build the image.
+
+- `template variable`_ pattern ``"foo": "{{env `BAR`}}"``
+
+  - What this does is take the value of the environment variable BAR which was set in the shell that kicks off the ``packer build jenkins_worker.json`` command (for example in a jenkins job) and pass it through to the user variable "foo".
+  - This the user variable "foo" will now be available globally within the template.
+  - If the environment variable is not set in the shell that kicks off the packer build command, the user variable value will be the empty string.
+
+- Regarding the `ansible-playbook command`_ that is used to run the jenkins_worker role's -e (--extra-vars) option
+
+  - `playbook variable`_ pattern ``-e 'bar={{ user `foo` }}'``
+  - Packer has determined the value of the template user variable "foo" from the local environment variable (see above)
+  - Ansible will use this as the value for the playbook variable "bar" when running the play.
+
+.. _packer configuration template: http://www.packer.io/docs/templates/introduction.html
+.. _template variable: http://www.packer.io/docs/templates/user-variables.html
+.. _ansible-playbook command: http://docs.ansible.com/playbooks_intro.html#executing-a-playbook
+.. _playbook variable: http://docs.ansible.com/playbooks_variables.html#passing-variables-on-the-command-line

util/packer/jenkins_worker.json

@@ -10,11 +10,14 @@
     "type": "amazon-ebs",
     "access_key": "{{user `aws_access_key`}}",
     "secret_key": "{{user `aws_secret_key`}}",
+    "ami_name": "jenkins_worker-{{isotime | clean_ami_name}}",
+    "instance_type": "m3.large",
     "region": "us-east-1",
     "source_ami": "{{user `ami`}}",
-    "instance_type": "m3.large",
     "ssh_username": "ubuntu",
-    "ami_name": "jenkins_worker-{{isotime | clean_ami_name}}"
+    "ami_description": "jenkins worker",
+    "iam_instance_profile": "jenkins-worker",
+    "security_group_id": "sg-75af5e18"
   }],
   "provisioners": [{
     "type": "shell",
@@ -35,7 +38,7 @@
       "pip install -q -r requirements.txt",
       "echo '[jenkins_worker]' > inventory.ini",
       "echo 'localhost' >> inventory.ini",
-      "ansible-playbook edx-east/jenkins_worker.yml -i inventory.ini -c local -e 'github_oauth_token={{user `github_oauth_token` }}' -vvvv"]
+      "ansible-playbook edx-east/jenkins_worker.yml -i inventory.ini -c local -e 'github_oauth_token={{ user `github_oauth_token` }}' -vvvv"]
   }, {
     "type": "shell",
     "inline": ["cd {{user `playbook_remote_dir`}}",