Install Python packages to the system

playbooks/roles/jenkins_worker/files/install_python_pkgs.sh

+#!/usr/bin/env bash
+set -e
+
+#####################################################
+#
+# python_pkgs.sh
+#
+# Use easy_install to download packages from
+# an S3 bucket and install them in the system.
+#
+# Usage:
+#
+#    python_pkgs.sh S3_URL
+# 
+# where `S3_URL` is the URL of an S3 bucket
+# containing .egg files
+#
+######################################################
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 S3_URL"
+    exit 1
+fi
+
+S3_URL=$1
+
+# Retrieve the list of files in the bucket
+echo "Downloading Python packages from S3..."
+curl $S3_URL | xml_grep 'Key' --text_only > /tmp/python_pkgs.txt
+
+# Install each package into the virtualenv
+# If an error occurs, print stderr but do not abort
+echo "Installing Python packages..."
+while read package; do
+    easy_install $S3_URL/$package || true
+done < /tmp/python_pkgs.txt

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -49,6 +49,8 @@
     - unzip
     - xml-twig-tools
 
+# Configure Mongo to use /mnt so we don't
+# run out of disk space
 - name: jenkins | Stop mongo service
   service: name=mongodb state=stopped
 
@@ -66,3 +68,17 @@
 
 - name: jenkins | Start the mongo service
   service: name=mongodb state=stopped
+
+
+# Install scripts to load Python packages from S3 on startup
+- name: jenkins | Install Python packages script
+  copy: src="install_python_pkgs.sh" dest="/usr/local/bin/install_python_pkgs.sh"
+        force=yes
+
+- name: jenkins | Make Python packages script executable
+  file: path="/usr/local/bin/install_python_pkgs.sh"
+        owner=root group=root
+        mode=755
+
+- name: jenkins | Install Python packages upstart script
+  template: src="python_pkgs.conf.j2" dest="/etc/init/python_pkgs.conf"

playbooks/roles/jenkins_worker/templates/python_pkgs.conf.j2

+# Pre-install Python packages from S3
+
+start on runlevel [2345]
+
+task
+exec install_python_pkgs.sh {{ PYTHON_PKG_URL }}

playbooks/roles/jenkins_worker/vars/main.yml

@@ -20,3 +20,6 @@ jscover_url: "http://superb-dca2.dl.sourceforge.net/project/jscover/JSCover-1.0.
 # Mongo config
 MONGO_DIR: "/mnt/mongodb"
 MONGO_LOG_DIR: "/mnt/logs/mongodb"
+
+# URL of S3 bucket containing pre-compiled Python packages
+PYTHON_PKG_URL: "https://s3.amazonaws.com/jenkins.python_pkgs"