Global virtualenv configuration

playbooks/roles/jenkins_worker/files/download_python_pkgs.sh

+#!/usr/bin/env bash
+set -e
+
+#####################################################
+#
+# download_python_pkgs.sh
+#
+# Use download .egg packages from an S3 bucket 
+#
+# Usage:
+#
+#    download_python_pkgs.sh S3_URL SAVE_DIR
+# 
+# where `S3_URL` is the URL of an S3 bucket
+# containing .egg files
+#
+# and `SAVE_DIR` is the directory in which to save 
+# the .egg files.
+#
+######################################################
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 S3_URL SAVE_DIR"
+    exit 1
+fi
+
+S3_URL=$1
+SAVE_DIR=$2
+
+# Create the save directory if it doesn't already exist
+mkdir -p $SAVE_DIR
+
+# Retrieve the list of files in the bucket
+echo "Downloading Python packages from S3..."
+curl $S3_URL | xml_grep 'Key' --text_only > $SAVE_DIR/python_pkgs.txt
+
+# Install each package into the virtualenv
+# If an error occurs, print stderr but do not abort
+echo "Installing Python packages..."
+while read package; do
+    curl $S3_URL/$package > $SAVE_DIR/$package || echo "Could not download $package"
+done < $SAVE_DIR/python_pkgs.txt

playbooks/roles/jenkins_worker/files/install_python_pkgs.sh

@@ -3,34 +3,48 @@ set -e
 
 #####################################################
 #
-# python_pkgs.sh
+# install_python_pkgs.sh
 #
-# Use easy_install to download packages from
-# an S3 bucket and install them in the system.
+# Use easy_install to install all
+# .egg files in a folder into a virtualenv.
 #
 # Usage:
 #
-#    python_pkgs.sh S3_URL
+#    install_python_pkgs.sh EGG_DIR VENV
 # 
-# where `S3_URL` is the URL of an S3 bucket
-# containing .egg files
+# where `EGG_DIR` is the directory containing
+# the .egg files
+#
+# and `VENV` is the virtualenv in which to install
+# the packages.  If the virtualenv does not yet
+# exist, it will be created.
+#
+# If the virtualenv already exists, it will not
+# be modified.
 #
 ######################################################
 
-if [ $# -ne 1 ]; then
-    echo "Usage: $0 S3_URL"
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 EGG_DIR VENV"
     exit 1
 fi
 
-S3_URL=$1
+EGG_DIR=$1
+VENV=$2
 
-# Retrieve the list of files in the bucket
-echo "Downloading Python packages from S3..."
-curl $S3_URL | xml_grep 'Key' --text_only > /tmp/python_pkgs.txt
+if [ -e $VENV/bin/activate ]; then
+    echo "$VENV already exists; skipping installation..."
+else
 
-# Install each package into the virtualenv
-# If an error occurs, print stderr but do not abort
-echo "Installing Python packages..."
-while read package; do
-    easy_install $S3_URL/$package || true
-done < /tmp/python_pkgs.txt
+    # Create and activate the virtualenv
+    echo "No virtualenv found; creating it..."
+    mkdir -p $VENV
+    virtualenv $VENV
+    . $VENV/bin/activate
+
+    # Install the .egg files into the virtualenv
+    echo "Installing Python eggs..."
+    for egg_file in $EGG_DIR/*.egg; do
+        easy_install $egg_file || true
+    done
+fi

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -76,15 +76,16 @@
   service: name=mongodb state=stopped
 
 
-# Install scripts to load Python packages from S3 on startup
-- name: jenkins | Install Python packages script
-  copy: src="install_python_pkgs.sh" dest="/usr/local/bin/install_python_pkgs.sh"
+# Install bash scripts
+- name: jenkins | Install Python packages scripts
+  copy: src="${item}" dest="/usr/local/bin/${item}"
         force=yes
-
-- name: jenkins | Make Python packages script executable
-  file: path="/usr/local/bin/install_python_pkgs.sh"
         owner=root group=root
         mode=755
+  with_items:
+    - download_python_pkgs.sh
+    - install_python_pkgs.sh
 
+# Install upstart script to download Python packages from S3
 - name: jenkins | Install Python packages upstart script
   template: src="python_pkgs.conf.j2" dest="/etc/init/python_pkgs.conf"

playbooks/roles/jenkins_worker/templates/mongodb_conf.j2

 # mongodb.conf
 
-dbpath={{ MONGO_DIR }}
-logpath={{ MONGO_LOG_DIR }}/mongodb.log
+dbpath={{ mongo_dir }}
+logpath={{ mongo_log_dir }}/mongodb.log
 
 logappend=true
 

playbooks/roles/jenkins_worker/templates/mongodb_upstart.j2

 # Ubuntu upstart file at /etc/init/mongodb.conf
 
 pre-start script
-    mkdir -p {{ MONGO_DIR }}
-    mkdir -p {{ MONGO_LOG_DIR }}
+    mkdir -p {{ mongo_dir }}
+    mkdir -p {{ mongo_log_dir }}
 
-    touch {{ MONGO_LOG_DIR }}/mongodb.log
+    touch {{ mongo_log_dir }}/mongodb.log
 
-    chown mongodb:nogroup -R {{ MONGO_DIR }}
-    chown mongodb:nogroup -R {{ MONGO_LOG_DIR }}
+    chown mongodb:nogroup -R {{ mongo_dir }}
+    chown mongodb:nogroup -R {{ mongo_log_dir }}
 end script
 
 start on runlevel [2345]

playbooks/roles/jenkins_worker/templates/python_pkgs.conf.j2

@@ -3,4 +3,16 @@
 start on runlevel [2345]
 
 task
-exec install_python_pkgs.sh {{ PYTHON_PKG_URL }}
+script
+
+    # Create the directory to hold Python virtualenvs
+    mkdir -p {{ python_virtualenv }}
+
+    # Download .egg files from S3
+    download_python_pkgs.sh {{ python_pkg_url }} {{ python_download_dir }}
+
+    # Give the Jenkins user access
+    chown {{ jenkins_user }}:{{ jenkins_group }} -R 600 {{ python_download_dir }}
+    chown {{ jenkins_user }}:{{ jenkins_group }} -R 600 {{ python_virtualenv }}
+
+end script

playbooks/roles/jenkins_worker/vars/main.yml

@@ -18,8 +18,10 @@ gem_home: "{{ ruby_base }}/.gem"
 jscover_url: "http://superb-dca2.dl.sourceforge.net/project/jscover/JSCover-1.0.2.zip"
 
 # Mongo config
-MONGO_DIR: "/mnt/mongodb"
-MONGO_LOG_DIR: "/mnt/logs/mongodb"
+mongo_dir: "/mnt/mongodb"
+mongo_log_dir: "/mnt/logs/mongodb"
 
 # URL of S3 bucket containing pre-compiled Python packages
-PYTHON_PKG_URL: "https://s3.amazonaws.com/jenkins.python_pkgs"
+python_pkg_url: "https://s3.amazonaws.com/jenkins.python_pkgs"
+python_download_dir: "/mnt/python_pkgs"
+python_virtualenv: "/mnt/venv"