Removed PhantomJS

Removed upstart scripts
Removed Python install scripts
Combined xvfb and browser playbooks
Changed Jenkins user base to /home/jenkins
Added wheel dir creation
Moved jenkins_env script
Changed data directory to /var
Install newer version of rake in rbenv role
Added edxlocal role
Remove system packages already installed by common and edxlocal
Added list of Python packages to pre-install

playbooks/jenkins_worker.yml

@@ -6,10 +6,8 @@
   hosts: jenkins_worker
   sudo: True
   gather_facts: True
-  vars:
-      COMMON_DATA_DIR: '/mnt'
-      COMMON_LOG_DIR: '/mnt/log'
   roles:
     - common
+    - edxlocal
     - mongo
     - jenkins_worker

playbooks/roles/common/defaults/main.yml

@@ -42,3 +42,5 @@ common_pip_pkgs:
 common_web_user: www-data
 common_web_group: www-data
 common_log_user: syslog
+
+common_git_ppa: "ppa:git-core/ppa"

playbooks/roles/common/tasks/main.yml

@@ -20,6 +20,16 @@
     path={{ COMMON_LOG_DIR }} state=directory owner=syslog
     group=syslog mode=0755
 
+# Need to install python-pycurl to use Ansible's apt_repository module
+- name: jenkins_worker | Install python-pycurl
+  apt: pkg=python-pycurl state=present update_cache=yes
+
+# Ensure that we get a current version of Git
+# GitHub requires version 1.7.10 or later
+# https://help.github.com/articles/https-cloning-errors
+- name: jenkins_worker | Add git apt repository
+  apt_repository: repo="{{ common_git_ppa }}"
+
 - name: common | Install role-independent useful system packages
   # do this before log dir setup; rsyslog package guarantees syslog user present
   apt: >

playbooks/roles/edxlocal/defaults/main.yml

@@ -3,4 +3,3 @@ edxlocal_debian_pkgs:
   - python-mysqldb
   - mysql-server-5.5
   - postfix
-  - python-pycurl

playbooks/roles/jenkins_worker/defaults/main.yml

 ---
-jenkins_workspace: "{{ COMMON_DATA_DIR }}/jenkins"
-jenkins_phantomjs_url: https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2
-jenkins_phantomjs_archive: phantomjs-1.9.1-linux-x86_64.tar.bz2
-jenkins_phantomjs_folder: phantomjs-1.9.1-linux-x86_64
 jenkins_user: "jenkins"
 jenkins_group: "jenkins"
-jenkins_user_home: /home/jenkins
+jenkins_home: /home/jenkins
 
 # System packages
 jenkins_debian_pkgs:
-    - ack-grep
     - build-essential
-    - git
     - gfortran
     - graphviz
-    - lynx-cur
     - libgraphviz-dev
     - libopenblas-dev
     - liblapack-dev
+    - libmysqlclient-dev
     - libxml2-dev
     - libgeos-dev
-    - libmysqlclient-dev
     - libxslt1-dev
     - npm
     - pkg-config
-    - python2.7
-    - python-pip
-    - python2.7-dev
-    - unzip
-    - xml-twig-tools
 
 # Chrome and ChromeDriver
 chrome_apt_key: "https://dl-ssl.google.com/linux/linux_signing_key.pub"
@@ -37,16 +25,98 @@ chromedriver_version: 2.3
 chromedriver_url: "http://chromedriver.storage.googleapis.com/{{ chromedriver_version }}/chromedriver_linux64.zip"
 
 # Ruby Specific Vars
-rbenv_root: "{{ jenkins_user_home }}/.rbenv"
-rbenv_repo: "https://github.com/sstephenson/rbenv.git"
-ruby_build_repo: "https://github.com/sstephenson/ruby-build.git"
-ruby_version: "1.9.3-p374"
+jenkins_rbenv_root: "{{ jenkins_home }}/.rbenv"
+jenkins_ruby_version: "1.9.3-p374"
 
 # JSCover direct download URL
 jscover_url: "http://superb-dca2.dl.sourceforge.net/project/jscover/JSCover-1.0.2.zip"
 jscover_version: "1.0.2"
 
-# URL of S3 bucket containing pre-compiled Python packages
-python_pkg_url: "https://s3.amazonaws.com/jenkins.python_pkgs"
-python_download_dir: "{{ COMMON_DATA_DIR }}/python_pkgs"
-python_virtualenv: "{{ COMMON_DATA_DIR}}/venv"
+# Python
+jenkins_venv: "{{ jenkins_home }}/wheel_venv"
+jenkins_pip: "{{ jenkins_venv }}/bin/pip"
+jenkins_wheel_dir: "{{ jenkins_home }}/wheelhouse"
+jenkins_wheels:
+    - { pkg: "numpy==1.6.2", wheel: "numpy-1.6.2-cp27-none-linux_x86_64.whl" }
+    - { pkg: "django-celery==3.0.17", wheel: "django_celery-3.0.17-py27-none-any.whl" }
+    - { pkg: "beautifulsoup4==4.1.3", wheel: "beautifulsoup4-4.1.3-py27-none-any.whl"}
+    - { pkg: "beautifulsoup==3.2.1", wheel: "BeautifulSoup-3.2.1-py27-none-any.whl" }
+    - { pkg: "bleach==1.2.2", wheel: "bleach-1.2.2-py27-none-any.whl" }
+    - { pkg: "html5lib==0.95", wheel: "html5lib-0.95-py27-none-any.whl" }
+    - { pkg: "boto==2.13.3", wheel: "boto-2.13.3-py27-none-any.whl" }
+    - { pkg: "celery==3.0.19", wheel: "celery-3.0.19-py27-none-any.whl" }
+    - { pkg: "dealer==0.2.3", wheel: "dealer-0.2.3-py27-none-any.whl" }
+    - { pkg: "django-countries==1.5", wheel: "django_countries-1.5-py27-none-any.whl" }
+    - { pkg: "django-filter==0.6.0", wheel: "django_filter-0.6-py27-none-any.whl" }
+    - { pkg: "django-followit==0.0.3", wheel: "django_followit-0.0.3-py27-none-any.whl" }
+    - { pkg: "django-kombu==0.9.4", wheel: "kombu-2.5.16-py27-none-any.whl" }
+    - { pkg: "django-mako==0.1.5pre", wheel: "django_mako-0.1.5pre-py27-none-any.whl" }
+    - { pkg: "django-model-utils==1.4.0", wheel: "django_model_utils-1.4.0-py27-none-any.whl" }
+    - { pkg: "django-masquerade==0.1.6", wheel: "django_masquerade-0.1.6-py27-none-any.whl" }
+    - { pkg: "django-mptt==0.5.5", wheel: "django_mptt-0.5.5-py27-none-any.whl" }
+    - { pkg: "django-openid-auth==0.4", wheel: "python_openid-2.2.5-py27-none-any.whl" }
+    - { pkg: "django-robots==0.9.1", wheel: "django_robots-0.9.1-py27-none-any.whl" }
+    - { pkg: "django-sekizai==0.6.1", wheel: "django_sekizai-0.6.1-py27-none-any.whl" }
+    - { pkg: "django-ses==0.4.1", wheel: "django_ses-0.4.1-py27-none-any.whl" }
+    - { pkg: "django-storages==1.1.5", wheel: "django_storages-1.1.5-py27-none-any.whl" }
+    - { pkg: "django-method-override==0.1.0", wheel: "django_method_override-0.1.0-py27-none-any.whl" }
+    - { pkg: "djangorestframework==2.3.5", wheel: "djangorestframework-2.3.5-py27-none-any.whl" }
+    - { pkg: "django==1.4.8", wheel: "Django-1.4.8-py27-none-any.whl" }
+    - { pkg: "feedparser==5.1.3", wheel: "feedparser-5.1.3-py27-none-any.whl" }
+    - { pkg: "fs==0.4.0", wheel: "fs-0.4.0-py27-none-any.whl" }
+    - { pkg: "GitPython==0.3.2.RC1", wheel: "GitPython-0.3.2.RC1-py27-none-any.whl" }
+    - { pkg: "glob2==0.3", wheel: "glob2-0.3-py27-none-any.whl" }
+    - { pkg: "gunicorn==0.17.4", wheel: "gunicorn-0.17.4-py27-none-any.whl" }
+    - { pkg: "lazy==1.1", wheel: "lazy-1.1-py27-none-any.whl" }
+    - { pkg: "lxml==3.0.1", wheel: "lxml-3.0.1-cp27-none-linux_x86_64.whl" }
+    - { pkg: "mako==0.7.3", wheel: "Mako-0.7.3-py27-none-any.whl" }
+    - { pkg: "Markdown==2.2.1", wheel: "Markdown-2.2.1-py27-none-any.whl" }
+    - { pkg: "networkx==1.7", wheel: "networkx-1.7-py27-none-any.whl" }
+    - { pkg: "nltk==2.0.4", wheel: "nltk-2.0.4-py27-none-any.whl" }
+    - { pkg: "oauthlib==0.5.1", wheel: "oauthlib-0.5.1-py27-none-any.whl" }
+    - { pkg: "paramiko==1.9.0", wheel: "paramiko-1.9.0-py27-none-any.whl" }
+    - { pkg: "path.py==3.0.1", wheel: "path.py-3.0.1-py27-none-any.whl" }
+    - { pkg: "Pillow==1.7.8", wheel: "Pillow-1.7.8-cp27-none-linux_x86_64.whl" }
+    - { pkg: "polib==1.0.3", wheel: "polib-1.0.3-py27-none-any.whl" }
+    - { pkg: "pycrypto>=2.6", wheel: "pycrypto-2.6.1-cp27-none-linux_x86_64.whl" }
+    - { pkg: "pygments==1.6", wheel: "Pygments-1.6-py27-none-any.whl" }
+    - { pkg: "pygraphviz==1.1", wheel: "pygraphviz-1.1-cp27-none-linux_x86_64.whl" }
+    - { pkg: "pymongo==2.4.1", wheel: "pymongo-2.4.1-cp27-none-linux_x86_64.whl" }
+    - { pkg: "pyparsing==1.5.6", wheel: "pyparsing-1.5.6-py27-none-any.whl" }
+    - { pkg: "python-memcached==1.48", wheel: "python_memcached-1.48-py27-none-any.whl" }
+    - { pkg: "python-openid==2.2.5", wheel: "django_openid_auth-0.4-py27-none-any.whl" }
+    - { pkg: "pytz==2012h", wheel: "pytz-2012h-py27-none-any.whl" }
+    - { pkg: "pysrt==0.4.7", wheel: "pysrt-0.4.7-py27-none-any.whl" }
+    - { pkg: "PyYAML==3.10", wheel: "PyYAML-3.10-cp27-none-linux_x86_64.whl" }
+    - { pkg: "requests==1.2.3", wheel: "requests-1.2.3-py27-none-any.whl" }
+    - { pkg: "scipy==0.11.0", wheel: "scipy-0.11.0-cp27-none-linux_x86_64.whl" }
+    - { pkg: "Shapely==1.2.16", wheel: "Shapely-1.2.16-cp27-none-linux_x86_64.whl" }
+    - { pkg: "singledispatch==3.4.0.2", wheel: "singledispatch-3.4.0.2-py27-none-any.whl" }
+    - { pkg: "sorl-thumbnail==11.12", wheel: "sorl_thumbnail-11.12-py27-none-any.whl" }
+    - { pkg: "South==0.7.6", wheel: "South-0.7.6-py27-none-any.whl" }
+    - { pkg: "sympy==0.7.1", wheel: "sympy-0.7.1-py27-none-any.whl" }
+    - { pkg: "xmltodict==0.4.1", wheel: "xmltodict-0.4.1-py27-none-any.whl" }
+    - { pkg: "django-ratelimit-backend==0.6", wheel: "django_ratelimit_backend-0.6-py27-none-any.whl" }
+    - { pkg: "ipython==0.13.1", wheel: "ipython-0.13.1-py27-none-any.whl" }
+    - { pkg: "watchdog==0.6.0", wheel: "watchdog-0.6.0-py27-none-any.whl" }
+    - { pkg: "dogapi==1.2.1", wheel: "dogapi-1.2.1-py27-none-any.whl" }
+    - { pkg: "newrelic==2.4.0.4", wheel: "newrelic-2.4.0.4-cp27-none-linux_x86_64.whl" }
+    - { pkg: "sphinx==1.1.3", wheel: "Sphinx-1.1.3-py27-none-any.whl" }
+    - { pkg: "Babel==1.3", wheel: "Babel-1.3-py27-none-any.whl" }
+    - { pkg: "transifex-client==0.9.1", wheel: "transifex_client-0.9.1-py27-none-any.whl" }
+    - { pkg: "coverage==3.6", wheel: "coverage-3.6-cp27-none-linux_x86_64.whl" }
+    - { pkg: "factory_boy==2.0.2", wheel: "factory_boy-2.0.2-py27-none-any.whl" }
+    - { pkg: "mock==1.0.1", wheel: "mock-1.0.1-py27-none-any.whl" }
+    - { pkg: "nosexcover==1.0.7", wheel: "nosexcover-1.0.7-py27-none-any.whl" }
+    - { pkg: "pep8==1.4.5", wheel: "pep8-1.4.5-py27-none-any.whl" }
+    - { pkg: "pylint==0.28", wheel: "pylint-0.28.0-py27-none-any.whl" }
+    - { pkg: "rednose==0.3", wheel: "rednose-0.3-py27-none-any.whl" }
+    - { pkg: "selenium==2.34.0", wheel: "selenium-2.34.0-py27-none-any.whl" }
+    - { pkg: "splinter==0.5.4", wheel: "splinter-0.5.4-py27-none-any.whl" }
+    - { pkg: "django_nose==1.1", wheel: "django_nose-1.1-py27-none-any.whl" }
+    - { pkg: "django_debug_toolbar", wheel: "django_debug_toolbar-0.10.2-py2.py3-none-any.whl" }
+    - { pkg: "django-debug-toolbar-mongo", wheel: "django_debug_toolbar_mongo-0.1.10-py27-none-any.whl" }
+    - { pkg: "nose-ignore-docstring", wheel: "nose_ignore_docstring-0.2-py27-none-any.whl" }
+    - { pkg: "nose-exclude", wheel: "nose_exclude-0.1.10-py27-none-any.whl" }
+    - { pkg: "django-crum==0.5", wheel: "django_crum-0.5-py27-none-any.whl" }
+    - { pkg: "MySQL-python==1.2.4", wheel: "MySQL_python-1.2.4-cp27-none-linux_x86_64.whl" }

playbooks/roles/jenkins_worker/files/download_python_pkgs.sh

-#!/usr/bin/env bash
-set -e
-
-#####################################################
-#
-# download_python_pkgs.sh
-#
-# Use download .egg packages from an S3 bucket 
-#
-# Usage:
-#
-#    download_python_pkgs.sh S3_URL SAVE_DIR
-# 
-# where `S3_URL` is the URL of an S3 bucket
-# containing .egg files
-#
-# and `SAVE_DIR` is the directory in which to save 
-# the .egg files.
-#
-######################################################
-
-if [ $# -ne 2 ]; then
-    echo "Usage: $0 S3_URL SAVE_DIR"
-    exit 1
-fi
-
-S3_URL=$1
-SAVE_DIR=$2
-
-# Create the save directory if it doesn't already exist
-mkdir -p $SAVE_DIR
-
-# Retrieve the list of files in the bucket
-echo "Downloading Python packages from S3..."
-curl $S3_URL | xml_grep 'Key' --text_only > $SAVE_DIR/python_pkgs.txt
-
-# Install each package into the virtualenv
-# If an error occurs, print stderr but do not abort
-echo "Installing Python packages..."
-while read package; do
-    curl $S3_URL/$package > $SAVE_DIR/$package || echo "Could not download $package"
-done < $SAVE_DIR/python_pkgs.txt

playbooks/roles/jenkins_worker/files/install_python_pkgs.sh

-#!/usr/bin/env bash
-set -e
-
-#####################################################
-#
-# install_python_pkgs.sh
-#
-# Use easy_install to install all
-# .egg files in a folder into a virtualenv.
-#
-# Usage:
-#
-#    install_python_pkgs.sh EGG_DIR VENV
-# 
-# where `EGG_DIR` is the directory containing
-# the .egg files
-#
-# and `VENV` is the virtualenv in which to install
-# the packages.  If the virtualenv does not yet
-# exist, it will be created.
-#
-# If the virtualenv has already been created
-# and the packages installed, then the script
-# will skip installation.
-#
-######################################################
-
-if [ $# -ne 2 ]; then
-    echo "Usage: $0 EGG_DIR VENV"
-    exit 1
-fi
-
-EGG_DIR=$1
-VENV=$2
-
-if [ -e $VENV/install_finished ]; then
-    echo "$VENV already exists; skipping installation..."
-else
-
-    # Create python egg cache and set correct permissions
-    PYTHON_EGG_CACHE=$HOME/.python-eggs
-    mkdir -p $PYTHON_EGG_CACHE
-    chmod 700 -R $PYTHON_EGG_CACHE
-
-    # Create and activate the virtualenv
-    echo "No virtualenv found; creating it..."
-    mkdir -p $VENV
-    virtualenv $VENV
-    . $VENV/bin/activate
-
-    # Install the .egg files into the virtualenv
-    echo "Installing Python eggs..."
-    for egg_file in $EGG_DIR/*.egg; do
-        easy_install $egg_file || true
-    done
-
-    # Create indicator that we finished successfully.
-    # If we were interrupted (maybe the job was aborted),
-    # then this file won't be created, so the next
-    # job will retry the intallation (instead of skipping it).
-    touch $VENV/install_finished
-fi

playbooks/roles/jenkins_worker/meta/main.yml

+---
+dependencies:
+  - role: rbenv
+    rbenv_user: "{{ jenkins_user }}"
+    rbenv_dir: "{{ jenkins_home }}"
+    rbenv_ruby_version: "{{ jenkins_ruby_version }}"

playbooks/roles/jenkins_worker/tasks/browsers.yml

 ---
+- name: jenkins_worker | install xvfb
+  apt: pkg=xvfb state=present
+
+- name: jenkins_worker | create xvfb upstart script
+  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
+
+- name: jenkins_worker | start xvfb
+  shell: start xvfb
+  ignore_errors: yes
+
 - name: jenkins_worker | Install Firefox
   apt: pkg=firefox
 
 - name: jenkins_worker | Install dbus-x11 (FF requirement)
   apt: pkg=dbus-x11
 
-- name: jenkins_worker | Download PhantomJS
-  get_url: url={{ jenkins_phantomjs_url }}
-           dest=/var/tmp/{{ jenkins_phantomjs_archive }}
-
-- name: jenkins_worker | Untar PhantomJS
-  command: tar -xjf /var/tmp/{{ jenkins_phantomjs_archive }} -C /var/tmp/
-           creates=/var/tmp/{{ jenkins_phantomjs_folder }}
-
-- name: jenkins_worker | Install PhantomJS
-  command: mv /var/tmp/{{ jenkins_phantomjs_folder }} /usr/local/bin/phantomjs
-           creates=/usr/local/bin/phantomjs
-
-- name: jenkins_worker | Set PhantomJS permissions
-  file: path=/usr/local/bin/phantomjs mode=0755 state=directory
-
 - name: jenkins_worker | Install Chrome dependencies
   apt: pkg=libgconf2-4,libxss1,libnss3-1d,libcurl3,xdg-utils
 

playbooks/roles/jenkins_worker/tasks/main.yml

@@ -10,8 +10,5 @@
 
 - include: system.yml
 - include: python.yml
-- include: ruby.yml
-- include: mongo.yml
-- include: xvfb.yml
 - include: browsers.yml
 - include: jscover.yml

playbooks/roles/jenkins_worker/tasks/mongo.yml

----
-
-# Configure Mongo to use {{ COMMON_DATA_DIR }} so we don't
-# run out of disk space
-- name: jenkins_worker | Stop mongo service
-  service: name=mongodb state=stopped
-
-- name: jenkins_worker | Configure Mongo upstart script
-  template: src=mongodb_upstart.j2
-    dest=/etc/init/mongodb.conf
-    owner=root
-    group=root
-
-- name: jenkins_worker | Start the mongo service
-  service: name=mongodb state=stopped

playbooks/roles/jenkins_worker/tasks/python.yml

 ---
 
-- name: jenkins_worker | Install virtualenv
-  pip: name=virtualenv state=present
-
-- name: jenkins_worker | Install virtualenv wrapper
-  pip: name=virtualenvwrapper state=present
-
-- name: jenkins_worker | Install requests
+# Install scripts requiring a GitHub OAuth token
+- name: jenkins_worker | Install requests Python library
   pip: name=requests state=present
 
-# Install bash scripts
-- name: jenkins_worker | Install Python packages scripts
-  copy: src="${item}" dest="/usr/local/bin/${item}"
-        force=yes
-        owner=root group=root
-        mode=755
-  with_items:
-    - download_python_pkgs.sh
-    - install_python_pkgs.sh
-
-# Install scripts requiring a GitHub OAuth token
 - fail: jenkins_worker | OAuth token not defined
   when: github_oauth_token is not defined
 
@@ -33,6 +17,30 @@
             owner=root group=root
             mode=755
 
-# Install upstart script to download Python packages from S3
-- name: jenkins_worker | Install Python packages upstart script
-  template: src="python_pkgs.conf.j2" dest="/etc/init/python_pkgs.conf"
+# Create wheelhouse to enable fast virtualenv creation
+- name: jenkins_worker | Create wheel virtualenv
+  command: /usr/local/bin/virtualenv {{ jenkins_venv }} creates={{ jenkins_venv }}
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Install wheel
+  pip: name=wheel virtualenv={{ jenkins_venv }} virtualenv_command=/usr/local/bin/virtualenv
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Create wheelhouse dir
+  file:
+    path={{ jenkins_wheel_dir }} state=directory
+    owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
+
+# (need to install each one in the venv to satisfy dependencies)
+- name: jenkins_worker | Create wheel archives
+  shell:
+    "{{ jenkins_pip }} wheel --wheel-dir={{ jenkins_wheel_dir }} \"${item.pkg}\" &&
+    {{ jenkins_pip }} install --use-wheel --no-index --find-links={{ jenkins_wheel_dir }} \"${item.pkg}\"
+    creates={{ jenkins_wheel_dir }}/${item.wheel}"
+  sudo_user: "{{ jenkins_user }}"
+  with_items: "{{ jenkins_wheels }}"
+
+- name: jenkins_worker | Add wheel_venv.sh script
+  template:
+    src=wheel_venv.sh.j2 dest={{ jenkins_home }}/wheel_venv.sh
+    owner={{ jenkins_user }} group={{ jenkins_group }} mode=700

playbooks/roles/jenkins_worker/tasks/ruby.yml

----
-
-- name: jenkins_worker | Install rbenv
-  git: repo={{ rbenv_repo }} dest={{ rbenv_root }}
-  sudo_user: "{{ jenkins_user }}"
-
-- name: jenkins_worker | Install ruby-build
-  git: repo={{ ruby_build_repo }} dest={{ rbenv_root }}/plugins/ruby-build
-  sudo_user: "{{ jenkins_user }}"
-
-- name: jenkins_worker | Install ruby
-  command: "{{ rbenv_root }}/bin/rbenv install {{ ruby_version }}
-           creates={{ rbenv_root }}/versions/{{ ruby_version }}"
-  sudo_user: "{{ jenkins_user }}"
-
-- name: jenkins_worker | Set global ruby
-  command: "{{ rbenv_root }}/bin/rbenv global {{ ruby_version }}"
-  sudo_user: "{{ jenkins_user }}"
-
-- name: jenkins_worker | Install bundler
-  command: "{{ rbenv_root }}/shims/gem install bundler"
-  sudo_user: "{{ jenkins_user }}"
-
-- name: jenkins_worker | Rbenv rehash
-  command: "{{ rbenv_root }}/bin/rbenv rehash"
-  sudo_user: "{{ jenkins_user }}"

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -5,48 +5,37 @@
 - name: jenkins_worker | Add the jenkins user to the group
   user: name={{ jenkins_user }} append=yes group={{ jenkins_group }}
 
-
-# We need the upstart script to create the build directory
-# so that (a) it will be run when a new instance is created
-# on the current EBS, and (b) it will be run as root.
-- name: jenkins_worker | Install upstart script to create build dir
-  template: src=jenkins_workspace.conf.j2
-    dest=/etc/init/jenkins_workspace.conf
-    owner=root group=root
-
-
 # Because of a bug in the latest release of the EC2 plugin
 # we need to use a key generated by Amazon (not imported)
 # To satisfy this, we allow users to log in as Jenkins
 # using the same keypair the instance was started with.
 - name: jenkins_worker | Create .ssh directory
-  file: path={{ jenkins_user_home }}/.ssh state=directory
-        owner={{ jenkins_user }} group={{ jenkins_group }}
+  file:
+    path={{ jenkins_home }}/.ssh state=directory
+    owner={{ jenkins_user }} group={{ jenkins_group }}
+  ignore_errors: yes
 
 - name: jenkins_worker | Copy ssh keys for jenkins
   command: cp /home/ubuntu/.ssh/authorized_keys /home/{{ jenkins_user }}/.ssh/authorized_keys
+  ignore_errors: yes
 
 - name: jenkins_worker | Set key permissions
-  file: path={{ jenkins_user_home }}/.ssh/authorized_keys
-        owner={{ jenkins_user }} group={{ jenkins_group }}
-        mode=400
-
-# Ensure that we get a current version of Git
-# GitHub requires version 1.7.10 or later
-# https://help.github.com/articles/https-cloning-errors
-- name: jenkins_worker | Add git apt repository
-  apt_repository: repo='ppa:git-core/ppa'
+  file:
+    path={{ jenkins_home }}/.ssh/authorized_keys
+    owner={{ jenkins_user }} group={{ jenkins_group }} mode=400
+  ignore_errors: yes
 
 - name: jenkins_worker | Install system packages
   apt: pkg={{','.join(jenkins_debian_pkgs)}}
        state=present update_cache=yes
 
 - name: jenkins_worker | Add script to set up environment variables
-  template: src=jenkins_env.sh.j2 dest=/usr/local/bin/jenkins_env.sh
-            owner=root group=root mode=0555
+  template:
+    src=jenkins_env.j2 dest={{ jenkins_home }}/jenkins_env
+    owner={{ jenkins_user }} group={{ jenkins_group }} mode=0500
 
 # Need to add Github to known_hosts to avoid
 # being prompted when using git through ssh
 - name: jenkins_worker | Add github.com to known_hosts if it does not exist
   shell: >
-     ssh-keygen -f {{ jenkins_user_home }}/.ssh/known_hosts -H -F github.com | grep -q found || ssh-keyscan -H github.com > {{ jenkins_user_home }}/.ssh/known_hosts
+     ssh-keygen -f {{ jenkins_home }}/.ssh/known_hosts -H -F github.com | grep -q found || ssh-keyscan -H github.com > {{ jenkins_home }}/.ssh/known_hosts

playbooks/roles/jenkins_worker/tasks/xvfb.yml

----
-- name: jenkins_worker | install xvfb
-  apt: pkg=xvfb state=present
-
-- name: jenkins_worker | create xvfb upstart script
-  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
-
-- name: jenkins_worker | start xvfb
-  shell: start xvfb
-  ignore_errors: yes

playbooks/roles/jenkins_worker/templates/jenkins_env.sh.j2 → playbooks/roles/jenkins_worker/templates/jenkins_env.j2

 # Configure Ruby
-export RBENV_ROOT="{{ rbenv_root }}"
-export PATH=$RBENV_ROOT/bin/:$PATH
-eval "$(rbenv init -)"
+export GEM_ROOT="{{ jenkins_home }}/.gem"
+export GEM_HOME="{{ jenkins_home }}/.gem"
+export PATH="{{ jenkins_rbenv_root }}/bin:{{jenkins_rbenv_root }}/shims:{{ jenkins_home }}/.gem/bin:$PATH"
+export RBENV_ROOT="{{ jenkins_rbenv_root }}"
 
 # Configure JavaScript coverage
 export JSCOVER_JAR=/usr/local/bin/JSCover-all-{{ jscover_version }}.jar

playbooks/roles/jenkins_worker/templates/jenkins_workspace.conf.j2

-# Create directory for Jenkins jobs
-
-start on runlevel [2345]
-
-task
-script
-    mkdir -p {{ jenkins_workspace }}
-    chown {{ jenkins_user }}:{{ jenkins_group }} {{ jenkins_workspace }}
-    chmod 700 {{ jenkins_workspace }}
-end script

playbooks/roles/jenkins_worker/templates/mongodb_upstart.j2

-# Ubuntu upstart file at /etc/init/mongodb.conf
-
-pre-start script
-    mkdir -p {{ mongo_data_dir }}
-    mkdir -p {{ mongo_log_dir }}
-
-    touch {{ mongo_log_dir }}/mongodb.log
-
-    chown mongodb:nogroup -R {{ mongo_data_dir }}
-    chown mongodb:nogroup -R {{ mongo_log_dir }}
-end script
-
-start on runlevel [2345]
-stop on runlevel [06]
-
-script
-    ENABLE_MONGODB="yes"
-    if [ -f /etc/default/mongodb ]; then . /etc/default/mongodb; fi
-    if [ "x$ENABLE_MONGODB" = "xyes" ]; then exec start-stop-daemon --start --quiet --chuid mongodb --exec  /usr/bin/mongod -- --config /etc/mongodb.conf; fi
-end script

playbooks/roles/jenkins_worker/templates/python_pkgs.conf.j2

-# Pre-install Python packages from S3
-
-start on runlevel [2345]
-
-task
-script
-
-    # Create the directory to hold Python virtualenvs
-    mkdir -p {{ python_virtualenv }}
-
-    # Download .egg files from S3
-    download_python_pkgs.sh {{ python_pkg_url }} {{ python_download_dir }}
-
-    # Give the Jenkins user access
-    chown {{ jenkins_user }}:{{ jenkins_group }} -R {{ python_download_dir }}
-    chmod 500 -R {{ python_download_dir }}
-
-    chown {{ jenkins_user }}:{{ jenkins_group }} -R {{ python_virtualenv }}
-    chmod 700 -R {{ python_virtualenv }}
-
-end script

playbooks/roles/jenkins_worker/templates/wheel_venv.sh.j2

+#! /usr/bin/env bash
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 VENV_DIR"
+    exit 1
+fi
+
+# Create and activate the new virtualenv
+VENV=$1
+mkdir -p $VENV
+/usr/local/bin/virtualenv $VENV
+. $VENV/bin/activate
+
+# Install each available wheel archive
+ls {{ jenkins_wheel_dir }} | cut -d- -f1 | while read line ; do
+    pip install --use-wheel --no-index --find-links={{ jenkins_wheel_dir }} $line ;
+done

playbooks/roles/rbenv/defaults/main.yml

@@ -2,6 +2,7 @@
 
 rbenv_version: 'v0.4.0'
 rbenv_bundler_version: '1.3.2'
+rbenv_rake_version: '10.0.3'
 rbenv_root: "{{ rbenv_dir }}/.rbenv"
 rbenv_gem_root: "{{ rbenv_dir }}/.gem"
 rbenv_gem_bin: "{{ rbenv_gem_root }}/bin"

playbooks/roles/rbenv/tasks/main.yml

@@ -114,6 +114,14 @@
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
 
+- name: rbenv | remove rbenv version of rake
+  file: path="{{ rbenv_dir }}/.rbenv/versions/{{ rbenv_ruby_version }}/bin/rake" state=absent
+
+- name: rbenv | install rake gem
+  shell: "gem install rake -v {{ rbenv_rake_version }}"
+  sudo_user: "{{ rbenv_user }}"
+  environment: "{{ rbenv_environment }}"
+
 - name: rbenv | rehash
   shell: "rbenv rehash"
   sudo_user: "{{ rbenv_user }}"