Merge pull request #242 from edx/will/jenkins-provision-updates

WIP: Jenkins Provision Updates

playbooks/jenkins.yml

-- name: Configure instance(s)
-  hosts: jenkins
-  sudo: True
-  gather_facts: True
-  roles:
-    - common
-    - edxlocal
-    - role: rbenv
-      rbenv_user: "{{ jenkins_user }}"
-      rbenv_user_home: "{{ jenkins_user_home }}"
-      rbenv_ruby_version: "{{ jenkins_ruby_version }}"
-    - jenkins

playbooks/jenkins_master.yml

+# Configure a Jenkins master instance
+# This has the Jenkins Java app, but none of the requirements
+# to run the tests.
+
+- name: Configure instance(s)
+  hosts: jenkins_master
+  sudo: True
+  gather_facts: True
+  roles:
+    - jenkins_master

playbooks/jenkins_worker.yml

+# Configure a Jenkins worker instance
+# This has all the requirements to run test jobs,
+# but not the Jenkins Java app.
+
+- name: Configure instance(s)
+  hosts: jenkins_worker
+  sudo: True
+  gather_facts: True
+  roles:
+    - jenkins_worker

playbooks/roles/jenkins/defaults/main.yml

----
-jenkins_home: /var/lib/jenkins
-jenkins_phantomjs_url: https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2
-jenkins_phantomjs_archive: phantomjs-1.9.1-linux-x86_64.tar.bz2
-jenkins_phantomjs_folder: phantomjs-1.9.1-linux-x86_64
-jenkins_user: "jenkins"
-jenkins_user_home: /home/jenkins
-jenkins_ruby_version: "1.9.3-p448"
-jenkins_git_identity_path: "{{secure_dir}}/files/git-identity"
-
-jenkins_debian_pkgs:
-    - ruby-bundler
-    - rubygems
-    - rbenv
-    - npm
-    - libgraphviz-dev
-    - gfortran
-    - libopenblas-dev
-    - liblapack-dev
-    - libxml2-dev
-    - libgeos-dev
-    - python-dev
-    - libmysqlclient-dev
-    - build-essential
-    - pkg-config
-    - libxslt1-dev
-    - rake
-jenkins_plugins:
-    - ant.hpi
-    - backup.hpi
-    - build-name-setter.hpi
-    - build-timeout.hpi
-    - cobertura.hpi
-    - credentials.hpi
-    - cvs.hpi
-    - dashboard-view.hpi
-    - external-monitor-job.hpi
-    - ghprb.hpi
-    - git.hpi
-    - github.hpi
-    - git-client.hpi
-    - github-api.hpi
-    - github-oauth.hpi
-    - git-notes.hpi
-    - htmlpublisher.hpi
-    - javadoc.hpi
-    - jobConfigHistory.hpi
-    - ldap.hpi
-    - mailer.hpi
-    - mercurial.hpi
-    - nested-view.hpi
-    - next-build-number.hpi
-    - notification.hpi
-    - pam-auth.hpi
-    - parameterized-trigger.hpi
-    - postbuild-task.hpi
-    - rbenv.hpi
-    - ruby-runtime.hpi
-    - shiningpanda.hpi
-    - ssh-credentials.hpi
-    - ssh-slaves.hpi
-    - subversion.hpi
-    - thinBackup.hpi
-    - tmpcleaner.hpi
-    - token-macro.hpi
-    - translation.hpi
-    - violations.hpi
-    - multiple-scms.hpi
-    - timestamper.hpi

playbooks/roles/jenkins/handlers/main.yml

----
-- name: start xvfb
-  service: name=xvfb state=started
-
-- name: restart Jenkins
-  service: name=jenkins state=restarted

playbooks/roles/jenkins/tasks/config-backup.yml

----
-- name: Make backup directory
-  sudo_user: jenkins
-  shell: mkdir {{ jenkins_home }}/backup
-
-- name: Git clone
-  sudo_user: jenkins
-  action: git repo=https://github.com/lapentab/jenkinsconfig.git dest={{jenkins_home}}/backup

playbooks/roles/jenkins/tasks/jenkins.yml

----
-- name: jenkins | Add the jenkins user to the edx group
-  user: name=jenkins append=yes groups="edx"
-
-- name: jenkins | install jenkins specific system packages
-  apt: pkg={{','.join(jenkins_debian_pkgs)}} state=present
-
-- name: jenkins | Install gcli
-  sudo_user: jenkins
-  shell: /home/jenkins/.rbenv/bin/rbenv exec gem install github_cli
-
-# This is done so that it will not report back to github while testing
-- name: jenkins | TEMPORARILY DISABLE gcli
-  shell: echo "#!/bin/bash\necho gcli" > /usr/local/bin/gcli
-
-- name: jenkins | Install Jenkins
-  shell: wget -q -O - http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add -
-
-- name: jenkins | Install Jenkins 2
-  shell: sh -c 'echo deb http://pkg.jenkins-ci.org/debian binary/ > /etc/apt/sources.list.d/jenkins.list'
-
-- name: jenkins | Install Jenkins 3
-  shell: apt-get update
-
-- name: jenkins | install jenkins 4
-  apt: pkg=jenkins state=present
-
-- name: jenkins | Make plugins directory
-  sudo_user: jenkins
-  shell: mkdir -p {{ jenkins_home }}/plugins
-
-- name: jenkins | Make virtualenv directory
-  shell: mkdir -p /mnt/virtualenvs
-
-- name: jenkins | Chown virtualenv directory
-  file: path=/mnt/virtualenvs/ state=directory owner=jenkins recurse=yes
-
-- name: jenkins | Make pip-cache directory
-  shell: mkdir -p /mnt/pip-cache
-
-- name: jenkins | Chown pip-cache directory
-  file: path=/mnt/pip-cache/ state=directory owner=jenkins recurse=yes
-
-- name: jenkins | Allow /usr/local/bin to be executable
-  shell: chmod -R go+x /usr/local/bin
-
-- name: jenkins | Install Jenkins plugins
-  sudo_user: jenkins
-  get_url: url=http://updates.jenkins-ci.org/latest/${item}
-           dest={{ jenkins_home }}/plugins/${item}
-  with_items: "{{ jenkins_plugins }}"
-    #- timestamper.hpi
-    #- maven-plugin.hpi
-  notify:
-    - restart Jenkins
-
-- name: jenkins | install ssh key for private git repos
-  copy: >
-    src="{{jenkins_git_identity_path}}"
-    dest="{{jenkins_user_home}}/.ssh/id_rsa"
-    force=yes
-    owner="{{jenkins_user}}"
-    group="{{jenkins_user}}"
-    mode=600
-  when: c_install_ssh_key is defined

playbooks/roles/jenkins/tasks/main.yml

----
-- include: jenkins.yml
-- include: xvfb.yml
-- include: browsers.yml
-- include: python-pkgs.yml

playbooks/roles/jenkins/tasks/python-pkgs.yml

----
-- name: python packages | Symbolic link site packages
-  file: src=/usr/local/lib/python2.7/site-packages dest=/usr/local/lib/python2.7/dist-packages state=link
-  ignore_errors: yes
-
-- name: python packages | install numpy
-  shell: pip install numpy==1.6.2
-
-- name: python packages | install scipy
-  shell: pip install scipy==0.11.0

playbooks/roles/jenkins/templates/xvfb.sh.j2

-# Set the display to the virtual frame buffer (Xvfb)
-export DISPLAY=:1

playbooks/roles/jenkins_master/defaults/main.yml

+jenkins_home: /var/lib/jenkins
+jenkins_user: "jenkins"
+jenkins_group: "edx"
+jenkins_server_name: "jenkins.testeng.edx.org"
+jenkins_port: 8080
+
+jenkins_plugins:
+    - { name: "build-name-setter", version: "1.3" }
+    - { name: "build-pipeline-plugin", version: "1.4" }
+    - { name: "build-timeout", version: "1.11" }
+    - { name: "cobertura", version: "1.9.2" }
+    - { name: "copyartifact", version: "1.28" }
+    - { name: "credentials", version: "1.8.3" }
+    - { name: "dashboard-view", version: "2.9.1" }
+    - { name: "ec2", version: "1.18" }
+    - { name: "git", version: "1.5.0" }
+    - { name: "github", version: "1.8" }
+    - { name: "github-api", version: "1.44" }
+    - { name: "github-oauth", version: "0.14" }
+    - { name: "htmlpublisher", version: "1.2" }
+    - { name: "javadoc", version: "1.1" }
+    - { name: "jobConfigHistory", version: "2.4" }
+    - { name: "jquery", version: "1.7.2-1" }
+    - { name: "nested-view", version: "1.10" }
+    - { name: "next-build-number", version: "1.0" }
+    - { name: "notification", version: "1.5" }
+    - { name: "pam-auth", version: "1.0" }
+    - { name: "parameterized-trigger", version: "2.20" }
+    - { name: "postbuild-task", version: "1.8" }
+    - { name: "s3", version: "0.5" }
+    - { name: "ssh-agent", version: "1.3" }
+    - { name: "ssh-credentials", version: "0.3" }
+    - { name: "ssh-slaves", version: "0.25" }
+    - { name: "shiningpanda", version: "0.20" }
+    - { name: "tmpcleaner", version: "1.1" }
+    - { name: "token-macro", version: "1.8.1" }
+    - { name: "translation", version: "1.10" }
+    - { name: "violations", version: "0.7.11" }
+    - { name: "multiple-scms", version: "0.2" }
+    - { name: "timestamper", version: "1.5.7" }
+
+jenkins_debian_pkgs:
+    - openjdk-7-jdk
+    - nginx
+    - git
+    - maven

playbooks/roles/jenkins_master/handlers/main.yml

+---
+- name: jenkins_master | restart Jenkins
+  service: name=jenkins state=restarted
+
+- name: jenkins_master | start nginx
+  service: name=nginx state=started
+
+- name: jenkins_master | reload nginx
+  service: name=nginx state=reloaded

playbooks/roles/jenkins_master/tasks/main.yml

+---
+
+- name: jenkins_master | install jenkins specific system packages
+  apt:
+    pkg={{','.join(jenkins_debian_pkgs)}}
+    state=present update_cache=yes
+  tags:
+  - jenkins
+
+- name: jenkins_master | Create jenkins group
+  group: name={{ jenkins_group }} state=present
+
+- name: jenkins_master | Add the jenkins user to the group
+  user: name={{ jenkins_user }} append=yes groups={{ jenkins_group }}
+
+- name: jenkins_master | Install Jenkins apt key
+  apt_key: url=http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key state=present
+
+- name: jenkins_master | Add apt repository
+  apt_repository: repo='deb http://pkg.jenkins-ci.org/debian binary/' state=present
+
+- name: jenkins_master | Install jenkins package
+  apt: pkg=jenkins state=present update_cache=yes
+
+- name: jenkins_master | Make plugins directory
+  sudo_user: jenkins
+  shell: mkdir -p {{ jenkins_home }}/plugins
+
+- name: jenkins_master | Install Jenkins plugins
+  sudo_user: jenkins
+  get_url: url=http://updates.jenkins-ci.org/download/plugins/${item.name}/${item.version}/${item.name}.hpi
+           dest={{ jenkins_home }}/plugins/${item.name}.hpi
+  with_items: "{{ jenkins_plugins }}"
+  notify:
+    - jenkins_master | restart Jenkins
+
+# We had to fork the git-client plugin repo
+# to fix an obscure bug.  If and when the PR gets
+# merged into the main repo and released,
+# we can use the regular plugin install process.
+# Until then, we need to compile and install it ourselves.
+- name: jenkins_master | Checkout custom git-client plugin repo
+  git: repo=https://github.com/edx/git-client-plugin.git
+       dest=/tmp/git-client-plugin
+
+- name: jenkins_master | Compile custom git-client plugin
+  command: mvn install chdir=/tmp/git-client-plugin
+           creates=/tmp/git-client-plugin/target/git-client.hpi
+
+- name: jenkins_master | Install custom git-client
+  command: mv /tmp/git-client-plugin/target/git-client.hpi
+           {{ jenkins_home }}/plugins/git-client.hpi
+           creates={{ jenkins_home }}/plugins/git-client.hpi
+  notify:
+    - jenkins_master | restart Jenkins
+
+- name: jenkins_master | Set git-client plugin permissions
+  file: path={{ jenkins_home }}/plugins/git-client.hpi
+        owner={{ jenkins_user }} group={{ jenkins_group }}
+        mode=700
+
+- name: jenkins_master | Setup nginix vhost
+  template:
+    src=etc/nginx/sites-available/jenkins.j2
+    dest=/etc/nginx/sites-available/jenkins
+
+- name: jenkins_master | enable jenkins vhost
+  file:
+    src=/etc/nginx/sites-available/jenkins
+    dest=/etc/nginx/sites-enabled/jenkins
+    state=link
+  notify: jenkins_master | start nginx

playbooks/roles/jenkins_master/templates/etc/nginx/sites-available/jenkins.j2

+server {
+  listen 80;
+  server_name {{ jenkins_server_name }};
+
+  location / {
+    proxy_pass              http://localhost:{{ jenkins_port }};
+
+    # Rewrite HTTPS requests from WAN to HTTP requests on LAN
+    proxy_redirect http:// https://;
+
+    # The following settings from https://wiki.jenkins-ci.org/display/JENKINS/Running+Hudson+behind+Nginx
+    sendfile off;
+
+    proxy_set_header        Host $host;
+    proxy_set_header        X-Real-IP $remote_addr;
+    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_connect_timeout   150;
+    proxy_send_timeout      100;
+    proxy_read_timeout      100;
+    proxy_buffers           4 32k;
+    client_max_body_size    8m;
+    client_body_buffer_size 128k;
+
+  }
+}
\ No newline at end of file

playbooks/roles/jenkins_worker/defaults/main.yml

+---
+jenkins_workspace: /mnt/jenkins
+jenkins_phantomjs_url: https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2
+jenkins_phantomjs_archive: phantomjs-1.9.1-linux-x86_64.tar.bz2
+jenkins_phantomjs_folder: phantomjs-1.9.1-linux-x86_64
+jenkins_user: "jenkins"
+jenkins_group: "jenkins"
+jenkins_user_home: /home/jenkins
+
+# System packages
+jenkins_debian_pkgs:
+    - ack-grep
+    - build-essential
+    - git
+    - gfortran
+    - graphviz
+    - lynx-cur
+    - libgraphviz-dev
+    - libopenblas-dev
+    - liblapack-dev
+    - libxml2-dev
+    - libgeos-dev
+    - libmysqlclient-dev
+    - libxslt1-dev
+    - mongodb
+    - npm
+    - pkg-config
+    - python2.7
+    - python-pip
+    - python2.7-dev
+    - unzip
+    - xml-twig-tools
+
+# Ruby Specific Vars
+rbenv_root: "{{ jenkins_user_home }}/.rbenv"
+rbenv_repo: "https://github.com/sstephenson/rbenv.git"
+ruby_build_repo: "https://github.com/sstephenson/ruby-build.git"
+ruby_version: "1.9.3-p374"
+
+# JSCover direct download URL
+jscover_url: "http://superb-dca2.dl.sourceforge.net/project/jscover/JSCover-1.0.2.zip"
+jscover_version: "1.0.2"
+
+# Mongo config
+mongo_dir: "/mnt/mongodb"
+mongo_log_dir: "/mnt/logs/mongodb"
+
+# URL of S3 bucket containing pre-compiled Python packages
+python_pkg_url: "https://s3.amazonaws.com/jenkins.python_pkgs"
+python_download_dir: "/mnt/python_pkgs"
+python_virtualenv: "/mnt/venv"

playbooks/roles/jenkins_worker/files/download_python_pkgs.sh

+#!/usr/bin/env bash
+set -e
+
+#####################################################
+#
+# download_python_pkgs.sh
+#
+# Use download .egg packages from an S3 bucket 
+#
+# Usage:
+#
+#    download_python_pkgs.sh S3_URL SAVE_DIR
+# 
+# where `S3_URL` is the URL of an S3 bucket
+# containing .egg files
+#
+# and `SAVE_DIR` is the directory in which to save 
+# the .egg files.
+#
+######################################################
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 S3_URL SAVE_DIR"
+    exit 1
+fi
+
+S3_URL=$1
+SAVE_DIR=$2
+
+# Create the save directory if it doesn't already exist
+mkdir -p $SAVE_DIR
+
+# Retrieve the list of files in the bucket
+echo "Downloading Python packages from S3..."
+curl $S3_URL | xml_grep 'Key' --text_only > $SAVE_DIR/python_pkgs.txt
+
+# Install each package into the virtualenv
+# If an error occurs, print stderr but do not abort
+echo "Installing Python packages..."
+while read package; do
+    curl $S3_URL/$package > $SAVE_DIR/$package || echo "Could not download $package"
+done < $SAVE_DIR/python_pkgs.txt

playbooks/roles/jenkins_worker/files/install_python_pkgs.sh

+#!/usr/bin/env bash
+set -e
+
+#####################################################
+#
+# install_python_pkgs.sh
+#
+# Use easy_install to install all
+# .egg files in a folder into a virtualenv.
+#
+# Usage:
+#
+#    install_python_pkgs.sh EGG_DIR VENV
+# 
+# where `EGG_DIR` is the directory containing
+# the .egg files
+#
+# and `VENV` is the virtualenv in which to install
+# the packages.  If the virtualenv does not yet
+# exist, it will be created.
+#
+# If the virtualenv has already been created
+# and the packages installed, then the script
+# will skip installation.
+#
+######################################################
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 EGG_DIR VENV"
+    exit 1
+fi
+
+EGG_DIR=$1
+VENV=$2
+
+if [ -e $VENV/install_finished ]; then
+    echo "$VENV already exists; skipping installation..."
+else
+
+    # Create python egg cache and set correct permissions
+    PYTHON_EGG_CACHE=$HOME/.python-eggs
+    mkdir -p $PYTHON_EGG_CACHE
+    chmod 700 -R $PYTHON_EGG_CACHE
+
+    # Create and activate the virtualenv
+    echo "No virtualenv found; creating it..."
+    mkdir -p $VENV
+    virtualenv $VENV
+    . $VENV/bin/activate
+
+    # Install the .egg files into the virtualenv
+    echo "Installing Python eggs..."
+    for egg_file in $EGG_DIR/*.egg; do
+        easy_install $egg_file || true
+    done
+
+    # Create indicator that we finished successfully.
+    # If we were interrupted (maybe the job was aborted),
+    # then this file won't be created, so the next
+    # job will retry the intallation (instead of skipping it).
+    touch $VENV/install_finished
+fi

playbooks/roles/jenkins/tasks/browsers.yml → playbooks/roles/jenkins_worker/tasks/browsers.yml

 ---
-- name: browsers | Install Firefox
+- name: jenkins_worker | Install Firefox
   apt: pkg=firefox
 
-- name: browsers | Download PhantomJS
+- name: jenkins_worker | Install dbus-x11 (FF requirement)
+  apt: pkg=dbus-x11
+
+- name: jenkins_worker | Download PhantomJS
   get_url: url={{ jenkins_phantomjs_url }}
            dest=/var/tmp/{{ jenkins_phantomjs_archive }}
 
-- name: browsers | Untar PhantomJS
+- name: jenkins_worker | Untar PhantomJS
   command: tar -xjf /var/tmp/{{ jenkins_phantomjs_archive }} -C /var/tmp/
            creates=/var/tmp/{{ jenkins_phantomjs_folder }}
 
-- name: browsers | Install PhantomJS
+- name: jenkins_worker | Install PhantomJS
   command: mv /var/tmp/{{ jenkins_phantomjs_folder }} /usr/local/bin/phantomjs
            creates=/usr/local/bin/phantomjs
 
-- name: browsers | Set PhantomJS permissions
+- name: jenkins_worker | Set PhantomJS permissions
   file: path=/usr/local/bin/phantomjs mode=0755 state=directory
 
-- name: browsers | Install chrome dependencies
+- name: jenkins_worker | Install Chrome dependencies
   apt: pkg=libgconf2-4,libxss1,libnss3-1d,libcurl3,xdg-utils
 
-- name: browsers | Install unzip O_o
-  apt: pkg=unzip
-
-- name: browsers | Install Google Chrome
+- name: jenkins_worker | Install Google Chrome
   get_url: url=https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb dest=/var/tmp/google-chrome-stable_current_amd64.deb
 
-- name: browsers | Install Google Chrome 2
+- name: jenkins_worker | Install Google Chrome 2
   command: dpkg -i google-chrome-stable_current_amd64.deb
            chdir=/var/tmp
   ignore_errors: True
 
-- name: browsers | Install ChromeDriver
+- name: jenkins_worker | Install ChromeDriver
   get_url: url=https://chromedriver.googlecode.com/files/chromedriver_linux64_2.1.zip dest=/var/tmp/chromedriver_linux64_2.1.zip
 
-- name: browsers | Install ChromeDriver 2
+- name: jenkins_worker | Install ChromeDriver 2
   shell: unzip /var/tmp/chromedriver_linux64_2.1.zip
          chdir=/var/tmp
 
-- name: browsers | Install ChromeDriver 3
+- name: jenkins_worker | Install ChromeDriver 3
   shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
 
-- name: browsers | Install Chromedriver 4
+- name: jenkins_worker | Install Chromedriver 4
   file: path=/usr/local/bin/chromedriver mode=0755
-

playbooks/roles/jenkins_worker/tasks/jscover.yml

+---
+- name: jenkins_worker | Install Java
+  apt: pkg=openjdk-7-jre-headless state=present
+
+- name: jenkins_worker | Download JSCover
+  get_url: url={{ jscover_url }} dest=/var/tmp/jscover.zip
+
+- name: jenkins_worker | Unzip JSCover
+  shell: unzip /var/tmp/jscover.zip -d /var/tmp/jscover
+         creates=/var/tmp/jscover
+
+- name: jenkins_worker | Install JSCover JAR
+  command: cp /var/tmp/jscover/target/dist/JSCover-all.jar /usr/local/bin/JSCover-all-{{ jscover_version }}.jar
+           creates=/usr/local/bin/JSCover-all-{{ jscover_version }}.jar
+
+- name: jenkins_worker | Set JSCover permissions
+  file: path="/usr/local/bin/JSCover-all-{{ jscover_version }}.jar" state=file
+        owner=root group=root mode=0755

playbooks/roles/jenkins_worker/tasks/main.yml

+---
+# jenkins
+#
+# Provision a Jenkins instance.
+#
+# Parameters:
+#   `jenkins_user`: jenkins
+#   `jenkins_home`: /var/lib/jenkins
+#   `jenkins_user_home`: /home/jenkins
+
+- include: system.yml
+- include: python.yml
+- include: ruby.yml
+- include: mongo.yml
+- include: xvfb.yml
+- include: browsers.yml
+- include: jscover.yml

playbooks/roles/jenkins_worker/tasks/mongo.yml

+---
+
+# Configure Mongo to use /mnt so we don't
+# run out of disk space
+- name: jenkins_worker | Stop mongo service
+  service: name=mongodb state=stopped
+
+- name: jenkins_worker | Configure Mongo
+  template: src=mongodb_conf.j2
+    dest=/etc/mongodb.conf
+    owner=root
+    group=root
+
+- name: jenkins_worker | Configure Mongo upstart script
+  template: src=mongodb_upstart.j2
+    dest=/etc/init/mongodb.conf
+    owner=root
+    group=root
+
+- name: jenkins_worker | Start the mongo service
+  service: name=mongodb state=stopped

playbooks/roles/jenkins_worker/tasks/python.yml

+---
+
+- name: jenkins_worker | Install virtualenv
+  pip: name=virtualenv state=present
+
+- name: jenkins_worker | Install virtualenv wrapper
+  pip: name=virtualenvwrapper state=present
+
+- name: jenkins_worker | Install requests
+  pip: name=requests state=present
+
+# Install bash scripts
+- name: jenkins_worker | Install Python packages scripts
+  copy: src="${item}" dest="/usr/local/bin/${item}"
+        force=yes
+        owner=root group=root
+        mode=755
+  with_items:
+    - download_python_pkgs.sh
+    - install_python_pkgs.sh
+
+# Install scripts requiring a GitHub OAuth token
+- fail: jenkins_worker | OAuth token not defined
+  when: github_oauth_token is not defined
+
+- name: jenkins_worker | Install Python GitHub PR auth script
+  template: src="github_pr_auth.py.j2" dest="/usr/local/bin/github_pr_auth.py"
+            owner=root group=root
+            mode=755
+
+- name: jenkins_worker | Install Python GitHub post status script
+  template: src="github_post_status.py.j2" dest="/usr/local/bin/github_post_status.py"
+            owner=root group=root
+            mode=755
+
+# Install upstart script to download Python packages from S3
+- name: jenkins_worker | Install Python packages upstart script
+  template: src="python_pkgs.conf.j2" dest="/etc/init/python_pkgs.conf"

playbooks/roles/jenkins_worker/tasks/ruby.yml

+---
+
+- name: jenkins_worker | Install rbenv
+  git: repo={{ rbenv_repo }} dest={{ rbenv_root }}
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Install ruby-build
+  git: repo={{ ruby_build_repo }} dest={{ rbenv_root }}/plugins/ruby-build
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Install ruby
+  command: "{{ rbenv_root }}/bin/rbenv install {{ ruby_version }}
+           creates={{ rbenv_root }}/versions/{{ ruby_version }}"
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Set global ruby
+  command: "{{ rbenv_root }}/bin/rbenv global {{ ruby_version }}"
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Install bundler
+  command: "{{ rbenv_root }}/shims/gem install bundler"
+  sudo_user: "{{ jenkins_user }}"
+
+- name: jenkins_worker | Rbenv rehash
+  command: "{{ rbenv_root }}/bin/rbenv rehash"
+  sudo_user: "{{ jenkins_user }}"

playbooks/roles/jenkins_worker/tasks/system.yml

+---
+- name: jenkins_worker | Create jenkins group
+  group: name={{ jenkins_group }} state=present
+
+- name: jenkins_worker | Add the jenkins user to the group
+  user: name={{ jenkins_user }} append=yes group={{ jenkins_group }}
+
+
+# We need the upstart script to create the build directory
+# so that (a) it will be run when a new instance is created
+# on the current EBS, and (b) it will be run as root.
+- name: jenkins_worker | Install upstart script to create build dir
+  template: src=jenkins_workspace.conf.j2
+    dest=/etc/init/jenkins_workspace.conf
+    owner=root group=root
+
+
+# Because of a bug in the latest release of the EC2 plugin
+# we need to use a key generated by Amazon (not imported)
+# To satisfy this, we allow users to log in as Jenkins
+# using the same keypair the instance was started with.
+- name: jenkins_worker | Create .ssh directory
+  file: path={{ jenkins_user_home }}/.ssh state=directory
+        owner={{ jenkins_user }} group={{ jenkins_group }}
+
+- name: jenkins_worker | Copy ssh keys for jenkins
+  command: cp /home/ubuntu/.ssh/authorized_keys /home/{{ jenkins_user }}/.ssh/authorized_keys
+
+- name: jenkins_worker | Set key permissions
+  file: path={{ jenkins_user_home }}/.ssh/authorized_keys
+        owner={{ jenkins_user }} group={{ jenkins_group }}
+        mode=400
+
+- name: jenkins_worker | Install system packages
+  apt: pkg={{','.join(jenkins_debian_pkgs)}}
+       state=present update_cache=yes
+
+- name: jenkins_worker | Add script to set up environment variables
+  template: src=jenkins_env.sh.j2 dest=/usr/local/bin/jenkins_env.sh
+            owner=root group=root mode=0555

playbooks/roles/jenkins/tasks/xvfb.yml → playbooks/roles/jenkins_worker/tasks/xvfb.yml

 ---
-- name: xvfb | install xvfb
+- name: jenkins_worker | install xvfb
   apt: pkg=xvfb state=present
 
-- name: xvfb | configure display
-  template: src=xvfb.sh.j2 dest=/etc/profile.d/xvfb.sh owner=root group=root mode=0755
-
-- name: xvfb | create xvfb upstart script
+- name: jenkins_worker | create xvfb upstart script
   template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
 
-- name: xvfb | start xvfb
+- name: jenkins_worker | start xvfb
   shell: start xvfb
-  ignore_errors: yes
+  ignore_errors: yes
\ No newline at end of file

playbooks/roles/jenkins_worker/templates/github_post_status.py.j2

+#!/usr/bin/env python
+
+"""
+Update the status of a GitHub commit.
+"""
+
+import sys
+import requests
+import json
+from textwrap import dedent
+
+# The Ansible script will fill in the GitHub OAuth token.
+# That way, we can give the jenkins user on the worker
+# execute-only access to this script, ensuring that
+# the jenkins user cannot retrieve the token.
+GITHUB_OAUTH_TOKEN = "{{ github_oauth_token }}"
+
+USAGE = "Usage: {0} ORG REPO SHA STATUS TARGET_URL DESCRIPTION"
+
+VALID_STATUS_LIST = ['pending', 'success', 'error', 'failure']
+
+
+def parse_args(arg_list):
+    """
+    Parse the list of arguments, returning a dict.
+    Prints an error message and exits if the arguments are invalid.
+    """
+    if len(arg_list) != 7:
+        print USAGE.format(arg_list[0])
+        exit(1)
+
+    # Check that the build status is valid
+    status = arg_list[4]
+    if not status in VALID_STATUS_LIST:
+        print "Invalid status: must be one of {0}".format(", ".join(VALID_STATUS_LIST))
+        exit(1)
+
+    return {
+        'org': arg_list[1],
+        'repo': arg_list[2],
+        'sha': arg_list[3],
+        'status': arg_list[4],
+        'target_url': arg_list[5],
+        'description': arg_list[6]
+    }
+
+
+def post_status(org, repo, sha, status, target_url, description):
+    """
+    Post a new status to GitHub.
+    See http://developer.github.com/v3/repos/statuses/ for details.
+
+    Prints an error message and exits if unsuccessful.
+    """
+    url = "https://api.github.com/repos/{0}/{1}/statuses/{2}?access_token={3}".format(
+        org, repo, sha, GITHUB_OAUTH_TOKEN
+    )
+
+    params = {
+        'state': status,
+        'target_url': target_url,
+        'description': description
+    }
+
+    response = requests.post(url, data=json.dumps(params))
+
+    if response.status_code != 201:
+        print dedent("""
+            Could not post status:
+            HTTP response code is {0}
+            Content: {1}
+        """).format(response.status_code, response.text).strip()
+        exit(1)
+
+
+def main():
+    """
+    Post the status to GitHub.
+    """
+    if not GITHUB_OAUTH_TOKEN:
+        print "No GitHub Oauth token configured."
+        exit(1)
+
+    arg_dict = parse_args(sys.argv)
+    post_status(
+        arg_dict['org'], arg_dict['repo'],
+        arg_dict['sha'], arg_dict['status'],
+        arg_dict['target_url'], arg_dict['description']
+    )
+
+
+if __name__ == "__main__":
+    main()

playbooks/roles/jenkins_worker/templates/github_pr_auth.py.j2

+#!/usr/bin/env python
+
+"""
+Determine whether we allow a GitHub PR to be
+built automatically.  Checks a whitelist
+of repo owners and compares to the HEAD
+repo of the pull request.
+
+Uses an environment variable `GITHUB_OWNER_WHITELIST`
+to check whether the owner of the PR repo is whitelisted.
+This is a comma-separated list of organizations and
+users.  For example, a bash script might define:
+
+    export GITHUB_OWNER_WHITELIST="edx,a_user,another_user"
+
+to allow PRs from repos owned by "edx", "a_usr", and "another_user"
+"""
+
+import sys
+import os
+import requests
+from textwrap import dedent
+
+# The Ansible script will fill in the GitHub OAuth token.
+# That way, we can give the jenkins user on the worker
+# execute-only access to this script, ensuring that
+# the jenkins user cannot retrieve the token.
+GITHUB_OAUTH_TOKEN = "{{ github_oauth_token }}"
+
+USAGE = "Usage: {0} ORG REPO PULL_REQUEST_NUM"
+
+
+def parse_args(arg_list):
+    """
+    Parse the list of arguments, returning a dict of the form
+
+        {
+            'org': GITHUB_ORG,
+            'repo': GITHUB_REPO,
+            'pr_num': GITHUB_PR_NUM
+        }
+
+    Prints an error message and exits if the arguments are invalid.
+    """
+    if len(arg_list) != 4:
+        print USAGE.format(arg_list[0])
+        exit(1)
+
+    # Retrieve the PR number and check that it's an integer
+    try:
+        pr_num = int(arg_list[3])
+    except TypeError:
+        print "'{0}' is not a number".format(arg_list[3])
+
+    return {
+        'org': arg_list[1],
+        'repo': arg_list[2],
+        'pr_num': pr_num
+    }
+
+
+def pr_repo_owner(org, repo, pr_num):
+    """
+    Return the name of the owner of the repo from the
+    HEAD of the PR.
+    """
+    # Query GitHub for information about the pull request
+    url = "https://api.github.com/repos/{0}/{1}/pulls/{2}?access_token={3}".format(
+        org, repo, pr_num, GITHUB_OAUTH_TOKEN
+    )
+    response = requests.get(url)
+
+    if response.status_code != 200:
+        print dedent("""
+        Could not retrieve info for pull request #{0}.
+        HTTP status code: {1}
+        """.format(pr_num, response.status_code)).strip()
+        exit(1)
+
+    # Parse the response as json
+    try:
+        pr_data = response.json()
+    except TypeError:
+        print "Could not parse info for pull request #{0}".format(pr_num)
+        exit(1)
+
+    # Retrieve the owner of the repo
+    try:
+        return pr_data['head']['repo']['owner']['login']
+
+    except KeyError:
+        print "Could not get repo owner from PR info"
+        exit(1)
+
+
+def main():
+    """
+    Exits with code 0 (success) if the PR is from a whitelisted
+    repo; otherwise, exits with status 1 (failure).
+    """
+    if not GITHUB_OAUTH_TOKEN:
+        print "No GitHub Oauth token configured."
+        exit(1)
+
+    arg_dict = parse_args(sys.argv)
+    owner = pr_repo_owner(arg_dict['org'], arg_dict['repo'], arg_dict['pr_num'])
+
+    # Check that the owner is whitelisted
+    whitelist_owners = os.environ.get('GITHUB_OWNER_WHITELIST', '').split(',')
+    if owner not in whitelist_owners:
+        print dedent("""
+            Owner '{0}' is not in the whitelist.
+            You can update the whitelist by setting the environment variable
+            `GITHUB_OWNER_WHITELIST` to a comma-separated list of organizations
+            and users.
+        """.format(owner)).strip()
+        exit(1)
+
+    else:
+        print "Owner '{0}' is authorized".format(owner)
+        exit(0)
+
+
+if __name__ == "__main__":
+    main()

playbooks/roles/jenkins_worker/templates/jenkins_env.sh.j2

+# Configure Ruby
+export RBENV_ROOT="{{ rbenv_root }}"
+export PATH=$RBENV_ROOT/bin/:$PATH
+eval "$(rbenv init -)"
+
+# Configure JavaScript coverage
+export JSCOVER_JAR=/usr/local/bin/JSCover-all.jar
+
+# Set the display to the virtual frame buffer (Xvfb)
+export DISPLAY=:1

playbooks/roles/jenkins_worker/templates/jenkins_workspace.conf.j2

+# Create directory for Jenkins jobs
+
+start on runlevel [2345]
+
+task
+script
+    mkdir -p {{ jenkins_workspace }}
+    chown {{ jenkins_user }}:{{ jenkins_group }} {{ jenkins_workspace }}
+    chmod 700 {{ jenkins_workspace }}
+end script

playbooks/roles/jenkins_worker/templates/mongodb_conf.j2

+# mongodb.conf
+
+dbpath={{ mongo_dir }}
+logpath={{ mongo_log_dir }}/mongodb.log
+
+logappend=true
+
+bind_ip = 127.0.0.1
+port = 27017
+
+journal=true

playbooks/roles/jenkins_worker/templates/mongodb_upstart.j2

+# Ubuntu upstart file at /etc/init/mongodb.conf
+
+pre-start script
+    mkdir -p {{ mongo_dir }}
+    mkdir -p {{ mongo_log_dir }}
+
+    touch {{ mongo_log_dir }}/mongodb.log
+
+    chown mongodb:nogroup -R {{ mongo_dir }}
+    chown mongodb:nogroup -R {{ mongo_log_dir }}
+end script
+
+start on runlevel [2345]
+stop on runlevel [06]
+
+script
+    ENABLE_MONGODB="yes"
+    if [ -f /etc/default/mongodb ]; then . /etc/default/mongodb; fi
+    if [ "x$ENABLE_MONGODB" = "xyes" ]; then exec start-stop-daemon --start --quiet --chuid mongodb --exec  /usr/bin/mongod -- --config /etc/mongodb.conf; fi
+end script

playbooks/roles/jenkins_worker/templates/python_pkgs.conf.j2

+# Pre-install Python packages from S3
+
+start on runlevel [2345]
+
+task
+script
+
+    # Create the directory to hold Python virtualenvs
+    mkdir -p {{ python_virtualenv }}
+
+    # Download .egg files from S3
+    download_python_pkgs.sh {{ python_pkg_url }} {{ python_download_dir }}
+
+    # Give the Jenkins user access
+    chown {{ jenkins_user }}:{{ jenkins_group }} -R {{ python_download_dir }}
+    chmod 500 -R {{ python_download_dir }}
+
+    chown {{ jenkins_user }}:{{ jenkins_group }} -R {{ python_virtualenv }}
+    chmod 700 -R {{ python_virtualenv }}
+
+end script

playbooks/util/github_oauth_token.py

+#!/usr/bin/env python
+
+"""
+Generate a GitHub OAuth token with a particular
+set of permissions.
+
+Usage:
+
+    github_oauth_token.py USERNAME PASSWORD [SCOPE ...]
+
+Example:
+
+    github_oauth_token.py jenkins_user repo:status public_repo
+
+This will prompt the user for the password.
+"""
+
+import sys
+import requests
+import json
+import getpass
+from textwrap import dedent
+
+USAGE = "Usage: {0} USERNAME NOTE [SCOPE ...]"
+
+
+def parse_args(arg_list):
+    """
+    Return a dict of the command line arguments.
+    Prints an error message and exits if the arguments are invalid.
+    """
+    if len(arg_list) < 4:
+        print USAGE.format(arg_list[0])
+        exit(1)
+
+    # Prompt for the password
+    password = getpass.getpass()
+
+    return {
+        'username': arg_list[1],
+        'password': password,
+        'note': arg_list[2],
+        'scopes': arg_list[3:],
+    }
+
+
+def get_oauth_token(username, password, scopes, note):
+    """
+    Create a GitHub OAuth token with the given scopes.
+    If unsuccessful, print an error message and exit.
+
+    Returns a tuple `(token, scopes)`
+    """
+    params = {'scopes': scopes, 'note': note}
+
+    response = response = requests.post(
+        'https://api.github.com/authorizations',
+        data=json.dumps(params),
+        auth=(username, password)
+    )
+
+    if response.status_code != 201:
+        print dedent("""
+            Could not create OAuth token.
+            HTTP status code: {0}
+            Content: {1}
+        """.format(response.status_code, response.text)).strip()
+        exit(1)
+
+    try:
+        token_data = response.json()
+        return token_data['token'], token_data['scopes']
+
+    except TypeError:
+        print "Could not parse response data."
+        exit(1)
+
+    except KeyError:
+        print "Could not retrieve data from response."
+        exit(1)
+
+
+def main():
+    arg_dict = parse_args(sys.argv)
+    token, scopes = get_oauth_token(
+        arg_dict['username'], arg_dict['password'],
+        arg_dict['scopes'], arg_dict['note']
+    )
+
+    print "Token: {0}".format(token)
+    print "Scopes: {0}".format(", ".join(scopes))
+
+
+if __name__ == "__main__":
+    main()