Refactor: sitespeed vs platform jenkins_worker should hinge on a var.

playbooks/edx-east/jenkins_worker_sitespeedio.yml

@@ -3,12 +3,13 @@
 # but not the Jenkins Java app, or edx-platform.
 
 - name: Configure instance(s)
-  hosts: jenkins_worker_sitespeedio
+  hosts: jenkins_worker
   sudo: True
   gather_facts: True
   vars:
     mongo_enable_journal: False
     serial_count: 1
+    sitespeed_worker: True
   serial: "{{ serial_count }}"
   vars_files:
     - roles/edxapp/defaults/main.yml
@@ -17,4 +18,4 @@
     - roles/xserver/defaults/main.yml
     - roles/forum/defaults/main.yml
   roles:
-    - jenkins_worker_sitespeed
+    - jenkins_worker

playbooks/roles/jenkins_worker/meta/main.yml

 ---
 dependencies:
   - common
-  - edxapp_common
-  - jscover
+  # dependencies for edx-app jenkins worker:
+  - role: edxapp_common
+    when: not sitespeed_worker
   - role: rbenv
     rbenv_user: "{{ jenkins_user }}"
     rbenv_dir: "{{ jenkins_home }}"
     rbenv_ruby_version: "{{ jenkins_ruby_version }}"
     jenkins_worker: true
+    when: not sitespeed_worker
+
+  # dependencies for sitespeed worker
+  - role: sitespeedio
+    when: sitespeed_worker
+
+  # dependencies for generic worker
+  - role: jscover
+    when: not sitespeed_worker

playbooks/roles/jenkins_worker/tasks/main.yml

@@ -12,5 +12,14 @@
 - include: system.yml
 - include: python.yml
 - include: ruby.yml
+  when: not sitespeed_worker
 - include: java.yml
+- include: python_platform_worker.yml
+  when: not sitespeed_worker
+
+# Run appropriate tests
 - include: test.yml
+- include: test_platform_worker.yml
+  when: not sitespeed_worker
+- include: test_sitespeed_worker.yml
+  when: sitespeed_worker

playbooks/roles/jenkins_worker/tasks/python.yml

 ---
 
-# Install scripts requiring a GitHub OAuth token
+# Requests library is required for both the github status
+# script, as well as the sitespeed cookie script.
 - name: Install requests Python library
   pip: name=requests state=present
 
+# Install scripts requiring a GitHub OAuth token
 - fail: OAuth token not defined
   when: github_oauth_token is not defined
 
@@ -11,82 +13,3 @@
   template: src="github_post_status.py.j2" dest="/usr/local/bin/github_post_status.py"
             owner=root group=root
             mode=755
-
-# Create a virtualenv for edx-platform by installing the requirements
-# and packaging the virtualenv.
-# A shallow clone is created off of master. The depth setting
-# refers to the --depth-setting of git clone. A value of 1
-# will truncate all history prior to the last revision.
-- name: Create shallow clone of edx-platform
-  git: >
-    repo=https://github.com/edx/edx-platform.git
-    dest={{ jenkins_home }}/shallow-clone
-    version=master
-    depth=1
-  sudo_user: "{{ jenkins_user }}"
-
-# pip-accel skipped due to conflicting versions of pip required
-# by the pip-accel package and edx-platform
-# - name: Pip installs that are needed for pip-accel to work for us
-#   pip: >
-#     name="{{ item }}"
-#     virtualenv={{ jenkins_home }}/edx-venv
-#     virtualenv_command=virtualenv-2.7
-#   sudo_user: "{{ jenkins_user }}"
-#   with_items: pip_accel_reqs
-
-# Install the platform requirements using pip.
-# Installing the platform requirements using pip-accel
-# would allow the binary distributions to be downloaded from S3
-# rather than compiled each time. This was previously enabled,
-# but reverted back to pip because the current version of pip-accel
-# (0.22.4) is only compatible with pip >= 1.4, < 1.5 and the current
-# version of pip in edx-platform is 6.0.8.
-- name: Install edx-platform requirements using pip
-  pip: >
-    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
-    extra_args="--exists-action=w"
-    virtualenv={{ jenkins_home }}/edx-venv
-    virtualenv_command=virtualenv-2.7
-    executable=pip
-  with_items:
-    - pre.txt
-    - github.txt
-    - base.txt
-    - paver.txt
-  sudo_user: "{{ jenkins_user }}"
-
-
-# These and the pip install below it can be removed some time after
-# https://github.com/edx/edx-platform/pull/7465 has merged.
-# This will keep us from breaking after that gets merged.
-- stat: path="{{ jenkins_home }}/shallow-clone/requirements/edx/post.txt"
-  register: post_requirements
-  sudo_user: "{{ jenkins_user }}"
-
-- name: Install edx-platform post requirements using pip
-  pip: >
-    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
-    extra_args="--exists-action=w"
-    virtualenv={{ jenkins_home }}/edx-venv
-    virtualenv_command=virtualenv-2.7
-    executable=pip
-  with_items:
-    - post.txt
-  sudo_user: "{{ jenkins_user }}"
-  when: post_requirements.stat.exists
-
-# Archive the current state of the virtualenv
-# as a starting point for new builds.
-# The edx-venv directory is deleted and then recreated
-# cleanly from the archive by the jenkins build scripts.
-- name: Create a clean virtualenv archive
-  command: >
-    tar -cpzf edx-venv_clean.tar.gz edx-venv
-    chdir={{ jenkins_home }}
-  sudo_user: "{{ jenkins_user }}"
-
-# Remove the shallow-clone directory now that we archive
-# done with it
-- name: Remove shallow-clone
-  file: path={{ jenkins_home }}/shallow-clone state=absent

playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml

+---
+
+# Install scripts requiring a GitHub OAuth token
+- name: Install requests Python library
+  pip: name=requests state=present
+
+- fail: OAuth token not defined
+  when: github_oauth_token is not defined
+
+- name: Install Python GitHub post status script
+  template: src="github_post_status.py.j2" dest="/usr/local/bin/github_post_status.py"
+            owner=root group=root
+            mode=755
+
+# Create a virtualenv for edx-platform by installing the requirements
+# and packaging the virtualenv.
+# A shallow clone is created off of master. The depth setting
+# refers to the --depth-setting of git clone. A value of 1
+# will truncate all history prior to the last revision.
+- name: Create shallow clone of edx-platform
+  git: >
+    repo=https://github.com/edx/edx-platform.git
+    dest={{ jenkins_home }}/shallow-clone
+    version=master
+    depth=1
+  sudo_user: "{{ jenkins_user }}"
+
+# pip-accel skipped due to conflicting versions of pip required
+# by the pip-accel package and edx-platform
+# - name: Pip installs that are needed for pip-accel to work for us
+#   pip: >
+#     name="{{ item }}"
+#     virtualenv={{ jenkins_home }}/edx-venv
+#     virtualenv_command=virtualenv-2.7
+#   sudo_user: "{{ jenkins_user }}"
+#   with_items: pip_accel_reqs
+
+# Install the platform requirements using pip.
+# Installing the platform requirements using pip-accel
+# would allow the binary distributions to be downloaded from S3
+# rather than compiled each time. This was previously enabled,
+# but reverted back to pip because the current version of pip-accel
+# (0.22.4) is only compatible with pip >= 1.4, < 1.5 and the current
+# version of pip in edx-platform is 6.0.8.
+- name: Install edx-platform requirements using pip
+  pip: >
+    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
+    extra_args="--exists-action=w"
+    virtualenv={{ jenkins_home }}/edx-venv
+    virtualenv_command=virtualenv-2.7
+    executable=pip
+  with_items:
+    - pre.txt
+    - github.txt
+    - base.txt
+    - paver.txt
+  sudo_user: "{{ jenkins_user }}"
+
+
+# These and the pip install below it can be removed some time after
+# https://github.com/edx/edx-platform/pull/7465 has merged.
+# This will keep us from breaking after that gets merged.
+- stat: path="{{ jenkins_home }}/shallow-clone/requirements/edx/post.txt"
+  register: post_requirements
+  sudo_user: "{{ jenkins_user }}"
+
+- name: Install edx-platform post requirements using pip
+  pip: >
+    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
+    extra_args="--exists-action=w"
+    virtualenv={{ jenkins_home }}/edx-venv
+    virtualenv_command=virtualenv-2.7
+    executable=pip
+  with_items:
+    - post.txt
+  sudo_user: "{{ jenkins_user }}"
+  when: post_requirements.stat.exists
+
+# Archive the current state of the virtualenv
+# as a starting point for new builds.
+# The edx-venv directory is deleted and then recreated
+# cleanly from the archive by the jenkins build scripts.
+- name: Create a clean virtualenv archive
+  command: >
+    tar -cpzf edx-venv_clean.tar.gz edx-venv
+    chdir={{ jenkins_home }}
+  sudo_user: "{{ jenkins_user }}"
+
+# Remove the shallow-clone directory now that we archive
+# done with it
+- name: Remove shallow-clone
+  file: path={{ jenkins_home }}/shallow-clone state=absent

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -38,6 +38,7 @@
   template:
     src=jenkins_env.j2 dest={{ jenkins_home }}/jenkins_env
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0500
+  when: not sitespeed_worker
 
 # Need to add Github to known_hosts to avoid
 # being prompted when using git through ssh
@@ -62,8 +63,10 @@
 - name: Create directory for pip-accel config file
   file: path={{ jenkins_home }}/.pip-accel state=directory
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0777 recurse=yes
+  when: not sitespeed_worker
 
 - name: Create pip-accel config file
   template:
     src=pip-accel.conf.j2 dest={{ jenkins_home }}/.pip-accel/pip-accel.conf
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0664
+  when: not sitespeed_worker

playbooks/roles/jenkins_worker/tasks/test.yml

@@ -2,18 +2,7 @@
 # Tests for this role
 
 
-# Set up #
-
-# To get a baseline comparison for timestamp comparisons
-# create a testfile and register its stat info
-- name: Create test file
-  file: path=testfile state=touch
-- name: Stat test file
-  stat: path=testfile
-  register: testfile
-
-
-# Tests #
+### Tests ###
 - name: Verify java cmd is using v 1.7
   shell: java -version
   register: java_version
@@ -29,16 +18,3 @@
     "github_post_status.py edx edx-platform
     dddac0b5dddf00c0950daf324e603e4935994954 success
     https://jenkins.testeng.edx.org/ \"Tests Passed\" \"CI Test Results\""
-
-# Verify the virtualenv tar is newly-built
-- name: Get info on virtualenv tar
-  stat: path={{ jenkins_home }}/edx-venv_clean.tar.gz
-  register: edxvenv
-- assert:
-    that:
-# Assert that it was modified at least within the hour
-      - "{{ testfile.stat.mtime }} - {{ edxvenv.stat.mtime }} < 3600"
-
-# Tear Down #
-- name: Remove test file
-  file: path=testfile state=absent

playbooks/roles/jenkins_worker_sitespeed/tasks/test.yml → playbooks/roles/jenkins_worker/tasks/test_platform_worker.yml

@@ -13,13 +13,26 @@
   register: testfile
 
 
-# Tests #
-- name: Verify java cmd is using v 1.7
-  shell: java -version
-  register: java_version
+### Tests ###
+
+# Firefox has a specific version, not the latest. This test also ensures it was not
+# pulled in via dependency or misuse/clobbering due to the sitespeed variable, which uses
+# the latest firefox.
+- name: Verify firefox version
+  shell: firefox --version
+  register: firefox_version
+- assert:
+    that:
+      - "'28.0' in firefox_version"
+
+# Verify the virtualenv tar is newly-built
+- name: Get info on virtualenv tar
+  stat: path={{ jenkins_home }}/edx-venv_clean.tar.gz
+  register: edxvenv
 - assert:
     that:
-      - "'1.7.0' in java_version.stderr"
+# Assert that it was modified at least within the hour
+      - "{{ testfile.stat.mtime }} - {{ edxvenv.stat.mtime }} < 3600"
 
 # Tear Down #
 - name: Remove test file

playbooks/roles/jenkins_worker/tasks/test_sitespeed_worker.yml

+---
+# Tests for this role
+
+
+### Tests ###
+
+# Sitespeed workers should have the latest version of firefox
+# Lite test. Ensures we are not using
+# the version of firefox specified in a different file.
+- name: Verify firefox version
+  shell: firefox --version
+  register: firefox_version
+- assert:
+    that:
+      - "'40.0' not in firefox_version"

playbooks/roles/jenkins_worker_sitespeed/defaults/main.yml

----
-jenkins_user: "jenkins"
-jenkins_group: "jenkins"
-jenkins_home: /home/jenkins
-
-# repo for nodejs
-jenkins_chrislea_ppa: "ppa:chris-lea/node.js"
-
-# System packages
-jenkins_debian_pkgs:
-    - nodejs
-    - pkg-config
-
-# packer direct download URL
-packer_url: "https://dl.bintray.com/mitchellh/packer/0.6.1_linux_amd64.zip"

playbooks/roles/jenkins_worker_sitespeed/meta/main.yml

----
-dependencies:
-  - common
-  - role: sitespeedio

playbooks/roles/jenkins_worker_sitespeed/tasks/java.yml

----
-- name: Install Java
-  apt: pkg=openjdk-7-jre-headless state=present

playbooks/roles/jenkins_worker_sitespeed/tasks/main.yml

----
-# jenkins
-#
-# Provision a Jenkins instance.
-#
-# Parameters:
-#   `jenkins_user`: jenkins
-#   `jenkins_home`: /var/lib/jenkins
-#   `jenkins_user_home`: /home/jenkins
-
-- include: packer.yml
-- include: system.yml
-- include: python.yml
-- include: java.yml
-- include: test.yml

playbooks/roles/jenkins_worker_sitespeed/tasks/packer.yml

----
-- name: Download packer
-  shell: "curl -L {{ packer_url }} -o /var/tmp/packer.zip"
-  args:
-      creates: /var/tmp/packer.zip
-
-- name: Unzip packer
-  unarchive: src=/var/tmp/packer.zip dest=/usr/local/bin copy=no

playbooks/roles/jenkins_worker_sitespeed/tasks/python.yml

----
-
-# Install requests library so we can use the
-# edx-sitespeed session cookie saver.
-- name: Install requests Python library
-  pip: name=requests state=present

playbooks/roles/jenkins_worker_sitespeed/tasks/system.yml

----
-- name: Create jenkins group
-  group: name={{ jenkins_group }} state=present
-
-# The Jenkins account needs a login shell because Jenkins uses scp
-- name: Add the jenkins user to the group and configure shell
-  user: name={{ jenkins_user }} append=yes group={{ jenkins_group }} shell=/bin/bash
-
-# Because of a bug in the latest release of the EC2 plugin
-# we need to use a key generated by Amazon (not imported)
-# To satisfy this, we allow users to log in as Jenkins
-# using the same keypair the instance was started with.
-- name: Create .ssh directory
-  file:
-    path={{ jenkins_home }}/.ssh state=directory
-    owner={{ jenkins_user }} group={{ jenkins_group }}
-  ignore_errors: yes
-
-- name: Copy ssh keys for jenkins
-  command: cp /home/ubuntu/.ssh/authorized_keys /home/{{ jenkins_user }}/.ssh/authorized_keys
-  ignore_errors: yes
-
-- name: Set key permissions
-  file:
-    path={{ jenkins_home }}/.ssh/authorized_keys
-    owner={{ jenkins_user }} group={{ jenkins_group }} mode=400
-  ignore_errors: yes
-
-# adding chris-lea nodejs repo
-- name: add ppas for current versions of nodejs
-  apt_repository: repo="{{ jenkins_chrislea_ppa }}"
-
-- name: Install system packages
-  apt: pkg={{','.join(jenkins_debian_pkgs)}}
-       state=present update_cache=yes
-
-# Need to add Github to known_hosts to avoid
-# being prompted when using git through ssh
-- name: Add github.com to known_hosts if it does not exist
-  shell: >
-     ssh-keygen -f {{ jenkins_home }}/.ssh/known_hosts -H -F github.com | grep -q found || ssh-keyscan -H github.com > {{ jenkins_home }}/.ssh/known_hosts

playbooks/roles/jenkins_worker_sitespeed/templates/github_post_status.py.j2

-#!/usr/bin/env python
-
-"""
-Update the status of a GitHub commit.
-"""
-
-import sys
-import requests
-import json
-from textwrap import dedent
-
-# The Ansible script will fill in the GitHub OAuth token.
-# That way, we can give the jenkins user on the worker
-# execute-only access to this script, ensuring that
-# the jenkins user cannot retrieve the token.
-GITHUB_OAUTH_TOKEN = "{{ github_oauth_token }}"
-
-USAGE = "Usage: {0} ORG REPO SHA STATUS TARGET_URL DESCRIPTION CONTEXT"
-
-VALID_STATUS_LIST = ['pending', 'success', 'error', 'failure']
-
-
-def parse_args(arg_list):
-    """
-    Parse the list of arguments, returning a dict.
-    Prints an error message and exits if the arguments are invalid.
-    """
-    if len(arg_list) != 8:
-        print USAGE.format(arg_list[0])
-        exit(1)
-
-    # Check that the build status is valid
-    status = arg_list[4]
-    if not status in VALID_STATUS_LIST:
-        print "Invalid status: must be one of {0}".format(", ".join(VALID_STATUS_LIST))
-        exit(1)
-
-    return {
-        'org': arg_list[1],
-        'repo': arg_list[2],
-        'sha': arg_list[3],
-        'status': arg_list[4],
-        'target_url': arg_list[5],
-        'description': arg_list[6],
-        'context': arg_list[7],
-    }
-
-
-def post_status(org, repo, sha, status, target_url, description, context):
-    """
-    Post a new status to GitHub.
-    See http://developer.github.com/v3/repos/statuses/ for details.
-
-    Prints an error message and exits if unsuccessful.
-    """
-    url = "https://api.github.com/repos/{0}/{1}/statuses/{2}?access_token={3}".format(
-        org, repo, sha, GITHUB_OAUTH_TOKEN
-    )
-
-    params = {
-        'state': status,
-        'target_url': target_url,
-        'description': description,
-        'context': context,
-    }
-
-    response = requests.post(url, data=json.dumps(params))
-
-    if response.status_code != 201:
-        print dedent("""
-            Could not post status:
-            HTTP response code is {0}
-            Content: {1}
-        """).format(response.status_code, response.text).strip()
-        exit(1)
-
-
-def main():
-    """
-    Post the status to GitHub.
-    """
-    if not GITHUB_OAUTH_TOKEN:
-        print "No GitHub Oauth token configured."
-        exit(1)
-
-    arg_dict = parse_args(sys.argv)
-    post_status(
-        arg_dict['org'], arg_dict['repo'],
-        arg_dict['sha'], arg_dict['status'],
-        arg_dict['target_url'], arg_dict['description'],
-        arg_dict['context'],
-    )
-
-
-if __name__ == "__main__":
-    main()

util/packer/jenkins_worker_sitespeedio.json

@@ -2,6 +2,7 @@
   "variables": {
     "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
     "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
+    "github_oauth_token": "{{env `GITHUB_TOKEN`}}",
     "playbook_remote_dir": "/tmp/packer-edx-playbooks",
     "ami": "{{env `JENKINS_WORKER_AMI`}}"
   },
@@ -53,8 +54,8 @@
       "virtualenv packer-venv",
       ". packer-venv/bin/activate",
       "pip install -q -r requirements.txt",
-      "echo '[jenkins_worker_sitespeedio]' > inventory.ini",
+      "echo '[jenkins_worker]' > inventory.ini",
       "echo 'localhost' >> inventory.ini",
-      "ansible-playbook edx-east/jenkins_worker_sitespeedio.yml -i inventory.ini -c local -vvvv"]
+      "ansible-playbook edx-east/jenkins_worker_sitespeedio.yml -i inventory.ini -c local -e 'github_oauth_token={{ user `github_oauth_token` }}' -vvvv"]
   }]
 }