Merge pull request #2222 from edx/zoldak/jenkins-sitespeed

Zoldak/jenkins sitespeed

playbooks/edx-east/jenkins_worker.yml

@@ -9,6 +9,7 @@
   vars:
     mongo_enable_journal: False
     serial_count: 1
+    platform_worker: True
   serial: "{{ serial_count }}"
   vars_files:
     - roles/edxapp/defaults/main.yml

playbooks/edx-east/jenkins_worker_sitespeedio.yml

+# Configure a Jenkins worker instance
+# This has all the requirements to run sitespeedio,
+# but not the Jenkins Java app, or edx-platform.
+
+- name: Configure instance(s)
+  hosts: jenkins_worker
+  sudo: True
+  gather_facts: True
+  vars:
+    mongo_enable_journal: False
+    serial_count: 1
+    sitespeed_worker: True
+  serial: "{{ serial_count }}"
+  vars_files:
+    - roles/edxapp/defaults/main.yml
+    - roles/ora/defaults/main.yml
+    - roles/xqueue/defaults/main.yml
+    - roles/xserver/defaults/main.yml
+    - roles/forum/defaults/main.yml
+  roles:
+    - jenkins_worker

playbooks/roles/jenkins_worker/meta/main.yml

 ---
 dependencies:
   - common
-  - edxapp_common
   - jscover
+  # dependencies for edx-app jenkins worker:
+  - role: edxapp_common
+    when: platform_worker is defined
   - role: rbenv
     rbenv_user: "{{ jenkins_user }}"
     rbenv_dir: "{{ jenkins_home }}"
     rbenv_ruby_version: "{{ jenkins_ruby_version }}"
     jenkins_worker: true
+    when: platform_worker is defined
+
+  # dependencies for sitespeed worker
+  - role: sitespeedio
+    when: sitespeed_worker is defined

playbooks/roles/jenkins_worker/tasks/main.yml

 ---
 # jenkins
 #
-# Provision a Jenkins instance.
-#
-# Parameters:
-#   `jenkins_user`: jenkins
-#   `jenkins_home`: /var/lib/jenkins
-#   `jenkins_user_home`: /home/jenkins
+# Provision a Jenkins worker instance.
+# - When sitespeed_worker is set, only apply the configuraiton necessary for running sitespeed.io
+# - When platform_worker is set, the resulting instance can run edx-platform tests
 
+# All jenkins workers
 - include: packer.yml
 - include: system.yml
 - include: python.yml
-- include: ruby.yml
 - include: java.yml
+
+# only platform workers
+- include: ruby.yml
+  when: platform_worker is defined
+- include: python_platform_worker.yml
+  when: platform_worker is defined
+
+# Run appropriate tests
 - include: test.yml
+- include: test_platform_worker.yml
+  when: platform_worker is defined
+- include: test_sitespeed_worker.yml
+  when: sitespeed_worker is defined

playbooks/roles/jenkins_worker/tasks/python.yml

 ---
 
-# Install scripts requiring a GitHub OAuth token
+# Requests library is required for both the github status
+# script, as well as the sitespeed cookie script.
 - name: Install requests Python library
   pip: name=requests state=present
-
-- fail: OAuth token not defined
-  when: github_oauth_token is not defined
-
-- name: Install Python GitHub post status script
-  template: src="github_post_status.py.j2" dest="/usr/local/bin/github_post_status.py"
-            owner=root group=root
-            mode=755
-
-# Create a virtualenv for edx-platform by installing the requirements
-# and packaging the virtualenv.
-# A shallow clone is created off of master. The depth setting
-# refers to the --depth-setting of git clone. A value of 1
-# will truncate all history prior to the last revision.
-- name: Create shallow clone of edx-platform
-  git: >
-    repo=https://github.com/edx/edx-platform.git
-    dest={{ jenkins_home }}/shallow-clone
-    version=master
-    depth=1
-  sudo_user: "{{ jenkins_user }}"
-
-# pip-accel skipped due to conflicting versions of pip required
-# by the pip-accel package and edx-platform
-# - name: Pip installs that are needed for pip-accel to work for us
-#   pip: >
-#     name="{{ item }}"
-#     virtualenv={{ jenkins_home }}/edx-venv
-#     virtualenv_command=virtualenv-2.7
-#   sudo_user: "{{ jenkins_user }}"
-#   with_items: pip_accel_reqs
-
-# Install the platform requirements using pip.
-# Installing the platform requirements using pip-accel
-# would allow the binary distributions to be downloaded from S3
-# rather than compiled each time. This was previously enabled,
-# but reverted back to pip because the current version of pip-accel
-# (0.22.4) is only compatible with pip >= 1.4, < 1.5 and the current
-# version of pip in edx-platform is 6.0.8.
-- name: Install edx-platform requirements using pip
-  pip: >
-    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
-    extra_args="--exists-action=w"
-    virtualenv={{ jenkins_home }}/edx-venv
-    virtualenv_command=virtualenv-2.7
-    executable=pip
-  with_items:
-    - pre.txt
-    - github.txt
-    - base.txt
-    - paver.txt
-  sudo_user: "{{ jenkins_user }}"
-
-
-# These and the pip install below it can be removed some time after
-# https://github.com/edx/edx-platform/pull/7465 has merged.
-# This will keep us from breaking after that gets merged.
-- stat: path="{{ jenkins_home }}/shallow-clone/requirements/edx/post.txt"
-  register: post_requirements
-  sudo_user: "{{ jenkins_user }}"
-
-- name: Install edx-platform post requirements using pip
-  pip: >
-    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
-    extra_args="--exists-action=w"
-    virtualenv={{ jenkins_home }}/edx-venv
-    virtualenv_command=virtualenv-2.7
-    executable=pip
-  with_items:
-    - post.txt
-  sudo_user: "{{ jenkins_user }}"
-  when: post_requirements.stat.exists
-
-# Archive the current state of the virtualenv
-# as a starting point for new builds.
-# The edx-venv directory is deleted and then recreated
-# cleanly from the archive by the jenkins build scripts.
-- name: Create a clean virtualenv archive
-  command: >
-    tar -cpzf edx-venv_clean.tar.gz edx-venv
-    chdir={{ jenkins_home }}
-  sudo_user: "{{ jenkins_user }}"
-
-# Remove the shallow-clone directory now that we archive
-# done with it
-- name: Remove shallow-clone
-  file: path={{ jenkins_home }}/shallow-clone state=absent

playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml

+---
+
+# Install scripts requiring a GitHub OAuth token
+- name: Install requests Python library
+  pip: name=requests state=present
+
+- fail: OAuth token not defined
+  when: github_oauth_token is not defined
+
+- name: Install Python GitHub post status script
+  template: src="github_post_status.py.j2" dest="/usr/local/bin/github_post_status.py"
+            owner=root group=root
+            mode=755
+
+# Create a virtualenv for edx-platform by installing the requirements
+# and packaging the virtualenv.
+# A shallow clone is created off of master. The depth setting
+# refers to the --depth-setting of git clone. A value of 1
+# will truncate all history prior to the last revision.
+- name: Create shallow clone of edx-platform
+  git: >
+    repo=https://github.com/edx/edx-platform.git
+    dest={{ jenkins_home }}/shallow-clone
+    version=master
+    depth=1
+  sudo_user: "{{ jenkins_user }}"
+
+# pip-accel skipped due to conflicting versions of pip required
+# by the pip-accel package and edx-platform
+# - name: Pip installs that are needed for pip-accel to work for us
+#   pip: >
+#     name="{{ item }}"
+#     virtualenv={{ jenkins_home }}/edx-venv
+#     virtualenv_command=virtualenv-2.7
+#   sudo_user: "{{ jenkins_user }}"
+#   with_items: pip_accel_reqs
+
+# Install the platform requirements using pip.
+# Installing the platform requirements using pip-accel
+# would allow the binary distributions to be downloaded from S3
+# rather than compiled each time. This was previously enabled,
+# but reverted back to pip because the current version of pip-accel
+# (0.22.4) is only compatible with pip >= 1.4, < 1.5 and the current
+# version of pip in edx-platform is 6.0.8.
+- name: Install edx-platform requirements using pip
+  pip: >
+    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
+    extra_args="--exists-action=w"
+    virtualenv={{ jenkins_home }}/edx-venv
+    virtualenv_command=virtualenv-2.7
+    executable=pip
+  with_items:
+    - pre.txt
+    - github.txt
+    - base.txt
+    - paver.txt
+  sudo_user: "{{ jenkins_user }}"
+
+
+# These and the pip install below it can be removed some time after
+# https://github.com/edx/edx-platform/pull/7465 has merged.
+# This will keep us from breaking after that gets merged.
+- stat: path="{{ jenkins_home }}/shallow-clone/requirements/edx/post.txt"
+  register: post_requirements
+  sudo_user: "{{ jenkins_user }}"
+
+- name: Install edx-platform post requirements using pip
+  pip: >
+    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
+    extra_args="--exists-action=w"
+    virtualenv={{ jenkins_home }}/edx-venv
+    virtualenv_command=virtualenv-2.7
+    executable=pip
+  with_items:
+    - post.txt
+  sudo_user: "{{ jenkins_user }}"
+  when: post_requirements.stat.exists
+
+# Archive the current state of the virtualenv
+# as a starting point for new builds.
+# The edx-venv directory is deleted and then recreated
+# cleanly from the archive by the jenkins build scripts.
+- name: Create a clean virtualenv archive
+  command: >
+    tar -cpzf edx-venv_clean.tar.gz edx-venv
+    chdir={{ jenkins_home }}
+  sudo_user: "{{ jenkins_user }}"
+
+# Remove the shallow-clone directory now that we archive
+# done with it
+- name: Remove shallow-clone
+  file: path={{ jenkins_home }}/shallow-clone state=absent

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -38,6 +38,7 @@
   template:
     src=jenkins_env.j2 dest={{ jenkins_home }}/jenkins_env
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0500
+  when: platform_worker is defined
 
 # Need to add Github to known_hosts to avoid
 # being prompted when using git through ssh
@@ -62,8 +63,10 @@
 - name: Create directory for pip-accel config file
   file: path={{ jenkins_home }}/.pip-accel state=directory
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0777 recurse=yes
+  when: platform_worker is defined
 
 - name: Create pip-accel config file
   template:
     src=pip-accel.conf.j2 dest={{ jenkins_home }}/.pip-accel/pip-accel.conf
     owner={{ jenkins_user }} group={{ jenkins_group }} mode=0664
+  when: platform_worker is defined

playbooks/roles/jenkins_worker/tasks/test.yml

@@ -2,43 +2,10 @@
 # Tests for this role
 
 
-# Set up #
-
-# To get a baseline comparison for timestamp comparisons
-# create a testfile and register its stat info
-- name: Create test file
-  file: path=testfile state=touch
-- name: Stat test file
-  stat: path=testfile
-  register: testfile
-
-
-# Tests #
+### Tests ###
 - name: Verify java cmd is using v 1.7
   shell: java -version
   register: java_version
 - assert:
     that:
       - "'1.7.0' in java_version.stderr"
-
-# The role is run with a github oauth token passed in
-# as github_oauth_token var value.
-# This test confirms that the key being used will work
-- name: ensure github token works
-  shell:
-    "github_post_status.py edx edx-platform
-    dddac0b5dddf00c0950daf324e603e4935994954 success
-    https://jenkins.testeng.edx.org/ \"Tests Passed\" \"CI Test Results\""
-
-# Verify the virtualenv tar is newly-built
-- name: Get info on virtualenv tar
-  stat: path={{ jenkins_home }}/edx-venv_clean.tar.gz
-  register: edxvenv
-- assert:
-    that:
-# Assert that it was modified at least within the hour
-      - "{{ testfile.stat.mtime }} - {{ edxvenv.stat.mtime }} < 3600"
-
-# Tear Down #
-- name: Remove test file
-  file: path=testfile state=absent

playbooks/roles/jenkins_worker/tasks/test_platform_worker.yml

+---
+# Tests for this role
+
+
+# Set up #
+
+# To get a baseline comparison for timestamp comparisons
+# create a testfile and register its stat info
+- name: Create test file
+  file: path=testfile state=touch
+- name: Stat test file
+  stat: path=testfile
+  register: testfile
+
+
+### Tests ###
+
+# Firefox has a specific version, not the latest. This test also ensures it was not
+# pulled in via dependency or misuse/clobbering due to the sitespeed variable, which uses
+# the latest firefox.
+- name: Verify firefox version
+  shell: firefox --version
+  register: firefox_version
+- assert:
+    that:
+      - "'28.0' in firefox_version"
+
+# The role is run with a github oauth token passed in
+# as github_oauth_token var value.
+# This test confirms that the key being used will work
+- name: ensure github token works
+  shell:
+    "github_post_status.py edx edx-platform
+    dddac0b5dddf00c0950daf324e603e4935994954 success
+    https://jenkins.testeng.edx.org/ \"Tests Passed\" \"CI Test Results\""
+
+
+# Verify the virtualenv tar is newly-built
+- name: Get info on virtualenv tar
+  stat: path={{ jenkins_home }}/edx-venv_clean.tar.gz
+  register: edxvenv
+- assert:
+    that:
+# Assert that it was modified at least within the hour
+      - "{{ testfile.stat.mtime }} - {{ edxvenv.stat.mtime }} < 3600"
+
+# Tear Down #
+- name: Remove test file
+  file: path=testfile state=absent

playbooks/roles/jenkins_worker/tasks/test_sitespeed_worker.yml

+---
+# Tests for this role
+
+
+### Tests ###
+
+# Sitespeed workers should have the latest version of firefox
+# Lite test. Ensures we are not using
+# the version of firefox specified in a different file.
+- name: Verify firefox version
+  shell: firefox --version
+  register: firefox_version
+- assert:
+    that:
+      - "'40.0' not in firefox_version"

playbooks/roles/sitespeedio/tasks/main.yml

@@ -10,15 +10,15 @@
 #
 #
 # Tasks for role sitespeedio
-# 
+#
 # Overview:
 #   Install the sitespeed.io package along with it's dependencies.
 #   See http://sitespeed.io/ for details on running.
-# 
+#
 #
 # Dependencies:
 #  none
-# 
+#
 # Example play:
 #  role:
 #    - sitespeedio
@@ -26,11 +26,26 @@
 
 # adding chris-lea nodejs repo
 - name: add ppas for current versions of nodejs
-  apt_repository: repo="{{ sitespeedio_chrislea_ppa }}"
+  apt_repository: "repo={{ sitespeedio_chrislea_ppa }}"
 
 # Install Sitespeed Dependencies
 - name: install sitespeed.io specific system packages
   apt: pkg={{','.join(sitespeedio_debian_pkgs)}} state=present
 
-- name: install sitespeed.io
-  npm: name=sitespeed.io global=yes state=latest
+# Install released version of sitespeedio
+#- name: install sitespeed.io
+#  npm: name=sitespeed.io global=yes state=latest
+
+# Temporary workaround to get an unreleased fix for sitespeed.io
+- name: Create a clone of the sitespeed.io repo
+  git: >
+    repo=https://github.com/sitespeedio/sitespeed.io.git
+    dest=/tmp/sitespeedio
+    version=master
+    depth=1
+
+- name: Install sitespeed.io from master
+  shell: >
+    npm install -g
+    chdir=/tmp/sitespeedio
+  sudo: yes

playbooks/roles/test_build_server/files/test-development-environment.sh

 #!/usr/bin/env bash
-
+set -e
 ################################################################################
 # This executes a small subset of the edx-platform tests.  It is intended as
 # a means of testing newly provisioned AMIs for our jenkins workers.

util/packer/jenkins_worker.json

@@ -61,7 +61,6 @@
     "type": "shell",
     "inline": ["cd {{user `playbook_remote_dir`}}",
       ". packer-venv/bin/activate",
-      "pip install -q -U ansible==1.7.1",
       "ansible-playbook run_role.yml -i inventory.ini -c local -e role=test_build_server -vvvv"]
   }]
 }

util/packer/jenkins_worker_simple.json

+{
+  "variables": {
+    "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
+    "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
+    "github_oauth_token": "{{env `GITHUB_TOKEN`}}",
+    "playbook_remote_dir": "/tmp/packer-edx-playbooks",
+    "ami": "{{env `JENKINS_WORKER_AMI`}}"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "ami_name": "jenkins_worker-{{isotime | clean_ami_name}}",
+    "instance_type": "m3.large",
+    "region": "us-east-1",
+    "source_ami": "{{user `ami`}}",
+    "ssh_username": "ubuntu",
+    "ami_description": "jenkins worker",
+    "iam_instance_profile": "jenkins-worker",
+    "security_group_id": "sg-75af5e18"
+  }],
+  "provisioners": [{
+    "type": "shell",
+    "inline": ["rm -rf {{user `playbook_remote_dir`}}",
+      "mkdir {{user `playbook_remote_dir`}}"]
+  }, {
+        "type": "file",
+        "source": "../../playbooks/run_role.yml",
+        "destination": "{{user `playbook_remote_dir`}}/run_role.yml"
+  }, {
+    "type": "file",
+    "source": "../../playbooks/roles",
+    "destination": "{{user `playbook_remote_dir`}}/roles"
+  }, {
+      "type": "file",
+      "source": "../../playbooks/edx-east",
+      "destination": "{{user `playbook_remote_dir`}}/edx-east"
+  }, {
+    "type": "file",
+    "source": "../../requirements.txt",
+    "destination": "{{user `playbook_remote_dir`}}/requirements.txt"
+  }, {
+    "type": "shell",
+    "inline": ["sudo add-apt-repository ppa:git-core/ppa -y"]
+  }, {
+    "type": "shell",
+    "inline": ["sudo apt-get update"]
+  }, {
+    "type": "shell",
+    "inline": ["sudo apt-get -y install git-core"]
+  }, {
+    "type": "shell",
+    "inline": ["cd {{user `playbook_remote_dir`}}",
+      "virtualenv packer-venv",
+      ". packer-venv/bin/activate",
+      "pip install -q -r requirements.txt",
+      "echo '[jenkins_worker]' > inventory.ini",
+      "echo 'localhost' >> inventory.ini",
+      "ansible-playbook run_role.yml -i inventory.ini -c local -e role=jenkins_worker -vvvv"]
+  }]
+}

util/packer/jenkins_worker_sitespeedio.json

+{
+  "variables": {
+    "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
+    "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
+    "github_oauth_token": "{{env `GITHUB_TOKEN`}}",
+    "playbook_remote_dir": "/tmp/packer-edx-playbooks",
+    "ami": "{{env `JENKINS_WORKER_AMI`}}"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "ami_name": "jenkins_worker_sitespeedio-{{isotime | clean_ami_name}}",
+    "instance_type": "m3.large",
+    "region": "us-east-1",
+    "source_ami": "{{user `ami`}}",
+    "ssh_username": "ubuntu",
+    "ami_description": "jenkins worker sitespeedio",
+    "iam_instance_profile": "jenkins-worker",
+    "security_group_id": "sg-75af5e18"
+  }],
+  "provisioners": [{
+    "type": "shell",
+    "inline": ["rm -rf {{user `playbook_remote_dir`}}",
+      "mkdir {{user `playbook_remote_dir`}}"]
+  }, {
+        "type": "file",
+        "source": "../../playbooks/run_role.yml",
+        "destination": "{{user `playbook_remote_dir`}}/run_role.yml"
+  }, {
+    "type": "file",
+    "source": "../../playbooks/roles",
+    "destination": "{{user `playbook_remote_dir`}}/roles"
+  }, {
+      "type": "file",
+      "source": "../../playbooks/edx-east",
+      "destination": "{{user `playbook_remote_dir`}}/edx-east"
+  }, {
+    "type": "file",
+    "source": "../../requirements.txt",
+    "destination": "{{user `playbook_remote_dir`}}/requirements.txt"
+  }, {
+    "type": "shell",
+    "inline": ["sudo add-apt-repository ppa:git-core/ppa -y"]
+  }, {
+    "type": "shell",
+    "inline": ["sudo apt-get update"]
+  }, {
+    "type": "shell",
+    "inline": ["sudo apt-get -y install git-core"]
+  }, {
+    "type": "shell",
+    "inline": ["cd {{user `playbook_remote_dir`}}",
+      "virtualenv packer-venv",
+      ". packer-venv/bin/activate",
+      "pip install -q -r requirements.txt",
+      "echo '[jenkins_worker]' > inventory.ini",
+      "echo 'localhost' >> inventory.ini",
+      "ansible-playbook edx-east/jenkins_worker_sitespeedio.yml -i inventory.ini -c local -vvvv"]
+  }]
+}

util/packer/webpagetest.json

+{
+  "variables": {
+    "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
+    "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
+    "ami": "{{env `WEBPAGETEST_BASE_AMI`}}"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "ami_name": "wpt-{{isotime | clean_ami_name}}",
+    "instance_type": "m3.medium",
+    "region": "us-east-1",
+    "source_ami": "{{user `ami`}}",
+    "ssh_username": "ubuntu",
+    "ami_description": "webpagetest",
+    "iam_instance_profile": "jenkins-worker",
+    "security_group_id": "sg-75af5e18"
+  }],
+  "provisioners": [{
+    "type": "shell",
+    "inline": ["echo 'THESE ARE THE WEBPAGETEST SETTINGS:'",
+    "cat /var/www/webpagetest/www/settings/settings.ini"]
+  }]
+}