Merge pull request #209 from edx/will/jenkins

Script for creating new jenkins server

AUTHORS

@@ -5,3 +5,6 @@ Vik Paruchuri <vik@edx.org>
 Jason Bau <jbau@stanford.edu>
 Ed Zarecor <ed@edx.org>
 John Kern <kern3020@gmail.com>
+Will Daly <will@edx.org>
+Bethany LaPenta <lapentab@mit.edu>
+Jay Zoldak <zoldak@edx.org>

playbooks/edx_jenkins_tests.yml

@@ -6,6 +6,7 @@
     migrate_db: "yes"
     openid_workaround: True
     ansible_ssh_private_key_file: /var/lib/jenkins/continuous-integration.pem
+    mysql5_workaround: True
   vars_files:
     - "{{ secure_dir }}/vars/edxapp_ref_users.yml"
     - "{{ secure_dir }}/vars/edxapp_sandbox.yml"

playbooks/jenkins.yml

+- name: Configure instance(s)
+  hosts: jenkins
+  sudo: True
+  gather_facts: True
+  roles:
+    - common
+    - edxlocal
+    - role: rbenv
+      rbenv_user: "{{ jenkins_user }}"
+      rbenv_user_home: "{{ jenkins_user_home }}"
+      rbenv_ruby_version: "{{ jenkins_ruby_version }}"
+    - jenkins

playbooks/roles/edxlocal/tasks/main.yml

@@ -4,7 +4,7 @@
 #  - common/tasks/main.yml
 #  - nginx/tasks/main.yml
 #
-#  This installs mysql-server-5.5 though 
+#  This installs mysql-server-5.5 though
 #  in production we use mysql-5.1.62.
 #
 #  We could install from source instead:
@@ -13,12 +13,12 @@
 ---
 - name: edxlocal | install python-pymongo (req for ansible)
   pip: name=pymongo
- 
+
 - name: edxlocal | install python-mysqldb (req for ansible)
   apt: pkg=python-mysqldb state=present
 
 - name: edxlocal | install mysql server and recommends
-  apt: pkg=mysql-server-5.5 state=present install_recommends=yes 
+  apt: pkg=mysql-server-5.5 state=present install_recommends=yes
 
 - name: edxlocal | create a database for edxapp
   mysql_db: >
@@ -38,7 +38,7 @@
     state=present
     encoding=utf8
 
-- name: edxlocal | install mongo server and recommends  
+- name: edxlocal | install mongo server and recommends
   apt: pkg=mongodb-server state=present install_recommends=yes
 
 - name: edxlocal | stop mongo service
@@ -71,4 +71,4 @@
     state=present
 
 - name: edxlocal | install memcached
-  apt: pkg=memcached state=present
+  apt: pkg=memcached state=present
\ No newline at end of file

playbooks/roles/elasticsearch/vars/main.yml

@@ -3,4 +3,4 @@
 elasticsearch_version: "0.90.2"
 elasticsearch_sha: "397227ce37f616734f39f6e93539c9eaa82bec9"
 elasticsearch_file: "elasticsearch-{{ elasticsearch_version }}.deb"
-elasticsearch_url: "https://download.elasticsearch.org/elasticsearch/elasticsearch/{{ elasticsearch_file }}"
+elasticsearch_url: "https://download.elasticsearch.org/elasticsearch/elasticsearch/{{ elasticsearch_file }}"
\ No newline at end of file

playbooks/roles/jenkins/handlers/main.yml

+---
+- name: start xvfb
+  service: name=xvfb state=started
+
+- name: restart Jenkins
+  service: name=jenkins state=restarted

playbooks/roles/jenkins/tasks/browsers.yml

+---
+- name: browsers | Install Firefox
+  apt: pkg=firefox
+
+- name: browsers | Download PhantomJS
+  get_url: url={{ jenkins_phantomjs_url }}
+           dest=/var/tmp/{{ jenkins_phantomjs_archive }}
+
+- name: browsers | Untar PhantomJS
+  command: tar -xjf /var/tmp/{{ jenkins_phantomjs_archive }} -C /var/tmp/
+           creates=/var/tmp/{{ jenkins_phantomjs_folder }}
+
+- name: browsers | Install PhantomJS
+  command: mv /var/tmp/{{ jenkins_phantomjs_folder }} /usr/local/bin/phantomjs
+           creates=/usr/local/bin/phantomjs
+
+- name: browsers | Set PhantomJS permissions
+  file: path=/usr/local/bin/phantomjs mode=0755 state=directory
+
+- name: browsers | Install chrome dependencies
+  apt: pkg=libgconf2-4,libxss1,libnss3-1d,libcurl3,xdg-utils
+
+- name: browsers | Install unzip O_o
+  apt: pkg=unzip
+
+- name: browsers | Install Google Chrome
+  get_url: url=https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb dest=/var/tmp/google-chrome-stable_current_amd64.deb
+
+- name: browsers | Install Google Chrome 2
+  command: dpkg -i google-chrome-stable_current_amd64.deb
+           chdir=/var/tmp
+  ignore_errors: True
+
+- name: browsers | Install ChromeDriver
+  get_url: url=https://chromedriver.googlecode.com/files/chromedriver_linux64_2.1.zip dest=/var/tmp/chromedriver_linux64_2.1.zip
+
+- name: browsers | Install ChromeDriver 2
+  shell: unzip /var/tmp/chromedriver_linux64_2.1.zip
+         chdir=/var/tmp
+
+- name: browsers | Install ChromeDriver 3
+  shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
+
+- name: browsers | Install Chromedriver 4
+  file: path=/usr/local/bin/chromedriver mode=0755
+

playbooks/roles/jenkins/tasks/config-backup.yml

+---
+- name: Make backup directory
+  sudo_user: jenkins
+  shell: mkdir {{ jenkins_home }}/backup
+
+- name: Git clone
+  sudo_user: jenkins
+  action: git repo=https://github.com/lapentab/jenkinsconfig.git dest={{jenkins_home}}/backup

playbooks/roles/jenkins/tasks/jenkins.yml

+---
+- name: jenkins | Add the jenkins user to the edx group
+  user: name=jenkins append=yes groups="edx"
+
+- name: jenkins | install jenkins specific system packages
+  apt: pkg={{','.join(jenkins_debian_pkgs)}} state=present
+
+- name: jenkins | Install gcli
+  sudo_user: jenkins
+  shell: /home/jenkins/.rbenv/bin/rbenv exec gem install github_cli
+
+# This is done so that it will not report back to github while testing
+- name: jenkins | TEMPORARILY DISABLE gcli
+  shell: echo "#!/bin/bash\necho gcli" > /usr/local/bin/gcli
+
+- name: jenkins | Install Jenkins
+  shell: wget -q -O - http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key | sudo apt-key add -
+
+- name: jenkins | Install Jenkins 2
+  shell: sh -c 'echo deb http://pkg.jenkins-ci.org/debian binary/ > /etc/apt/sources.list.d/jenkins.list'
+
+- name: jenkins | Install Jenkins 3
+  shell: apt-get update
+
+- name: jenkins | install jenkins 4
+  apt: pkg=jenkins state=present
+
+- name: jenkins | Make plugins directory
+  sudo_user: jenkins
+  shell: mkdir -p {{ jenkins_home }}/plugins
+
+- name: jenkins | Make virtualenv directory
+  shell: mkdir -p /mnt/virtualenvs
+
+- name: jenkins | Chown virtualenv directory
+  file: path=/mnt/virtualenvs/ state=directory owner=jenkins recurse=yes
+
+- name: jenkins | Make pip-cache directory
+  shell: mkdir -p /mnt/pip-cache
+
+- name: jenkins | Chown pip-cache directory
+  file: path=/mnt/pip-cache/ state=directory owner=jenkins recurse=yes
+
+- name: jenkins | Allow /usr/local/bin to be executable
+  shell: chmod -R go+x /usr/local/bin
+
+- name: jenkins | Install Jenkins plugins
+  sudo_user: jenkins
+  get_url: url=http://updates.jenkins-ci.org/latest/${item}
+           dest={{ jenkins_home }}/plugins/${item}
+  with_items: "{{ jenkins_plugins }}"
+    #- timestamper.hpi
+    #- maven-plugin.hpi
+  notify:
+    - restart Jenkins

playbooks/roles/jenkins/tasks/main.yml

+---
+- include: jenkins.yml
+- include: xvfb.yml
+- include: browsers.yml
+- include: python-pkgs.yml

playbooks/roles/jenkins/tasks/python-pkgs.yml

+---
+- name: python packages | Symbolic link site packages
+  file: src=/usr/local/lib/python2.7/site-packages dest=/usr/local/lib/python2.7/dist-packages state=link
+  ignore_errors: yes
+
+- name: python packages | install numpy
+  shell: pip install numpy==1.6.2
+
+- name: python packages | install scipy
+  shell: pip install scipy==0.11.0

playbooks/roles/jenkins/tasks/xvfb.yml

+---
+- name: xvfb | install xvfb
+  apt: pkg=xvfb state=present
+
+- name: xvfb | configure display
+  template: src=xvfb.sh.j2 dest=/etc/profile.d/xvfb.sh owner=root group=root mode=0755
+
+- name: xvfb | create xvfb upstart script
+  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
+
+- name: xvfb | start xvfb
+  shell: start xvfb
+  ignore_errors: yes
\ No newline at end of file

playbooks/roles/jenkins/templates/xvfb.conf.j2

+description     "Xvfb X Server"
+start on (net-device-up
+          and local-filesystems
+          and runlevel [2345])
+stop on runlevel [016]
+exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
+respawn
+respawn limit 15 5
\ No newline at end of file

playbooks/roles/jenkins/templates/xvfb.sh.j2

+# Set the display to the virtual frame buffer (Xvfb)
+export DISPLAY=:1

playbooks/roles/jenkins/vars/main.yml

+---
+jenkins_home: /var/lib/jenkins
+jenkins_phantomjs_url: https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2
+jenkins_phantomjs_archive: phantomjs-1.9.1-linux-x86_64.tar.bz2
+jenkins_phantomjs_folder: phantomjs-1.9.1-linux-x86_64
+jenkins_user: "jenkins"
+jenkins_user_home: /home/jenkins
+jenkins_ruby_version: "1.9.3-p448"
+jenkins_debian_pkgs:
+    - ruby-bundler
+    - rubygems
+    - rbenv
+    - npm
+    - libgraphviz-dev
+    - gfortran
+    - libopenblas-dev
+    - liblapack-dev
+    - libxml2-dev
+    - libgeos-dev
+    - python-dev
+    - libmysqlclient-dev
+    - build-essential
+    - pkg-config
+    - libxslt1-dev
+    - rake
+jenkins_plugins:
+    - ant.hpi
+    - backup.hpi
+    - build-name-setter.hpi
+    - build-timeout.hpi
+    - cobertura.hpi
+    - credentials.hpi
+    - cvs.hpi
+    - dashboard-view.hpi
+    - external-monitor-job.hpi
+    - ghprb.hpi
+    - git.hpi
+    - github.hpi
+    - git-client.hpi
+    - github-api.hpi
+    - github-oauth.hpi
+    - git-notes.hpi
+    - htmlpublisher.hpi
+    - javadoc.hpi
+    - jobConfigHistory.hpi
+    - ldap.hpi
+    - mailer.hpi
+    - mercurial.hpi
+    - nested-view.hpi
+    - next-build-number.hpi
+    - notification.hpi
+    - pam-auth.hpi
+    - parameterized-trigger.hpi
+    - postbuild-task.hpi
+    - rbenv.hpi
+    - ruby-runtime.hpi
+    - shiningpanda.hpi
+    - ssh-credentials.hpi
+    - ssh-slaves.hpi
+    - subversion.hpi
+    - thinBackup.hpi
+    - tmpcleaner.hpi
+    - token-macro.hpi
+    - translation.hpi
+    - violations.hpi
\ No newline at end of file

playbooks/vars/.gitignore

-secure