First pass at a DE-specific Jenkins role based on jenkins_common.

This is using the jenkins_common machinery from Test Engineering to spin
up and configure Jenkins cleanly.  Primarily, we use the latest version
of Jenkins and removed much of the configure and many of the plugins
that we don't need or care about.

playbooks/edx-east/jenkins_de.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    COMMON_ENABLE_DATADOG: True
+    COMMON_ENABLE_SPLUNKFORWARDER: False
+    COMMON_SECURITY_UPDATES: yes
+    SECURITY_UPGRADE_ON_ANSIBLE: true
+
+  roles:
+    - aws
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - jenkins_de

playbooks/roles/jenkins_de/defaults/main.yml

+de_jenkins_user_uid: 1002
+de_jenkins_group_gid: 1004
+de_jenkins_version: jenkins-2.73.2
+de_jenkins_common_war_source: https://edx-analytics-public.s3.amazonaws.com/packages
+de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx8192m -Djenkins.install.runSetupWizard=false'
+de_jenkins_configuration_scripts:
+  - 1addJarsToClasspath.groovy
+  - 2checkInstalledPlugins.groovy
+  - 3importCredentials.groovy
+  - 3setGlobalProperties.groovy
+  - 3shutdownCLI.groovy
+  - 4configureGit.groovy
+  - 4configureJobConfigHistory.groovy
+  - 4configureMailerPlugin.groovy
+  - 4configureMaskPasswords.groovy
+  - 5createLoggers.groovy
+
+# plugins
+de_jenkins_plugins_list:
+  - name: 'antisamy-markup-formatter'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'script-security'
+    version: '1.27'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'mailer'
+    version: '1.16'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'cvs'
+    version: '2.12'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ldap'
+    version: '1.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ant'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'matrix-auth'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'matrix-project'
+    version: '1.4.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'credentials'
+    version: '1.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ssh-credentials'
+    version: '1.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'external-monitor-job'
+    version: '1.4'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'translation'
+    version: '1.12'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'subversion'
+    version: '2.4.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'junit'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'pam-auth'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'maven-plugin'
+    version: '2.8'
+    group: 'org.jenkins-ci.main'
+  - name: 'ssh-slaves'
+    version: '1.9'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'javadoc'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ansicolor'
+    version: '0.4.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'buildgraph-view'
+    version: '1.1.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-name-setter'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-timeout'
+    version: '1.14.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-user-vars-plugin'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'cobertura'
+    version: '1.9.6'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'copyartifact'
+    version: '1.32.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'credentials-binding'
+    version: '1.7'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ec2'
+    version: '1.28'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'envinject'
+    version: '1.92.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'exclusive-execution'
+    version: '0.8'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'flexible-publish'
+    version: '0.15.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'gradle'
+    version: '1.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'groovy'
+    version: '1.29'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'groovy-postbuild'
+    version: '2.2'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'hockeyapp'
+    version: '1.2.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'htmlpublisher'
+    version: '1.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'jobConfigHistory'
+    version: '2.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'job-dsl'
+    version: '1.45'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'mask-passwords'
+    version: '2.8'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'monitoring'
+    version: '1.56.0'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'multiple-scms'
+    version: '0.6'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'nodelabelparameter'
+    version: '1.7.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'parameterized-trigger'
+    version: '2.25'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'PrioritySorter'
+    version: '2.9'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'rebuild'
+    version: '1.25'
+    group: 'com.sonyericsson.hudson.plugins.rebuild'
+  - name: 'run-condition'
+    version: '1.0'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'shiningpanda'
+    version: '0.21'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ssh-agent'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'text-finder'
+    version: '1.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'timestamper'
+    version: '1.5.15'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'violations'
+    version: '0.7.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'xunit'
+    version: '1.93'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'reverse-proxy-auth-plugin'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+
+# ghprb
+de_jenkins_ghprb_white_list_phrase: '.*[Aa]dd\W+to\W+whitelist.*'
+de_jenkins_ghprb_ok_phrase: '.*ok\W+to\W+test.*'
+de_jenkins_ghprb_retest_phrase: '.*jenkins\W+run\W+all.*'
+de_jenkins_ghprb_skip_phrase: '.*\[[Ss]kip\W+ci\].*'
+de_jenkins_ghprb_cron_schedule: 'H/5 * * * *'
+
+# github
+JENKINS_GITHUB_CONFIG: ''
+
+# hipchat
+de_jenkins_hipchat_room: 'Data Engineering'
+
+# ec2
+de_jenkins_instance_cap: '250'
+
+# seed
+de_jenkins_seed_name: 'manually_seed_one_job'
+
+# logs
+de_jenkins_log_list: {}
+
+# job config history
+de_jenkins_history_max_days: '15'
+de_jenkins_history_exclude_pattern: 'queue|nodeMonitors|UpdateCenter|global-build-stats|GhprbTrigger'

playbooks/roles/jenkins_de/meta/main.yml

+---
+dependencies:
+  - common
+  - role: jenkins_common
+    JENKINS_SERVER_NAME: 'scheduler.analytics.edx.org'
+
+    jenkins_common_version: '{{ de_jenkins_version }}'
+    jenkins_common_war_source: '{{ de_jenkins_common_war_source }}'
+    jenkins_common_user_uid: '{{ de_jenkins_user_uid }}'
+    jenkins_common_group_gid: '{{ de_jenkins_group_gid }}'
+    jenkins_common_jvm_args: '{{ de_jenkins_jvm_args }}'
+    jenkins_common_configuration_scripts: '{{ de_jenkins_configuration_scripts }}'
+    jenkins_common_template_files: '{{ de_jenkins_template_files }}'
+    jenkins_common_plugins_list: '{{ de_jenkins_plugins_list }}'
+    jenkins_common_ghprb_white_list_phrase: '{{ de_jenkins_ghprb_white_list_phrase }}'
+    jenkins_common_ghprb_ok_phrase: '{{ de_jenkins_ghprb_ok_phrase }}'
+    jenkins_common_ghprb_retest_phrase: '{{ de_jenkins_ghprb_retest_phrase }}'
+    jenkins_common_ghprb_skip_phrase: '{{ de_jenkins_ghprb_skip_phrase }}'
+    jenkins_common_ghprb_cron_schedule: '{{ de_jenkins_ghprb_cron_schedule }}'
+    jenkins_common_github_configs: '{{ JENKINS_GITHUB_CONFIG }}'
+    jenkins_common_hipchat_room: '{{ de_jenkins_hipchat_room }}'
+    jenkins_common_instance_cap: '{{ de_jenkins_instance_cap }}'
+    jenkins_common_seed_name: '{{ de_jenkins_seed_name }}'
+    jenkins_common_log_list: '{{ de_jenkins_log_list }}'
+    jenkins_common_history_max_days: '{{ de_jenkins_history_max_days }}'
+    jenkins_common_history_exclude_pattern: '{{ de_jenkins_history_exclude_pattern }}'
+    jenkins_common_server_name: '{{ JENKINS_SERVER_NAME }}'
+    JENKINS_MAIN_GITHUB_OWNER_WHITELIST: ''