Merge branch 'master' into jia/scorm-setting

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,4 +8,4 @@ Make sure that the following steps are done before merging:
     - [ ] Update the appropriate internal repo (be sure to update for all our environments)
     - [ ] If you are updating a secure value rather than an internal one, file a DEVOPS ticket with details.
     - [ ] Add an entry to the CHANGELOG.
-  - [ ] Have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/display/EdxOps/Ops+Ansible+Testing+Checklist)?
+  - [ ] If you are making a complicated change, have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/display/EdxOps/Ops+Ansible+Testing+Checklist)?  Adding a new variable does not require the full list (although testing on a sandbox is a great idea to ensure it links with your downstream code changes).

playbooks/callback_plugins/hipchat_plugin.py

-import os
-import time
-from ansible import utils
-try:
-    import prettytable
-except ImportError:
-    prettytable = None
-try:
-    import hipchat
-except ImportError:
-    hipchat = None
-from ansible.plugins.callback import CallbackBase
-
-
-class CallbackModule(CallbackBase):
-    """Send status updates to a HipChat channel during playbook execution.
-
-    This plugin makes use of the following environment variables:
-        HIPCHAT_TOKEN (required): HipChat API token
-        HIPCHAT_ROOM  (optional): HipChat room to post in. Default: ansible
-        HIPCHAT_FROM  (optional): Name to post as. Default: ansible
-        HIPCHAT_NOTIFY (optional): Add notify flag to important messages ("true" or "false"). Default: true
-        HIPCHAT_MSG_PREFIX (option): Optional prefix to add to all hipchat messages
-        HIPCHAT_MSG_COLOR (option): Optional color for hipchat messages
-        HIPCHAT_CONDENSED (option): Condense the task summary output
-
-    Requires:
-        prettytable
-
-    """
-
-    def __init__(self):
-        self.enabled = "HIPCHAT_TOKEN" in os.environ
-        if not self.enabled:
-            return
-
-        # make sure we got our imports
-        if not hipchat:
-            raise ImportError(
-                "The hipchat plugin requires the hipchat Python module, "
-                "which is not installed or was not found."
-            )
-        if not prettytable:
-            raise ImportError(
-                "The hipchat plugin requires the prettytable Python module, "
-                "which is not installed or was not found."
-            )
-        self.start_time = time.time()
-        self.task_report = []
-        self.last_task = None
-        self.last_task_changed = False
-        self.last_task_count = 0
-        self.last_task_delta = 0
-        self.last_task_start = time.time()
-        self.condensed_task_report = (os.getenv('HIPCHAT_CONDENSED', True) == True)
-        self.room = os.getenv('HIPCHAT_ROOM', 'ansible')
-        self.from_name = os.getenv('HIPCHAT_FROM', 'ansible')
-        self.allow_notify = (os.getenv('HIPCHAT_NOTIFY') != 'false')
-        try:
-            self.hipchat_conn = hipchat.HipChat(token=os.getenv('HIPCHAT_TOKEN'))
-        except Exception as e:
-            utils.warning("Unable to connect to hipchat: {}".format(e))
-        self.hipchat_msg_prefix = os.getenv('HIPCHAT_MSG_PREFIX', '')
-        self.hipchat_msg_color = os.getenv('HIPCHAT_MSG_COLOR', '')
-        self.printed_playbook = False
-        self.playbook_name = None
-
-    def _send_hipchat(self, message, room=None, from_name=None, color=None, message_format='text'):
-
-        if not room:
-            room = self.room
-        if not from_name:
-            from_name = self.from_name
-        if not color:
-            color = self.hipchat_msg_color
-        try:
-            self.hipchat_conn.message_room(room, from_name, message, color=color, message_format=message_format)
-        except Exception as e:
-            utils.warning("Could not submit message to hipchat: {}".format(e))
-
-    def _flush_last_task(self):
-        if self.last_task:
-            delta = time.time() - self.last_task_start
-            self.task_report.append(dict(
-                                    changed=self.last_task_changed,
-                                    count=self.last_task_count,
-                                    delta="{:0>.1f}".format(self.last_task_delta),
-                                    task=self.last_task))
-        self.last_task_count = 0
-        self.last_task_changed = False
-        self.last_task = None
-        self.last_task_delta = 0
-
-    def _process_message(self, msg, msg_type='STATUS'):
-
-        if msg_type == 'OK' and self.last_task:
-            if msg.get('changed', True):
-                self.last_task_changed = True
-            if msg.get('delta', False):
-                (hour, minute, sec) = msg['delta'].split(':')
-                total = float(hour) * 1200 + float(minute) * 60 + float(sec)
-                self.last_task_delta += total
-            self.last_task_count += 1
-        else:
-            self._flush_last_task()
-
-        if msg_type == 'TASK_START':
-            self.last_task = msg
-            self.last_task_start = time.time()
-        elif msg_type == 'FAILED':
-            self.last_task_start = time.time()
-            if 'msg' in msg:
-                self._send_hipchat('/code {}: The ansible run returned the following error:\n\n {}'.format(
-                    self.hipchat_msg_prefix, msg['msg']), color='red', message_format='text')
-        else:
-            # move forward the last task start time
-            self.last_task_start = time.time()
-
-
-    def on_any(self, *args, **kwargs):
-        pass
-
-    def runner_on_failed(self, host, res, ignore_errors=False):
-        if self.enabled:
-            self._process_message(res, 'FAILED')
-
-    def runner_on_ok(self, host, res):
-        if self.enabled:
-            # don't send the setup results
-            if 'invocation' in res and 'module_name' in res['invocation'] and res['invocation']['module_name'] != "setup":
-                self._process_message(res, 'OK')
-
-
-    def runner_on_error(self, host, msg):
-        if self.enabled:
-            self._process_message(msg, 'ERROR')
-
-    def runner_on_skipped(self, host, item=None):
-        if self.enabled:
-            self._process_message(item, 'SKIPPED')
-
-    def runner_on_unreachable(self, host, res):
-        pass
-
-    def runner_on_no_hosts(self):
-        pass
-
-    def runner_on_async_poll(self, host, res, jid, clock):
-        if self.enabled:
-            self._process_message(res, 'ASYNC_POLL')
-
-    def runner_on_async_ok(self, host, res, jid):
-        if self.enabled:
-            self._process_message(res, 'ASYNC_OK')
-
-    def runner_on_async_failed(self, host, res, jid):
-        if self.enabled:
-            self._process_message(res, 'ASYNC_FAILED')
-
-    def playbook_on_start(self):
-        pass
-
-    def playbook_on_notify(self, host, handler):
-        pass
-
-    def playbook_on_no_hosts_matched(self):
-        pass
-
-    def playbook_on_no_hosts_remaining(self):
-        pass
-
-    def playbook_on_task_start(self, name, is_conditional):
-        if self.enabled:
-            self._process_message(name, 'TASK_START')
-
-
-    def playbook_on_vars_prompt(self, varname, private=True, prompt=None,
-                                encrypt=None, confirm=False, salt_size=None,
-                                salt=None, default=None):
-        pass
-
-    def playbook_on_setup(self):
-        pass
-
-    def playbook_on_import_for_host(self, host, imported_file):
-        pass
-
-    def playbook_on_not_import_for_host(self, host, missing_file):
-        pass
-
-    def playbook_on_play_start(self, pattern):
-        if self.enabled:
-            """Display Playbook and play start messages"""
-            self.start_time = time.time()
-            self.playbook_name, _ = os.path.splitext(os.path.basename(self.play.playbook.filename))
-            host_list = self.play.playbook.inventory.host_list
-            inventory = os.path.basename(os.path.realpath(host_list))
-            subset = self.play.playbook.inventory._subset
-            msg = "<b>{description}</b>: Starting ansible run for play <b><i>{play}</i></b>".format(description=self.hipchat_msg_prefix, play=self.playbook_name)
-            if self.play.playbook.only_tags and 'all' not in self.play.playbook.only_tags:
-                msg = msg + " with tags <b><i>{}</i></b>".format(','.join(self.play.playbook.only_tags))
-            if subset:
-                msg = msg + " on hosts <b><i>{}</i></b>".format(','.join(subset))
-            self._send_hipchat(msg,  message_format='html')
-
-    def playbook_on_stats(self, stats):
-        if self.enabled:
-            self._flush_last_task()
-            delta = time.time() - self.start_time
-            self.start_time = time.time()
-            """Display info about playbook statistics"""
-            hosts = sorted(stats.processed.keys())
-            task_column = '{} - Task'.format(self.hipchat_msg_prefix)
-            task_summary = prettytable.PrettyTable([task_column, 'Time', 'Count', 'Changed'])
-            task_summary.align[task_column] = "l"
-            task_summary.align['Time'] = "r"
-            task_summary.align['Count'] = "r"
-            task_summary.align['Changed'] = "r"
-
-
-
-            for task in self.task_report:
-                if self.condensed_task_report:
-                    # for the condensed task report skip all tasks
-                    # that are not marked as changed and that have
-                    # a time delta less than 1
-                    if not task['changed'] and float(task['delta']) < 1:
-                        continue
-                task_summary.add_row([task['task'], task['delta'], str(task['count']), str(task['changed'])])
-
-            summary_table = prettytable.PrettyTable(['Ok', 'Changed', 'Unreachable', 'Failures'])
-            self._send_hipchat("/code " + str(task_summary) )
-
-            summary_all_host_output = []
-            for host in hosts:
-                stats = stats.summarize(host)
-                summary_output = "<b>{}</b>: <i>{}</i> - ".format(self.hipchat_msg_prefix, host)
-                for summary_item in ['ok', 'changed', 'unreachable', 'failures']:
-                    if stats[summary_item] != 0:
-                        summary_output += "<b>{}</b> - {} ".format(summary_item, stats[summary_item])
-                summary_all_host_output.append(summary_output)
-            self._send_hipchat("<br />".join(summary_all_host_output), message_format='html')
-            msg = "<b>{description}</b>: Finished Ansible run for <b><i>{play}</i> in {min:02} minutes, {sec:02} seconds</b><br /><br />".format(
-                description=self.hipchat_msg_prefix,
-                play=self.playbook_name,
-                min=int(delta / 60),
-                sec=int(delta % 60))
-            self._send_hipchat(msg, message_format='html')

playbooks/edx-east/jenkins_build.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - aws
+    - jenkins_build

playbooks/edx-east/jenkins_test.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - aws
+    - role: jenkins_build
+      build_jenkins_server_name: test-jenkins.testeng.edx.org
+      build_jenkins_configuration_scripts:
+        - 1addJarsToClasspath.groovy
+        - 2checkInstalledPlugins.groovy
+        - 3mainConfiguration.groovy
+        - 3shutdownCLI.groovy
+        - 4configureEc2Plugin.groovy
+        - 4configureGHOAuth.groovy
+        - 4configureGHPRB.groovy
+        - 4configureGit.groovy
+        - 4configureHipChat.groovy
+        - 4configureJobConfigHistory.groovy
+        - 5addSeedJob.groovy
+        - 5createLoggers.groovy
+        - 6importCredentials.groovy

playbooks/edx-east/mongo_3_2.yml

@@ -10,6 +10,12 @@
 # ADDING A NEW CLUSTER MEMBER
 # If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory
 # ansible-playbook mongo_3_2.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
 - name: Deploy MongoDB
   hosts: all
   become: True

playbooks/roles/edxapp/defaults/main.yml

@@ -307,6 +307,9 @@ EDXAPP_RATE_LIMITED_USER_AGENTS: []
 EDXAPP_LANG: 'en_US.UTF-8'
 EDXAPP_LANGUAGE_CODE : 'en'
 EDXAPP_LANGUAGE_COOKIE: 'openedx-language-preference'
+EDXAPP_CERTIFICATE_TEMPLATE_LANGUAGES:
+  'en': 'English'
+  'es': 'Español'
 EDXAPP_TIME_ZONE: 'America/New_York'
 
 EDXAPP_HELP_TOKENS_BOOKS:
@@ -728,6 +731,8 @@ EDXAPP_ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES:
 
 EDXAPP_ENTERPRISE_ENROLLMENT_API_URL: "{{ EDXAPP_LMS_ROOT_URL }}/api/enrollment/v1/"
 
+EDXAPP_ENTERPRISE_SUPPORT_URL: ''
+
 # The assigned ICP license number for display in the platform footer
 EDXAPP_ICP_LICENSE: !!null
 
@@ -1018,6 +1023,7 @@ generic_env_config:  &edxapp_generic_env
   TIME_ZONE: "{{ EDXAPP_TIME_ZONE }}"
   LANGUAGE_CODE: "{{ EDXAPP_LANGUAGE_CODE }}"
   LANGUAGE_COOKIE: "{{ EDXAPP_LANGUAGE_COOKIE }}"
+  CERTIFICATE_TEMPLATE_LANGUAGES: "{{ EDXAPP_CERTIFICATE_TEMPLATE_LANGUAGES }}"
   MKTG_URL_LINK_MAP: "{{ EDXAPP_MKTG_URL_LINK_MAP }}"
   MKTG_URLS: "{{ EDXAPP_MKTG_URLS }}"
   SUPPORT_SITE_LINK: "{{ EDXAPP_SUPPORT_SITE_LINK }}"
@@ -1205,6 +1211,7 @@ lms_env_config:
   PASSWORD_MIN_LENGTH: "{{ EDXAPP_PASSWORD_MIN_LENGTH }}"
   PASSWORD_MAX_LENGTH: "{{ EDXAPP_PASSWORD_MAX_LENGTH }}"
   ENTERPRISE_ENROLLMENT_API_URL: "{{ EDXAPP_ENTERPRISE_ENROLLMENT_API_URL }}"
+  ENTERPRISE_SUPPORT_URL: "{{ EDXAPP_ENTERPRISE_SUPPORT_URL }}"
   PARENTAL_CONSENT_AGE_LIMIT: "{{ EDXAPP_PARENTAL_CONSENT_AGE_LIMIT }}"
   ACE_ENABLED_CHANNELS: "{{ EDXAPP_ACE_ENABLED_CHANNELS }}"
   ACE_ENABLED_POLICIES: "{{ EDXAPP_ACE_ENABLED_POLICIES }}"

playbooks/roles/jenkins_build/defaults/main.yml

+build_jenkins_version: jenkins_1.651.3
+build_jenkins_configuration_scripts:
+  - 1addJarsToClasspath.groovy
+  - 2checkInstalledPlugins.groovy
+  - 3mainConfiguration.groovy
+  - 3shutdownCLI.groovy
+  - 4configureEc2Plugin.groovy
+  - 4configureGHOAuth.groovy
+  - 4configureGHPRB.groovy
+  - 4configureGit.groovy
+  - 4configureHipChat.groovy
+  - 4configureJobConfigHistory.groovy
+  - 4configureMailerPlugin.groovy
+  - 5addSeedJob.groovy
+  - 5createLoggers.groovy
+  - 6importCredentials.groovy
+
+# plugins
+build_jenkins_plugins_list:
+  - name: 'antisamy-markup-formatter'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'script-security'
+    version: '1.27'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'mailer'
+    version: '1.16'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'cvs'
+    version: '2.12'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ldap'
+    version: '1.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'windows-slaves'
+    version: '1.0'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ant'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'matrix-auth'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'matrix-project'
+    version: '1.4.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'credentials'
+    version: '1.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ssh-credentials'
+    version: '1.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'external-monitor-job'
+    version: '1.4'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'translation'
+    version: '1.12'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'subversion'
+    version: '2.4.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'junit'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'pam-auth'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'maven-plugin'
+    version: '2.8'
+    group: 'org.jenkins-ci.main'
+  - name: 'ssh-slaves'
+    version: '1.9'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'javadoc'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ansicolor'
+    version: '0.4.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'bouncycastle-api'
+    version: '1.0.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-flow-plugin'
+    version: '0.17'
+    group: 'com.cloudbees.plugins'
+  - name: 'build-flow-test-aggregator'
+    version: '1.0'
+    group: 'org.zeroturnaround.jenkins'
+  - name: 'build-flow-toolbox-plugin'
+    version: '0.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'buildgraph-view'
+    version: '1.1.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-name-setter'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-timeout'
+    version: '1.14.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-user-vars-plugin'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'cobertura'
+    version: '1.9.6'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'copyartifact'
+    version: '1.32.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'credentials-binding'
+    version: '1.7'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ec2'
+    version: '1.28'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'envinject'
+    version: '1.92.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'exclusive-execution'
+    version: '0.8'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'flexible-publish'
+    version: '0.15.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ghprb'
+    version: '1.22.4'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'github'
+    version: '1.14.0'
+    group: 'com.coravy.hudson.plugins.github'
+  - name: 'github-oauth'
+    version: '0.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'gradle'
+    version: '1.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'groovy'
+    version: '1.29'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'groovy-postbuild'
+    version: '2.2'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'hipchat'
+    version: '0.1.9'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'hockeyapp'
+    version: '1.2.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'htmlpublisher'
+    version: '1.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'jobConfigHistory'
+    version: '2.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'job-dsl'
+    version: '1.45'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'mask-passwords'
+    version: '2.8'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'monitoring'
+    version: '1.56.0'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'multiple-scms'
+    version: '0.6'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'nodelabelparameter'
+    version: '1.7.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'parameterized-trigger'
+    version: '2.25'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'PrioritySorter'
+    version: '2.9'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'rebuild'
+    version: '1.25'
+    group: 'com.sonyericsson.hudson.plugins.rebuild'
+  - name: 'run-condition'
+    version: '1.0'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'shiningpanda'
+    version: '0.21'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ssh-agent'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'text-finder'
+    version: '1.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'thinBackup'
+    version: '1.7.4'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'timestamper'
+    version: '1.5.15'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'violations'
+    version: '0.7.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'xunit'
+    version: '1.93'
+    group: 'org.jenkins-ci.plugins'
+
+# ghprb
+build_jenkins_ghprb_ok_phrase: '.*ok\W+to\W+test.*'
+build_jenkins_ghprb_retest_phrase: '.*jenkins\W+run\W+all.*'
+build_jenkins_ghprb_skip_phrase: '.*\[[Ss]kip\W+ci\].*'
+build_jenkins_ghprb_cron_schedule: 'H/5 * * * *'
+
+# hipchat
+build_jenkins_hipchat_room: 'testeng'
+
+# ec2
+build_jenkins_instance_cap: '250'
+
+# seed
+build_jenkins_seed_name: 'manually_seed_one_job'
+
+# logs
+build_jenkins_log_list:
+  - LOG_RECORDER: 'Ghprb'
+    LOGGERS:
+      - name: 'org.jenkinsci.plugins.ghprb.GhprbPullRequest'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.ghprb.GhprbRootAction'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.ghprb.GhprbRepository'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.ghprb.GhprbGitHub'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.ghprb.Ghprb'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.ghprb.GhprbTrigger'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.ghprb.GhprbBuilds'
+        log_level: 'ALL'
+  - LOG_RECORDER: 'GithubPushLogs'
+    LOGGERS:
+      - name: 'com.cloudbees.jenkins.GitHubPushTrigger'
+        log_level: 'ALL'
+      - name: 'org.jenkinsci.plugins.github.webhook.WebhookManager'
+        log_level: 'ALL'
+      - name: 'com.cloudbees.jenkins.GitHubWebHook'
+        log_level: 'ALL'
+      - name: 'hudson.plugins.git.GitSCM'
+        log_level: 'ALL'
+
+# job config history
+build_jenkins_history_max_days: '15'
+build_jenkins_history_exclude_pattern: 'queue|nodeMonitors|UpdateCenter|global-build-stats|GhprbTrigger'

playbooks/roles/jenkins_build/meta/main.yml

+---
+dependencies:
+  - common
+  - role: jenkins_common
+    jenkins_common_version: '{{ build_jenkins_version }}'
+    jenkins_common_configuration_scripts: '{{ build_jenkins_configuration_scripts }}'
+    jenkins_common_template_files: '{{ build_jenkins_template_files }}'
+    jenkins_common_plugins_list: '{{ build_jenkins_plugins_list }}'
+    jenkins_common_ghprb_ok_phrase: '{{ build_jenkins_ghprb_ok_phrase }}'
+    jenkins_common_ghprb_retest_phrase: '{{ build_jenkins_ghprb_retest_phrase }}'
+    jenkins_common_ghprb_skip_phrase: '{{ build_jenkins_ghprb_skip_phrase }}'
+    jenkins_common_ghprb_cron_schedule: '{{ build_jenkins_ghprb_cron_schedule }}'
+    jenkins_common_hipchat_room: '{{ build_jenkins_hipchat_room }}'
+    jenkins_common_instance_cap: '{{ build_jenkins_instance_cap }}'
+    jenkins_common_seed_name: '{{ build_jenkins_seed_name }}'
+    jenkins_common_log_list: '{{ build_jenkins_log_list }}'
+    jenkins_common_history_max_days: '{{ build_jenkins_history_max_days }}'
+    jenkins_common_history_exclude_pattern: '{{ build_jenkins_history_exclude_pattern }}'
+    jenkins_common_server_name: '{{ build_jenkins_server_name }}'

playbooks/roles/jenkins_common/defaults/main.yml

+jenkins_common_user: jenkins
+jenkins_common_group: edx
+jenkins_common_home: /var/lib/jenkins
+jenkins_common_config_path: /init-configs
+jenkins_common_port: 8080
+jenkins_common_version: jenkins_1.651.3
+jenkins_common_war_source: https://s3.amazonaws.com/edx-testeng-tools/jenkins
+jenkins_common_nginx_port: 80
+jenkins_common_protocol_https: true
+jenkins_common_server_name: jenkins.example.org
+
+jenkins_common_debian_pkgs:
+  - nginx
+  - git
+  - curl
+  - maven
+  - daemon
+  - psmisc
+
+jenkins_common_configuration_git_url: https://github.com/edx/jenkins-configuration.git
+jenkins_common_configuration_src_path: src/main/groovy
+jenkins_common_git_home: /git
+
+jenkins_common_configuration_scripts: []
+jenkins_common_non_plugin_template_files:
+  - credentials
+  - ec2_config
+  - ghprb_config
+  - git_config
+  - hipchat_config
+  - jenkins_config_history
+  - log_config
+  - mailer_config
+  - main_config
+  - security
+  - seed_config
+
+# Jenkins default config values
+
+# main
+jenkins_common_main_system_message: ''
+jenkins_common_main_num_executors: 1
+jenkins_common_main_labels:
+  - 'dsl-seed-runner'
+  - 'backup-runner'
+jenkins_common_main_quiet_period: 5
+jenkins_common_main_scm_retry: 2
+jenkins_common_main_disable_remember: true
+jenkins_common_main_env_vars:
+  - NAME: 'BROWSERMOB_PROXY_PORT'
+    VALUE: '9090'
+  - NAME: 'GITHUB_OWNER_WHITELIST'
+    VALUE: '{{ JENKINS_MAIN_GITHUB_OWNER_WHITELIST }}'
+jenkins_common_main_executable: '/bin/bash'
+
+
+JENKINS_MAIN_URL: 'https://jenkins.example.org/'
+JENKINS_MAIN_ADMIN_EMAIL: 'jenkins <admin@example.org>'
+
+# plugins
+jenkins_common_plugins_list: []
+
+# ec2
+jenkins_common_use_instance_profile_for_creds: false
+jenkins_common_instance_cap: ''
+
+JENKINS_EC2_PRIVATE_KEY: ''
+JENKINS_EC2_REGION: ''
+JENKINS_EC2_ACCESS_KEY_ID: ''
+JENKINS_EC2_SECRET_ACCESS_KEY: ''
+JENKINS_EC2_AMIS: []
+
+# ghprb
+jenkins_common_ghprb_server: 'https://api.github.com'
+
+jenkins_common_ghprb_request_testing: ''
+jenkins_common_ghprb_white_list_phrase: ''
+jenkins_common_ghprb_ok_phrase: ''
+jenkins_common_ghprb_retest_phrase: ''
+jenkins_common_ghprb_skip_phrase: ''
+jenkins_common_ghprb_cron_schedule: ''
+
+jenkins_common_ghprb_use_comments: false
+jenkins_common_ghprb_use_detailed_comments: false
+jenkins_common_ghprb_manage_webhooks: false
+jenkins_common_ghprb_failure_as: 'failure'
+jenkins_common_ghprb_auto_close_fails: false
+jenkins_commmon_ghprb_display_errors: false
+
+jenkins_common_ghprb_github_auth: ''
+jenkins_common_ghprb_simple_status: ''
+jenkins_common_ghprb_publish_jenkins_url: ''
+jenkins_common_ghprb_build_log_lines:
+jenkins_common_ghprb_results:
+  - STATUS: 'FAILURE'
+    MESSAGE: 'Test FAILed.'
+  - STATUS: 'SUCCESS'
+    MESSAGE: 'Test PASSed.'
+
+JENKINS_GHPRB_TOKEN: ''
+JENKINS_GHPRB_ADMIN_LIST: []
+JENKINS_GHPRB_BLACK_LIST: []
+JENKINS_GHPRB_WHITE_LIST: []
+
+# credentials
+JENKINS_SECRET_FILES_LIST: []
+JENKINS_USERNAME_PASSWORD_LIST: []
+JENKINS_SECRET_TEXT_LIST: []
+JENKINS_CERTIFICATES_LIST: []
+JENKINS_SSH_LIST: []
+
+# security
+jenkins_common_security_scopes: 'read:org,user:email'
+
+JENKINS_SECURITY_CLIENT_ID: ''
+JENKINS_SECURITY_CLIENT_SECRET: ''
+JENKINS_SECURITY_GROUPS: []
+
+# git
+JENKINS_GIT_NAME: 'jenkins'
+JENKINS_GIT_EMAIL: 'jenkins@example.com'
+
+# hipchat
+jenkins_common_hipchat_room: ''
+JENKINS_HIPCHAT_API_TOKEN: ''
+
+# seed
+jenkins_common_seed_name: 'seed_job'
+jenkins_common_seed_path: '{{ jenkins_common_config_path }}/xml/seed_job.xml'
+
+# logs
+jenkins_common_log_list:
+  - LOG_RECORDER: 'Sample Log'
+    LOGGERS:
+      - name: 'org.jenkinsci.plugins.example.Class'
+        log_level: 'ALL'
+
+# job config history
+jenkins_common_history_root: ''
+jenkins_common_history_max_entries: ''
+jenkins_common_history_max_days: ''
+jenkins_common_history_max_entries_page: ''
+jenkins_common_history_skip_duplicates: true
+jenkins_common_history_exclude_pattern: ''
+jenkins_common_history_save_module_config: false
+jenkins_common_history_show_build_badges: 'always'
+jenkins_common_history_excluded_users: ''
+
+# mailer
+jenkins_common_mailer_port: 465
+jenkins_common_mailer_use_ssl: true
+jenkins_common_mailer_char_set: 'UTF-8'
+JENKINS_MAILER_SMTP_SERVER: ''
+JENKINS_MAILER_REPLY_TO_ADDRESS: 'jenkins'
+JENKINS_MAILER_DEFAULT_SUFFIX: '@example.com'
+JENKINS_MAILER_SMTP_AUTH_USERNAME: ''
+JENKINS_MAILER_SMTP_AUTH_PASSWORD: ''
\ No newline at end of file

playbooks/roles/jenkins_common/files/ec2/mongo_init_script.sh

+# This confirms that mongo is running and is accessible on localhost
+# It could expose internal network problems, in which case the worker should not be used
+# Mongo seems to spend a bit of time starting.
+i=0
+
+while [ $i -lt 45 ]; do
+    mongo --quiet --eval 'db.getMongo().getDBNames()' 2>/dev/null 1>&2
+    if [ $? -eq 0 ]; then
+        break
+    else
+        sleep 2
+        i=$[$i+1]
+    fi
+done
+
+mongo --quiet --eval 'db.getMongo().getDBNames()'

playbooks/roles/jenkins_common/files/xml/seed_job.xml

+<?xml version='1.0' encoding='UTF-8'?>
+<project>
+  <actions/>
+  <description>Run one dsl job at a time.</description>
+  <keepDependencies>false</keepDependencies>
+  <properties>
+    <jenkins.model.BuildDiscarderProperty>
+      <strategy class="hudson.tasks.LogRotator">
+        <daysToKeep>-1</daysToKeep>
+        <numToKeep>20</numToKeep>
+        <artifactDaysToKeep>-1</artifactDaysToKeep>
+        <artifactNumToKeep>-1</artifactNumToKeep>
+      </strategy>
+    </jenkins.model.BuildDiscarderProperty>
+    <hudson.model.ParametersDefinitionProperty>
+      <parameterDefinitions>
+        <hudson.model.StringParameterDefinition>
+          <name>DSL_SCRIPT</name>
+          <description>Path to dsl script to run, from the root of the https://github.com/edx/jenkins-job-dsl repo (i.e. sample/jobs/sampleJob.groovy)</description>
+          <defaultValue>sample/jobs/sampleJob.groovy</defaultValue>
+        </hudson.model.StringParameterDefinition>
+        <hudson.model.StringParameterDefinition>
+          <name>BRANCH</name>
+          <description>Branch of jenkins-job-dsl repo to use</description>
+          <defaultValue>*/master</defaultValue>
+        </hudson.model.StringParameterDefinition>
+      </parameterDefinitions>
+    </hudson.model.ParametersDefinitionProperty>
+  </properties>
+  <scm class="hudson.plugins.git.GitSCM" plugin="git@2.2.4">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <url>https://github.com/edx/jenkins-job-dsl.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>${BRANCH}</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <submoduleCfg class="list"/>
+    <extensions/>
+  </scm>
+  <canRoam>true</canRoam>
+  <disabled>false</disabled>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <triggers/>
+  <concurrentBuild>false</concurrentBuild>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/usr/bin/env bash
+
+      # exit if user-supplied parameter does not exisit
+      if [ ! -e ${DSL_SCRIPT} ]; then
+        echo &quot;DSL Script &apos;{DSL_SCRIPT}&apos; does not exist. Please try again&quot;
+          exit 1
+      fi
+      </command>
+    </hudson.tasks.Shell>
+    <hudson.plugins.gradle.Gradle plugin="gradle@1.24">
+      <description>tert</description>
+      <switches></switches>
+      <tasks>libs
+      assemble</tasks>
+      <rootBuildScriptDir></rootBuildScriptDir>
+      <buildFile></buildFile>
+      <gradleName>(Default)</gradleName>
+      <useWrapper>true</useWrapper>
+      <makeExecutable>true</makeExecutable>
+      <fromRootBuildScriptDir>true</fromRootBuildScriptDir>
+      <useWorkspaceAsHome>true</useWorkspaceAsHome>
+    </hudson.plugins.gradle.Gradle>
+    <javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.45">
+      <targets>${DSL_SCRIPT}</targets>
+      <usingScriptText>false</usingScriptText>
+      <ignoreExisting>false</ignoreExisting>
+      <removedJobAction>IGNORE</removedJobAction>
+      <removedViewAction>IGNORE</removedViewAction>
+      <lookupStrategy>JENKINS_ROOT</lookupStrategy>
+      <additionalClasspath>lib/snakeyaml-1.17.jar
+      src/main/groovy</additionalClasspath>
+    </javaposse.jobdsl.plugin.ExecuteDslScripts>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

playbooks/roles/jenkins_common/meta/main.yml

+---
+dependencies:
+  - common
+  - role: nginx
+    nginx_app_dir: "/etc/nginx"
+    nginx_log_dir: "/var/log/nginx"
+    nginx_data_dir: "{{ nginx_app_dir }}"
+    nginx_conf_dir: "{{ nginx_app_dir }}/conf.d"
+    nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available"
+    nginx_sites_enabled_dir: "{{ nginx_app_dir }}/sites-enabled"
+    nginx_server_static_dir: "{{ nginx_data_dir }}/server-static"
+    nginx_htpasswd_file: "{{ nginx_app_dir }}/nginx.htpasswd"
+    nginx_default_sites: "jenkins"
+    nginx_template_dir: "etc/nginx/sites-available"
+    nginx_sites: jenkins
+    jenkins_nginx_port: "{{ jenkins_common_nginx_port }}"
+    jenkins_server_name: "{{ jenkins_common_server_name }}"
+    jenkins_port: "{{ jenkins_common_port }}"
+    jenkins_protocol_https: "{{ jenkins_common_protocol_https }}"
+  - role: oraclejdk
+    tags: java

playbooks/roles/jenkins_common/tasks/main.yml

+---
+- name: Install jenkins specific system packages
+  apt:
+    name: '{{ item }}'
+    state: present
+    update_cache: yes
+  with_items: '{{ jenkins_common_debian_pkgs }}'
+  tags:
+    - jenkins
+    - install
+    - install:system-requirements
+
+- name: Create jenkins group
+  group:
+    name: '{{ jenkins_common_group }}'
+    state: present
+  tags:
+    - install
+    - install:system-requirements
+
+- name: Add the jenkins user to the group
+  user:
+    name: '{{ jenkins_common_user }}'
+    append: yes
+    groups: '{{ jenkins_common_group }}'
+  tags:
+    - install
+    - install:system-requirements
+
+- name: Create necessary folders
+  file:
+    path: '{{ item }}'
+    state: directory
+    owner: '{{ jenkins_common_user }}'
+  with_items:
+    - /usr/share/jenkins
+    - '{{ jenkins_common_home }}/init.groovy.d'
+    - '{{ jenkins_common_config_path }}'
+    - '{{ jenkins_common_home }}/utils'
+    - '{{ jenkins_common_home }}/plugins'
+    - '{{ jenkins_common_git_home }}'
+  tags:
+    - install
+    - install:base
+
+- name: Download Jenkins war file
+  get_url:
+    url: '{{ jenkins_common_war_source }}/{{ jenkins_common_version }}.war'
+    dest: /usr/share/jenkins/jenkins.war
+    force: yes
+  tags:
+    - install
+    - install:app-requirements
+
+- name: Add Jenkins systemd configuration
+  template:
+    src:  "etc/systemd/system/jenkins.service.j2"
+    dest: "/etc/systemd/system/jenkins.service"
+  tags:
+    - install
+    - install:system-requirements
+
+- name: Add env vars
+  template:
+    src: "jenkins-env.sh.j2"
+    dest: "/etc/profile.d/jenkins-env.sh"
+    owner: root
+    group: root
+    mode: "0755"
+  tags:
+    - install
+    - install:base
+
+- name: Download jenkins-configuration repo
+  git:
+      repo: '{{ jenkins_common_configuration_git_url }}'
+      dest: '{{ jenkins_common_git_home }}/jenkins-configuration'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Run gradle libs
+  shell: './gradlew libs'
+  args:
+    chdir: '{{ jenkins_common_git_home }}/jenkins-configuration'
+  environment:
+    UTILS_PATH: '{{ jenkins_common_home }}/utils'
+    JENKINS_VERSION: '{{ jenkins_common_version }}'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Copy init scripts into init.groovy.d
+  command: 'cp {{ jenkins_common_git_home }}/jenkins-configuration/{{ jenkins_common_configuration_src_path }}/{{ item }} {{ jenkins_common_home }}/init.groovy.d/'
+  with_items: '{{ jenkins_common_configuration_scripts }}'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Create jenkins config sub folders
+  file:
+    path: '{{ item }}'
+    state: directory
+    owner: '{{ jenkins_common_user }}'
+  with_items:
+    - '{{ jenkins_common_config_path }}/credentials'
+    - '{{ jenkins_common_config_path }}/ec2'
+    - '{{ jenkins_common_config_path }}/xml'
+  tags:
+     - install
+     - install:base
+
+- name: Copy non plugins template files
+  template:
+    src: '{{ role_path }}/templates/config/{{ item }}.yml.j2'
+    dest: '{{ jenkins_common_config_path }}/{{ item }}.yml'
+  with_items: '{{ jenkins_common_non_plugin_template_files }}'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Copy plugins.yml config file
+  template:
+    src: '{{ role_path }}/templates/config/plugins.yml.j2'
+    dest: '{{jenkins_common_config_path }}/plugins.yml'
+  tags:
+    - install
+    - install:base
+    - install:plugins
+    - install:jenkins-configuration
+
+- name: Copy ec2 config files
+  template: src={{ item }} dest='{{ jenkins_common_config_path }}/ec2/'
+  with_fileglob:
+    - '{{ role_path }}/files/ec2/*'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Copy xml config files
+  template: src={{ item }} dest='{{ jenkins_common_config_path }}/xml/'
+  with_fileglob:
+    - '{{ role_path }}/files/xml/*'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Run plugins.gradle
+  shell: './gradlew -b plugins.gradle plugins'
+  args:
+    chdir: '{{ jenkins_common_git_home }}/jenkins-configuration'
+  environment:
+    PLUGIN_OUTPUT_DIR: '{{ jenkins_common_home }}/plugins'
+    PLUGIN_CONFIG: '{{ jenkins_common_config_path }}/plugins.yml'
+  tags:
+    - install
+    - install:base
+    - install:plugins
+    - install:jenkins-configuration
+
+- name: Copy credentials into files
+  copy:
+    content: "{{ item.content }}"
+    dest: '{{ jenkins_common_config_path }}/credentials/{{ item.name }}'
+  with_items: '{{ JENKINS_SECRET_FILES_LIST }}'
+  no_log: yes
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Copy ec2 key
+  copy:
+    content: '{{ JENKINS_EC2_PRIVATE_KEY }}'
+    dest: '{{ jenkins_common_config_path }}/ec2/id_rsa'
+  no_log: yes
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+
+- name: Start Jenkins Service
+  systemd:
+    name: jenkins
+    daemon_reload: yes
+    state: restarted
+  tags:
+    - manage
+    - manage:start
+    - install:plugins
+    - install:jenkins-configuration

playbooks/roles/jenkins_common/templates/config/credentials.yml.j2

+---
+{% for file in JENKINS_SECRET_FILES_LIST %}
+- credentialType: 'secretFile'
+  scope: '{{ file.scope }}'
+  name: '{{ file.name }}'
+  path: 'credentials/{{ file.name }}'
+  description: '{{ file.description }}'
+  id: '{{ file.id }}'
+{% endfor %}
+{% for userPass in JENKINS_USERNAME_PASSWORD_LIST %}
+- credentialType: 'usernamePassword'
+  scope: '{{ userPass.scope }}'
+  username: '{{ userPass.username }}'
+  password: '{{ userPass.password }}'
+  description: '{{ userPass.password }}'
+  id: '{{ userPass.id }}'
+{% endfor %}
+{% for text in JENKINS_SECRET_TEXT_LIST %}
+- credentialType: 'secretText'
+  scope: '{{ text.scope }}'
+  secretText: '{{ text.secretText }}'
+  description: '{{ text.description }}'
+  id: '{{ text.id }}'
+{% endfor %}
+{% for cert in JENKINS_CERTIFICATES_LIST %}
+- credentialType: 'certificate'
+  scope: '{{ cert.scope }}'
+  path: '{{ cert.path }}'
+  password: ''{{ cert.password }}'
+  description: '{{ cert.description }}'
+  id: '{{ cert.id }}'
+{% endfor %}
+{% for ssh in JENKINS_SSH_LIST %}
+- credentialType: 'ssh'
+  scope: '{{ ssh.scope }}'
+  username: '{{ ssh.username }}'
+  isJenkinsMasterSsh: '{{ ssh.isJenkinsMasterSsh}}'
+{% if not isJenkinsMasterSsh %}
+  path: '{{ ssh.path }}'
+{% endif %}
+  passphrase: '{{ ssh.passphrase }}'
+  description: '{{ ssh.description }}'
+{% endfor %}

playbooks/roles/jenkins_common/templates/config/ec2_config.yml.j2

+---
+CLOUDS:
+    - NAME: '{{ JENKINS_EC2_REGION }}'
+      ACCESS_KEY_ID: '{{ JENKINS_EC2_ACCESS_KEY_ID }}'
+      SECRET_ACCESS_KEY: '{{ JENKINS_EC2_SECRET_ACCESS_KEY }}'
+      USE_INSTANCE_PROFILE_FOR_CREDS: '{{ jenkins_common_use_instance_profile_for_creds }}'
+      REGION: '{{ JENKINS_EC2_REGION }}'
+      EC2_PRIVATE_KEY_PATH: '{{ jenkins_common_config_path }}/ec2/id_rsa'
+      INSTANCE_CAP: '{{ jenkins_common_instance_cap }}'
+      AMIS:
+{% for ami in JENKINS_EC2_AMIS %}
+          - AMI_ID: '{{ ami.AMI_ID }}'
+            AVAILABILITY_ZONE: '{{ ami.AVAILABILITY_ZONE }}'
+            SPOT_CONFIG:
+                SPOT_MAX_BID_PRICE: '{{ ami.SPOT_CONFIG.SPOT_MAX_BID_PRICE }}'
+                SPOT_INSTANCE_BID_TYPE: '{{ ami.SPOT_CONFIG.SPOT_INSTANCE_BID_TYPE }}'
+            SECURITY_GROUPS: '{{ ami.SECURITY_GROUPS }}'
+            REMOTE_FS_ROOT: '{{ ami.REMOTE_FS_ROOT }}'
+            SSH_PORT: '{{ ami.SSH_PORT }}'
+            INSTANCE_TYPE: '{{ ami.INSTANCE_TYPE }}'
+            LABEL_STRING: '{{ ami.LABEL_STRING }}'
+            MODE: '{{ ami.MODE }}'
+            DESCRIPTION: '{{ ami.DESCRIPTION }}'
+            INIT_SCRIPT_PATH: '{{ ami.INIT_SCRIPT_PATH }}'
+            TEMP_DIR: '{{ ami.TEMP_DIR }}'
+            USER_DATA: '{{ ami.USER_DATA }}'
+            NUM_EXECUTORS: '{{ ami.NUM_EXECUTORS }}'
+            REMOTE_ADMIN: '{{ ami.REMOTE_ADMIN }}'
+            ROOT_COMMAND_PREFIX: '{{ ami.ROOT_COMMAND_PREFIX }}'
+            JVM_OPTIONS: '{{ ami.JVM_OPTIONS }}'
+            STOP_ON_TERMINATE: {{ ami.STOP_ON_TERMINATE }}
+            SUBNET_ID: '{{ ami.SUBNET_ID }}'
+            TAGS:
+{% for tag in ami.TAGS %}
+                - NAME: '{{ tag.NAME }}'
+                  VALUE: '{{ tag.VALUE }}'
+{% endfor %}
+            IDLE_TERMINATION_MINUTES: '{{ ami.IDLE_TERMINATION_MINUTES }}'
+            USE_PRIVATE_DNS_NAME: {{ ami.USE_PRIVATE_DNS_NAME }}
+            INSTANCE_CAP: '{{ ami.INSTANCE_CAP }}'
+            IAM_INSTANCE_PROFILE: '{{ ami.IAM_INSTANCE_PROFILE }}'
+            USE_EPHEMERAL_DEVICES: {{ ami.USE_EPHEMERAL_DEVICES }}
+            LAUNCH_TIMEOUT: '{{ ami.LAUNCH_TIMEOUT }}'
+{% endfor %}

playbooks/roles/jenkins_common/templates/config/ghprb_config.yml.j2

+---
+SERVER_API_URL: '{{ jenkins_common_ghprb_server }}'
+ACCESS_TOKEN: '{{ JENKINS_GHPRB_TOKEN }}'
+ADMIN_LIST:
+{% for admin in JENKINS_GHPRB_ADMIN_LIST %}
+    - '{{ admin }}'
+{% endfor %}
+REQUEST_TESTING_PHRASE: '{{ jenkins_common_ghprb_request_testing }}'
+WHITE_LIST_PHRASE: '{{ jenkins_common_ghprb_white_list_phrase }}'
+OK_PHRASE: '{{ jenkins_common_ghprb_ok_phrase }}'
+RETEST_PHRASE: '{{ jenkins_common_ghprb_retest_phrase }}'
+SKIP_PHRASE: '{{ jenkins_common_ghprb_skip_phrase }}'
+CRON_SCHEDULE: '{{ jenkins_common_ghprb_cron_schedule }}'
+USE_COMMENTS: {{ jenkins_common_ghprb_use_comments }}
+USE_DETAILED_COMMENTS: {{ jenkins_common_ghprb_use_detailed_comments }}
+MANAGE_WEBHOOKS:  {{ jenkins_common_ghprb_manage_webhooks }}
+UNSTABLE_AS: '{{ jenkins_common_ghprb_failure_as }}'
+AUTO_CLOSE_FAILED_PRS: {{ jenkins_common_ghprb_auto_close_fails }}
+DISPLAY_ERRORS_DOWNSTREAM: {{ jenkins_commmon_ghprb_display_errors }}
+BLACK_LIST_LABELS:
+{% for blacklist in JENKINS_GHPRB_BLACK_LIST %}
+    - '{{ blacklist }}'
+{% endfor %}
+WHITE_LIST_LABELS:
+{% for whitelist in JENKINS_GHPRB_WHITE_LIST %}
+    - '{{ whitelist }}'
+{% endfor %}
+GITHUB_AUTH: '{{ jenkins_common_ghprb_github_auth }}'
+SIMPLE_STATUS: '{{ jenkins_common_ghprb_simple_status }}'
+PUBLISH_JENKINS_URL: '{{ jenkins_common_ghprb_publish_jenkins_url }}'
+BUILD_LOG_LINES_TO_DISPLAY: {{ jenkins_common_ghprb_build_log_lines }}
+RESULT_MESSAGES:
+{% for message in jenkins_common_ghprb_results %}
+    - STATUS: '{{ message.STATUS }}'
+      MESSAGE: '{{ message.MESSAGE }}'
+{% endfor %}

playbooks/roles/jenkins_common/templates/config/git_config.yml.j2

+---
+NAME: '{{ JENKINS_GIT_NAME }}'
+EMAIL: '{{ JENKINS_GIT_EMAIL }}'

playbooks/roles/jenkins_common/templates/config/hipchat_config.yml.j2

+---
+API_TOKEN: '{{ JENKINS_HIPCHAT_API_TOKEN }}'
+ROOM: '{{ jenkins_common_hipchat_room }}'

playbooks/roles/jenkins_common/templates/config/jenkins_config_history.yml.j2

+---
+HISTORY_ROOT_DIR: '{{ jenkins_common_history_root }}'
+MAX_HISTORY_ENTRIES: '{{ jenkins_common_history_max_entries }}'
+MAX_DAYS_TO_KEEP_ENTRIES: '{{ jenkins_common_history_max_days }}
+MAX_ENTRIES_PER_PAGE: '{{ jenkins_common_history_max_entries_page }}
+SKIP_DUPLICATE_HISTORY: '{{ jenkins_common_history_skip_duplicates }}'
+EXCLUDE_PATTERN: '{{ jenkins_common_history_exclude_pattern }}'
+SAVE_MODULE_CONFIGURATION: '{{ jenkins_common_history_save_module_config }}'
+SHOW_BUILD_BADGES: '{{ jenkins_common_history_show_build_badges }}'
+EXCLUDED_USERS: '{{ jenkins_common_history_excluded_users }}'

playbooks/roles/jenkins_common/templates/config/log_config.yml.j2

+---
+{% for recorder in jenkins_common_log_list %}
+- LOG_RECORDER: '{{ recorder.LOG_RECORDER }}'
+  LOGGERS:
+{% for log in recorder.LOGGERS %}
+    - name: '{{ log.name }}'
+      log_level: '{{ log.log_level }}'
+{% endfor %}
+{% endfor %}

playbooks/roles/jenkins_common/templates/config/mailer_config.yml.j2

+---
+SMTP_SERVER: '{{ JENKINS_MAILER_SMTP_SERVER }}'
+REPLY_TO_ADDRESS: '{{ JENKINS_MAILER_REPLY_TO_ADDRESS }}'
+DEFAULT_SUFFIX: '{{ JENKINS_MAILER_DEFAULT_SUFFIX }}'
+SMTP_AUTH_USERNAME: '{{ JENKINS_MAILER_SMTP_AUTH_USERNAME }}'
+SMTP_AUTH_PASSWORD: '{{ JENKINS_MAILER_SMTP_AUTH_PASSWORD }}'
+SMTP_PORT: '{{ jenkins_common_mailer_port }}'
+USE_SSL: '{{ jenkins_common_mailer_use_ssl }}'
+CHAR_SET: '{{ jenkins_common_mailer_char_set }}'

playbooks/roles/jenkins_common/templates/config/main_config.yml.j2

+---
+MAIN:
+    WORKSPACE_ROOT_DIR: '${ITEM_ROOTDIR}/workspace'
+    BUILD_RECORD_ROOT_DIR: '${ITEM_ROOTDIR}/builds'
+    SYSTEM_MESSAGE: '{{ jenkins_common_main_system_message }}'
+    NUMBER_OF_EXECUTORS: {{ jenkins_common_main_num_executors }}
+    LABELS:
+{% for label in jenkins_common_main_labels %}
+        - '{{ label }}'
+{% endfor %}
+    USAGE: 'EXCLUSIVE'
+    QUIET_PERIOD: {{ jenkins_common_main_quiet_period }}
+    SCM_RETRY_COUNT: {{ jenkins_common_main_scm_retry }}
+    DISABLE_REMEMBER_ME: {{ jenkins_common_main_disable_remember }}
+GLOBAL_PROPERTIES:
+    ENVIRONMENT_VARIABLES:
+{% for env in jenkins_common_main_env_vars %}
+        - NAME: '{{ env.NAME }}'
+          VALUE: '{{ env.VALUE }}'
+{% endfor %}
+    TOOL_LOCATIONS:
+LOCATION:
+    URL: '{{ JENKINS_MAIN_URL }}'
+    ADMIN_EMAIL: '{{ JENKINS_MAIN_ADMIN_EMAIL }}'
+SHELL:
+    EXECUTABLE: '{{ jenkins_common_main_executable }}'
+CLI:
+    CLI_ENABLED: false
+

playbooks/roles/jenkins_common/templates/config/plugins.yml.j2

+---
+{% for plugin in jenkins_common_plugins_list %}
+- name: '{{ plugin.name }}'
+  version: '{{ plugin.version }}'
+  group: '{{ plugin.group }}'
+{% endfor %}
+

playbooks/roles/jenkins_common/templates/config/security.yml.j2

+---
+OAUTH_SETTINGS:
+  GITHUB_WEB_URI: 'https://github.com'
+  GITHUB_API_URI: 'https://api.github.com'
+  CLIENT_ID: '{{ JENKINS_SECURITY_CLIENT_ID }}'
+  CLIENT_SECRET: '{{ JENKINS_SECURITY_CLIENT_SECRET }}'
+  SCOPES: '{{ jenkins_common_security_scopes }}'
+SECURITY_GROUPS:
+{% for group in JENKINS_SECURITY_GROUPS %}
+  - NAME: '{{ group.NAME }}'
+    PERMISSIONS:
+{% for permission in group.PERMISSIONS %}
+      - {{ permission }}
+{% endfor %}
+    USERS:
+{% for user in group.USERS %}
+      - {{ user }}
+{% endfor %}
+{% endfor %}

playbooks/roles/jenkins_common/templates/config/seed_config.yml.j2

+---
+NAME: '{{ jenkins_common_seed_name }}'
+XML_PATH: '{{ jenkins_common_seed_path }}'

playbooks/roles/jenkins_common/templates/etc/systemd/system/jenkins.service.j2

+[Unit]
+Description=Jenkins
+
+[Service]
+Environment=JENKINS_HOME={{ jenkins_common_home }}
+Environment=JENKINS_CONFIG_PATH={{ jenkins_common_config_path }}
+ExecStart=/usr/bin/java -jar /usr/share/jenkins/jenkins.war --httpPort={{ jenkins_common_port }}
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/jenkins_common/templates/jenkins-env.sh.j2

+export JENKINS_HOME='{{ jenkins_common_home }}'
+export JENKINS_CONFIG_PATH='{{ jenkins_common_config_path }}'
+export JENKINS_VERSION='{{ jenkins_common_version }}'
+export JENKINS_WAR_SOURCE='{{ jenkins_common_war_source}}'

util/jenkins/cloudflare-hit-rate.py

@@ -30,7 +30,7 @@ def calcualte_cache_hit_rate(zone_id, auth_key, email, threshold):
             sys.exit(1)
 
     except Exception as error:
-        print "JSON Error: {} \n Content returned from API call: {}".format(error, res.text)
+        print("JSON Error: {} \n Content returned from API call: {}".format(error, res.text))
 
 
 

util/parsefiles.py

@@ -9,9 +9,11 @@ import argparse
 
 TRAVIS_BUILD_DIR = os.environ.get("TRAVIS_BUILD_DIR")
 DOCKER_PATH_ROOT = pathlib2.Path(TRAVIS_BUILD_DIR, "docker", "build")
+DOCKER_PLAYS_PATH = pathlib2.Path(TRAVIS_BUILD_DIR, "docker", "plays")
 CONFIG_FILE_PATH = pathlib2.Path(TRAVIS_BUILD_DIR, "util", "parsefiles_config.yml")
 LOGGER = logging.getLogger(__name__)
 
+
 def build_graph(git_dir, roles_dirs, aws_play_dirs, docker_play_dirs):
     """
     Builds a dependency graph that shows relationships between roles and playbooks.
@@ -149,6 +151,7 @@ def _open_yaml_file(file_str):
             LOGGER.error("error in configuration file: %s" % str(exc))
             sys.exit(1)
 
+
 def change_set_to_roles(files, git_dir, roles_dirs, playbooks_dirs, graph):
     """
     Converts change set consisting of a number of files to the roles that they represent/contain.
@@ -181,6 +184,7 @@ def change_set_to_roles(files, git_dir, roles_dirs, playbooks_dirs, graph):
                 items.add(_get_role_name_from_file(file_path))
     return items
 
+
 def get_plays(files, git_dir, playbooks_dirs):
     """
     Determines which files in the change set are aws playbooks
@@ -211,6 +215,7 @@ def get_plays(files, git_dir, playbooks_dirs):
 
     return plays
 
+
 def _get_playbook_name_from_file(path):
     """
     Gets name of playbook from the filepath, which is the last part of the filepath.
@@ -235,6 +240,7 @@ def _get_role_name_from_file(path):
     # name of role is the next part of the file path after "roles"
     return dirs[dirs.index("roles")+1]
 
+
 def get_dependencies(roles, graph):
     """
     Determines all roles dependent on set of roles and returns set containing both.
@@ -257,6 +263,7 @@ def get_dependencies(roles, graph):
 
     return items
 
+
 def get_docker_plays(roles, graph):
     """Gets all docker plays that contain at least role in common with roles."""
 
@@ -291,6 +298,7 @@ def get_docker_plays(roles, graph):
 
     return items
 
+
 def filter_docker_plays(plays, repo_path):
     """Filters out docker plays that do not have a Dockerfile."""
 
@@ -306,6 +314,7 @@ def filter_docker_plays(plays, repo_path):
 
     return items
 
+
 def _get_role_name(role):
     """
     Resolves a role name from either a simple declaration or a dictionary style declaration.
@@ -347,6 +356,23 @@ def _get_modified_dockerfiles(files, git_dir):
 
             if play is not None:
                 items.add(play)
+
+    return items
+
+
+def get_modified_dockerfiles_plays(files, git_dir):
+    """
+    Return changed files under docker/plays directory
+    :param files:
+    :param git_dir:
+    :return:
+    """
+    items = set()
+    candidate_files = {f for f in DOCKER_PLAYS_PATH.glob("*.yml")}
+    for f in files:
+        file_path = pathlib2.Path(git_dir, f)
+        if file_path in candidate_files:
+            items.add(_get_playbook_name_from_file(file_path))
     return items
 
 
@@ -372,6 +398,7 @@ def _get_play_name(path):
         return suffix_parts[0]
     return None
 
+
 def arg_parse():
 
     parser = argparse.ArgumentParser(description = 'Given a commit range, analyze Ansible dependencies between roles and playbooks '
@@ -432,5 +459,9 @@ if __name__ == '__main__':
     # Add playbooks to the list whose docker file has been modified
     modified_docker_files = _get_modified_dockerfiles(change_set, TRAVIS_BUILD_DIR)
 
-    all_plays = set(set(docker_plays) | set( modified_docker_files))
+    # Add plays to the list which got changed in docker/plays directory
+    docker_plays_dir = get_modified_dockerfiles_plays(change_set, TRAVIS_BUILD_DIR)
+
+    all_plays = set(set(docker_plays) | set( modified_docker_files) | set(docker_plays_dir))
+
     print " ".join(all_plays)