install and configure splunk plugin

playbooks/edx-east/jenkins_build.yml

@@ -12,57 +12,10 @@
   gather_facts: True
   vars:
     COMMON_ENABLE_DATADOG: False
-    COMMON_ENABLE_SPLUNKFORWARDER: True
     COMMON_ENABLE_NEWRELIC: True
     COMMON_SECURITY_UPDATES: yes
     SECURITY_UPGRADE_ON_ANSIBLE: true
 
-    SPLUNKFORWARDER_LOG_ITEMS:
-      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/junitResult.xml'
-        recursive: true
-        index: 'testeng'
-        sourcetype: junit
-        followSymlink: false
-        blacklist: coverage|private|subset|specific|custom|special|\.gz$
-        crcSalt: '<SOURCE>'
-
-      - source: '/var/lib/jenkins/jobs/*/builds/*/build.xml'
-        index: 'testeng'
-        recursive: true
-        sourcetype: build_result
-        followSymlink: false
-        crcSalt: '<SOURCE>'
-        blacklist: '\.gz$'
-
-      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/.../test_root/log/timing.*.log'
-        index: 'testeng'
-        recursive: true
-        sourcetype: 'json_timing_log'
-        followSymlink: false
-        crcSalt: '<SOURCE>'
-        blacklist: coverage|private|subset|specific|custom|special|\.gz$
-
-      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/test_root/log/timing.*.log'
-        index: 'testeng'
-        recursive: true
-        sourcetype: 'json_timing_log'
-        followSymlink: false
-        crcSalt: '<SOURCE>'
-        blacklist: coverage|private|subset|specific|custom|special|\.gz$
-
-      - source: '/var/lib/jenkins/jobs/count-workers/builds/*/log'
-        index: 'testeng'
-        recursive: true
-        sourcetype: 'count_worker_log'
-        followSymlink: false
-        crcSalt: '<SOURCE>'
-
-      - source: '/var/log/jenkins/jenkins.log'
-        index: 'testeng'
-        recursive: false
-        followSymlink: false
-        blacklist: '\.gz$'
-
   roles:
     - aws
     - role: datadog
@@ -73,15 +26,6 @@
 
     - jenkins_build
 
-    # run just the splunkforwarder role by using '--tags "splunkonly"'
-    # e.g. ansible-playbook jenkins_testeng_master.yml -i inventory.ini --tags "splunkonly" -vvvv
-    - role: splunkforwarder
-      when: COMMON_ENABLE_SPLUNKFORWARDER
-      tags:
-        - splunkonly
-        - jenkins:promote-to-production
-      become: True
-
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC
       tags:

playbooks/roles/jenkins_build/defaults/main.yml

@@ -18,6 +18,7 @@ build_jenkins_configuration_scripts:
   - 4configureJobConfigHistory.groovy
   - 4configureMailerPlugin.groovy
   - 4configureMaskPasswords.groovy
+  - 4configureSplunk.groovy
   - 5createLoggers.groovy
 
 # plugins
@@ -190,6 +191,9 @@ build_jenkins_plugins_list:
   - name: 'shiningpanda'
     version: '0.21'
     group: 'org.jenkins-ci.plugins'
+  - name: 'splunk-devops'
+    version: '1.6.4'
+    group: 'com.splunk.splunkins'
   - name: 'ssh-agent'
     version: '1.14'
     group: 'org.jenkins-ci.plugins'
@@ -279,3 +283,28 @@ build_jenkins_log_list:
 # job config history
 build_jenkins_history_max_days: '15'
 build_jenkins_history_exclude_pattern: 'queue|nodeMonitors|UpdateCenter|global-build-stats|GhprbTrigger'
+
+# splunk
+build_jenkins_splunk_ignored_jobs: '(^((?!edx-(platform|e2e)).)*$)|.*private.*'
+build_jenkins_splunk_script_type: 'inline'
+build_jenkins_splunk_script_path: 'splunk/splunk.groovy'
+build_jenkins_splunk_file_path: '{{ role_path }}/../jenkins_build/files/splunk/splunk.groovy'
+build_jenkins_splunk_metadata:
+  - data_source: 'Default'
+    config_item: 'Source Type'
+    value: 'json:jenkins:old'
+  - data_source: 'Build Report'
+    config_item: 'Source Type'
+    value: 'build_report'
+  - data_source: 'Build Event'
+    config_item: 'Source Type'
+    value: 'build_event'
+  - data_source: 'Console Log'
+    config_item: 'Source Type'
+    value: 'console_log'
+  - data_source: 'Queue Information'
+    config_item: 'Source Type'
+    value: 'queue_information'
+  - data_source: 'Slave Information'
+    config_item: 'Source Type'
+    value: 'slave_information'

playbooks/roles/jenkins_build/files/splunk/splunk.groovy

+//send job metadata and junit reports with page size set to 50 (each event contains max 50 test cases)
+splunkins.sendTestReport(50)
+
+
+// Send paver timing logs to Splunk
+splunkins.archive("**/timing*.log", null, false, "10MB")

playbooks/roles/jenkins_build/meta/main.yml

@@ -22,3 +22,11 @@ dependencies:
     jenkins_common_history_max_days: '{{ build_jenkins_history_max_days }}'
     jenkins_common_history_exclude_pattern: '{{ build_jenkins_history_exclude_pattern }}'
     jenkins_common_server_name: '{{ JENKINS_SERVER_NAME }}'
+    jenkins_common_splunk_ignored_jobs: '{{ build_jenkins_splunk_ignored_jobs }}'
+    jenkins_common_splunk_script_type: '{{ build_jenkins_splunk_script_type }}'
+    jenkins_common_splunk_script_path: '{{ build_jenkins_splunk_script_path }}'
+    jenkins_common_splunk_metadata: '{{ build_jenkins_splunk_metadata }}'
+    jenkins_common_splunk_master_hostname:  '{{ build_jenkins_splunk_master_hostname }}'
+    jenkins_common_splunk_event_source: '{{ build_jenkins_splunk_event_source }}'
+    jenkins_common_splunk_enabled: '{{ BUILD_JENKINS_SPLUNK_ENABLED }}'
+    jenkins_common_splunk_file_path: '{{ build_jenkins_splunk_file_path }}'

playbooks/roles/jenkins_common/defaults/main.yml

@@ -39,6 +39,7 @@ jenkins_common_non_plugin_template_files:
   - properties_config
   - security
   - seed_config
+  - splunk_config
 
 # Jenkins default config values
 jenkins_common_jvm_args: ''
@@ -183,3 +184,24 @@ JENKINS_MAILER_SMTP_AUTH_PASSWORD: ''
 # mask passwords
 JENKINS_MASK_PASSWORDS_CLASSES: []
 JENKINS_MASK_PASSWORDS_PAIRS: []
+
+# splunk
+jenkins_common_splunk_enabled: true
+jenkins_common_splunk_use_ssl: true
+jenkins_common_splunk_raw_events_enabled: true
+jenkins_common_splunk_batch_size: 4096
+jenkins_common_splunk_retries: 3
+jenkins_common_splunk_ignored_jobs: ''
+jenkins_common_splunk_master_hostname: ''
+jenkins_common_splunk_event_source: ''
+jenkins_common_splunk_script_type: 'path'
+jenkins_common_splunk_script_path: ''
+jenkins_common_splunk_file_path: '{{ role_path }}/files/splunk/*'
+jenkins_common_splunk_metadata:
+    - data_source: 'Default'
+      config_item: 'Source Type'
+      value: 'json:jenkins:old'
+JENKINS_SPLUNK_HOSTNAME: ''
+JENKINS_SPLUNK_PORT: 8088
+JENKINS_SPLUNK_APP_URL: ''
+JENKINS_SPLUNK_TOKEN: ''

playbooks/roles/jenkins_common/files/splunk/splunk.groovy

+//send job metadata and junit reports with page size set to 50 (each event contains max 50 test cases)
+splunkins.sendTestReport(50)

playbooks/roles/jenkins_common/tasks/main.yml

@@ -192,6 +192,7 @@
     - '{{ jenkins_common_config_path }}/credentials'
     - '{{ jenkins_common_config_path }}/ec2'
     - '{{ jenkins_common_config_path }}/xml'
+    - '{{ jenkins_common_config_path }}/splunk'
   tags:
      - install
      - install:base
@@ -273,6 +274,20 @@
     - install:jenkins-configuration
     - jenkins:local-dev
 
+- name: Copy splunk config script
+  template:
+    src: '{{ item }}'
+    dest: '{{ jenkins_common_config_path }}/splunk/'
+    owner: '{{ jenkins_common_user }}'
+    group: '{{ jenkins_common_group }}'
+  with_fileglob:
+    - '{{ jenkins_common_splunk_file_path }}'
+  tags:
+    - install
+    - install:base
+    - install:jenkins-configuration
+    - jenkins:local-dev
+
 - name: Run plugins.gradle
   shell: './gradlew -b plugins.gradle plugins'
   args:

playbooks/roles/jenkins_common/templates/config/splunk_config.yml.j2

+---
+SPLUNK_APP_ENABLED: {{ jenkins_common_splunk_enabled }}
+SPLUNK_HOSTNAME: '{{ JENKINS_SPLUNK_HOSTNAME }}'
+SPLUNK_HOST_PORT: {{ JENKINS_SPLUNK_PORT }}
+USE_SSL: {{ jenkins_common_splunk_use_ssl }}
+SPLUNK_TOKEN: '{{ JENKINS_SPLUNK_TOKEN }}'
+RAW_EVENTS_ENABLED: {{ jenkins_common_splunk_raw_events_enabled }}
+MAX_EVENT_BATCH_SIZE: {{ jenkins_common_splunk_batch_size }}
+SPLUNK_APP_URL: '{{ JENKINS_SPLUNK_APP_URL }}'
+RETRIES_ON_ERROR: {{ jenkins_common_splunk_retries }}
+IGNORED_JOBS_PATTERN: {{ jenkins_common_splunk_ignored_jobs }}
+MASTER_HOSTNAME: '{{ jenkins_common_splunk_master_hostname }}'
+EVENT_SOURCE: '{{ jenkins_common_splunk_event_source }}'
+SCRIPT_TYPE: '{{ jenkins_common_splunk_script_type }}'
+SCRIPT_PATH: '{{ jenkins_common_splunk_script_path }}'
+METADATA:
+{% for metadata in jenkins_common_splunk_metadata %}
+  - DATA_SOURCE: '{{ metadata.data_source }}'
+    CONFIG_ITEM: '{{ metadata.config_item }}'
+    VALUE: '{{ metadata.value }}'
+{% endfor %}