Merge pull request #3152 from edx/max/jenkins-splunk

Add jenkins to splunk

playbooks/edx-east/jenkins_admin.yml

@@ -9,3 +9,12 @@
   roles:
     - aws
     - jenkins_admin
+    # This requires an override of the following form:
+    # SPLUNKFORWARDER_LOG_ITEMS:
+    #   - source: /edx/var/jenkins/jobs/*/builds/*/log
+    #     index: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-jenkins'
+    #     sourcetype: jenkins_build
+    #     followSymlink: false
+    #     crcSalt: <SOURCE>
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER

playbooks/edx-east/jenkins_tools.yml

@@ -9,3 +9,12 @@
   roles:
     - aws
     - tools_jenkins
+    # This requires an override of the following form:
+    # SPLUNKFORWARDER_LOG_ITEMS:
+    #   - source: /edx/var/jenkins/jobs/*/builds/*/log
+    #     index: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-jenkins'
+    #     sourcetype: jenkins_build
+    #     followSymlink: false
+    #     crcSalt: <SOURCE>
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER

playbooks/roles/splunkforwarder/tasks/main.yml

@@ -84,7 +84,7 @@
   notify: restart splunkforwarder
 
 - name: Update admin pasword
-  shell: "{{ splunkforwarder_output_dir }}/bin/splunk edit user admin -password {{ SPLUNKFORWARDER_PASSWORD }} -auth admin:changeme --accept-license --answer-yes --no-prompt"
+  shell: "{{ splunkforwarder_output_dir }}/bin/splunk edit user admin -password '{{ SPLUNKFORWARDER_PASSWORD }}' -auth admin:changeme --accept-license --answer-yes --no-prompt"
   when: download_deb.changed
   notify: restart splunkforwarder
 

playbooks/roles/splunkforwarder/templates/opt/splunkforwarder/etc/system/local/inputs.conf.j2

@@ -3,7 +3,11 @@
 {% for loggable in SPLUNKFORWARDER_LOG_ITEMS%}
 [monitor://{{ loggable.source }}]
 blacklist = \.(gz)$
-recursive = {{ loggable.recursive|default(false) }}
+{% if loggable.recursive | default(False) %}
+{# There's a bug in which "recursive" must be unset for logs to be forwarded #}
+{# See https://answers.splunk.com/answers/420901/splunk-not-matching-files-with-wildcard-in-monitor.html #}
+recursive = true
+{% endif %}
 {% if loggable.sourcetype is defined %}
 sourcetype = {{ loggable.sourcetype }}
 {% endif %}