Merge pull request #1878 from edx/zoldak/splunk-jenkins

Add splunk forwarder role to jenkins.
Merging in as-is for now to get it in the codebase. I think the inputs.conf values for the files that we're monitoring still need to be tweaked a bit. I'll put that in a subsequent PR. 
@benpatterson fwiw I did end up getting an initial stab at the build logs into this one.

playbooks/edx-east/jenkins_testeng_master.yml

@@ -9,8 +9,36 @@
   vars:
     COMMON_DATA_DIR: "/mnt"
     COMMON_ENABLE_DATADOG: True
+    COMMON_ENABLE_SPLUNKFORWARDER: True
+
+    SPLUNKFORWARDER_LOG_ITEMS:
+      - source: '/var/lib/jenkins/jobs/*/builds/.../junitResult.xml'
+        recursive: true
+        index: 'testeng'
+        sourcetype: junit
+        followSymlink: false
+      - source: '/var/lib/jenkins/jobs/*/builds/.../build.xml'
+        index: 'testeng'
+        recursive: true
+        sourcetype: build_result
+        followSymlink: false
+        initCrcLength: 1024
+      - source: '/var/lib/jenkins/jobs/*/builds/.../log'
+        index: 'testeng'
+        recursive: true
+        sourcetype: build_log
+        followSymlink: false
+        initCrcLength: 2048
+
   roles:
     - common
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - jenkins_master
+
+    # run just the splunkforwarder role by using '--tags "splunkonly"'
+    # e.g. ansible-playbook jenkins_testeng_master.yml -i inventory.ini --tags "splunkonly" -vvvv
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+      tags: splunkonly
+      sudo: True

playbooks/roles/splunkforwarder/templates/opt/splunkforwarder/etc/system/local/inputs.conf.j2

@@ -12,4 +12,10 @@ index = {{ loggable.index }}
 {% if loggable._TCP_ROUTING is defined %}
 _TCP_ROUTING = {{ loggable._TCP_ROUTING }}
 {% endif %}
+{% if loggable.followSymlink is defined %}
+followSymlink = {{ loggable.followSymlink }}
+{% endif %}
+{% if loggable.initCrcLength is defined %}
+initCrcLength = {{ loggable.initCrcLength }}
+{% endif %}
 {% endfor %}