Merge pull request #4167 from edx/aali/OPS-1703_splunk_playbook_edx

splunk changes for edx

playbooks/roles/splunk-server/defaults/main.yml

@@ -9,13 +9,12 @@
 #
 ##
 # Defaults for role splunk-server
-# 
+#
 
 #
 # vars are namespaced with the module name.
 #
-SPLUNK_INDEXES: 
-  - "default"
+SPLUNK_INDEXES: []
 
 SPLUNK_ALERTS: []
   # A list of dicts with the following keys:
@@ -83,6 +82,7 @@ SPLUNK_SMTP_USERNAME: username
 SPLUNK_SMTP_PASSWORD: password
 SPLUNK_FROM_ADDRESS: no-reply@example.com
 SPLUNK_EMAIL_FOOTER: Generated by {{ SPLUNK_HOSTNAME }}
+SPLUNK_SSL_HOSTNAME: splunk.example.com:443
 
 # SSL settings. Either all or none of these must be defined.
 # For more details about setting up splunk with SSL, see
@@ -94,7 +94,7 @@ SPLUNK_SSL_ROOT_CA: !!null
 splunk-server_role_name: splunk-server
 
 splunk_user: "splunk"
-splunk_root: "/vol/splunk/storage"
+splunk_root: "/vol/splunk"
 splunk_home: "/opt/splunk"
 
 splunk_hot_dir: "{{ splunk_root }}/hot"

playbooks/roles/splunk-server/tasks/main.yml

@@ -55,7 +55,7 @@
     dest: "{{ splunk_home }}/etc/system/local/inputs.conf"
     owner: splunk
     group: splunk
-    mode: "0644"
+    mode: "0600"
 
 - name: Create bucket directories
   file:
@@ -90,7 +90,7 @@
     dest: "{{ splunk_home }}/etc/apps/search/local/indexes.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
-    mode: 0700
+    mode: 0600
   tags:
     - "install"
     - "install:configuration"
@@ -101,7 +101,7 @@
     dest: "{{ splunk_home }}/etc/system/local/alert_actions.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
-    mode: 0700
+    mode: 0600
   tags:
     - install
     - install:configuration
@@ -112,7 +112,7 @@
     dest: "{{ splunk_home }}/etc/apps/search/local/savedsearches.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
-    mode: 0700
+    mode: 0600
   tags:
     - "install"
     - "install:configuration"

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/indexes.conf.j2

-{% for name in SPLUNK_INDEXES %}
-[{{ name }}]
-coldPath = {{ splunk_cold_dir }}/{{ name }}/colddb
-homePath = {{ splunk_hot_dir }}/{{ name }}/db
-thawedPath = {{ splunk_thawed_dir }}/{{ name }}/thaweddb
-coldToFrozenDir = {{ splunk_frozen_dir }}/{{ name }}/frozendb
+{% for index in SPLUNK_INDEXES %}
+[{{ index.name }}]
+{% if index.coldPath is defined %}
+coldPath = {{ index.coldPath }}
+{% else %}
+coldPath = {{ splunk_cold_dir }}/{{ index.name }}/colddb
+{% endif %}
+{% if index.homePath is defined %}
+homePath = {{ index.homePath }}
+{% else %}
+homePath = {{ splunk_hot_dir }}/{{ index.name }}/db
+{% endif %}
+{% if index.maxTotalDataSizeMB is defined %}
+maxTotalDataSizeMB = {{ index.maxTotalDataSizeMB }}
+{% endif %}
+{% if index.thawedPath is defined %}
+thawedPath = {{ index.thawedPath }}
+{% else %}
+thawedPath = {{ splunk_thawed_dir }}/{{ index.name }}/thaweddb
+{% endif %}
+{% if index.coldToFrozenDir is not defined %}
+coldToFrozenDir = {{ splunk_frozen_dir }}/{{ index.name }}/frozendb
+{% endif %}
+{% if index.disabled is defined %}
+disabled = {{ index.disabled }}
+{% endif %}
+{% if index.home is defined %}
+home = {{ index.home }}
+{% endif %}
+{% if index.enableDataIntegrityControl is defined %}
+enableDataIntegrityControl = {{ index.enableDataIntegrityControl }}
+{% endif %}
+{% if index.enableTsidxReduction is defined %}
+enableTsidxReduction = {{ index.enableTsidxReduction }}
+{% endif %}
 
 {% endfor %}
+

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/props.conf.j2

@@ -4,6 +4,43 @@
 {% elif 'sourcetype' in extraction %}
 [{{ extraction.sourcetype }}]
 {% endif %}
+{% if extraction.break_before is defined%}
+BREAK_ONLY_BEFORE = {{ extraction.break_before }}
+{% endif %}
+{% if extraction.max_events is defined%}
+MAX_EVENTS = {{ extraction.max_events }}
+{% endif %}
+{% if extraction.datetime_config is defined %}
+DATETIME_CONFIG = {{ extraction.datetime_config }}
+{% endif %}
+{% if extraction.indexed_extractions is defined %}
+INDEXED_EXTRACTIONS = {{ extraction.indexed_extractions }}
+{% endif %}
+{% if extraction.no_binary_check is defined %}
+NO_BINARY_CHECK = {{ extraction.no_binary_check }}
+{% endif %}
+{% if extraction.timestamp_fields is defined %}
+TIMESTAMP_FIELDS = {{ extraction.timestamp_fields }}
+{% endif %}
+{% if extraction.category is defined %}
+category = {{ extraction.category }}
+{% endif %}
+{% if extraction.description is defined %}
+description = {{ extraction.description }}
+{% endif %}
+{% if extraction.disabled is defined %}
+disabled = {{ extraction.disabled }}
+{% endif %}
+{% if extraction.pulldown_type is defined %}
+pulldown_type = {{ extraction.pulldown_type }}
+{% endif %}
+{% if extraction.name is defined %}
 EXTRACT-{{ extraction.name }} = {{ extraction.regex }}
+{% endif %}
+{% if 'sourcetype' in extraction and extraction.sourcetype == "build_log" %}
+EXTRACT-run-results = Setting status of .* and message: 'Build finished. (?P<num_run>\d+) tests run, (?P<num_skipped>\d+) skipped, (?P<num_failed>\d+) failed.
+EXTRACT-error_msg = \n?ERROR: (?P<error_msg>[^\n]*)
+EXTRACT-buildResult = Finished: (?P<buildResult>[A-Z]+)$
+{% endif %}
 
 {% endfor %}

playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/alert_actions.conf.j2

 [email]
-mailserver = {{ SPLUNK_SMTP_SERVER }}
-pdf.header_left = none
-pdf.header_right = none
 auth_password = {{ SPLUNK_SMTP_PASSWORD }}
 auth_username = {{ SPLUNK_SMTP_USERNAME }}
 footer.text = {{ SPLUNK_EMAIL_FOOTER }}
-hostname = {{ SPLUNK_HOSTNAME }}
+hostname = {{ SPLUNK_SSL_HOSTNAME }}
+mailserver = {{ SPLUNK_SMTP_SERVER }}
+reportServerURL =
+use_tls = 1
+pdf.header_left = none
+pdf.header_right = none
+use_ssl = 0
 from = {{ SPLUNK_FROM_ADDRESS }}
-pdf.footer_enabled = 0
-pdf.header_enabled = 0
-use_tls = 1
\ No newline at end of file

playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/inputs.conf.j2

@@ -6,8 +6,8 @@ host = {{ SPLUNK_HOSTNAME }}
 compressed = true
 
 [SSL]
-serverCert = $SPLUNK_HOME/{{ splunk_cert_path }}/forwarder.pem
 password = {{ SPLUNK_SSL_PASSWORD }}
 requireClientCert = false
 rootCA = $SPLUNK_HOME/{{ splunk_cert_path }}/cacert.pem
-{% endif %}
\ No newline at end of file
+serverCert = $SPLUNK_HOME/{{ splunk_cert_path }}/forwarder.pem
+{% endif %}