add splunk searches via ansible (#3057)

* add splunk searches via ansible

* Fix jinja syntax, add docs

* Add splunk field extractions via ansible

* Use a boolean rather than a number

The fact that splunk uses "1" and "0" to represent booleans is
orthogonal to booleans in yaml. Switching to true booleans for a better
user experience.

* Additional cleanup around splunk searches interface

* Switch splunk role to using common_vars and overrides

This way, we can support environments where we don't, for example, use
datadog.

* Make splunk playbook actually call splunk role

* Fix all the bugs

* Add splunk email config

* Make realtime alerts work

* Add more documentation around splunk alerts

* Address comments

playbooks/edx-east/splunk.yml

+# Usage: ansible-playbook splunk.yml -e@/path/to/environment-deployment.yml
+
 - name: Deploy Splunk
   hosts: all
   sudo: True
   gather_facts: True
   vars:
-    COMMON_APP_DIR: "/edx/app"
-    common_web_group: "www-data"
-    ENABLE_DATADOG: True
-    ENABLE_SPLUNKFORWARDER: True
-    ENABLE_NEWRELIC: True 
   roles:
     - aws
-    - datadog
-    - splunkforwarder
-    - newrelic
+    - splunk-server
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/roles/splunk-server/defaults/main.yml

@@ -14,6 +14,70 @@
 #
 # vars are namespaced with the module name.
 #
+SPLUNK_INDEXES: 
+  - "default"
+
+SPLUNK_ALERTS: []
+  # A list of dicts with the following keys:
+  #
+  #   name: (string, required)
+  #     The name of the alert
+  #
+  #   description: (string, optional)
+  #     A description of the alert. Appears in the Splunk UI.
+  #
+  #   email: (list[string], optional)
+  #     List of email addresses to send to when alert is triggered
+  #
+  #   message: (string, optional)
+  #     Body of the alert email. You can include information from the alert via the tokens documented here:
+  #     http://docs.splunk.com/Documentation/Splunk/6.4.1/Alert/EmailNotificationTokens
+  #
+  #   search: (string, required)
+  #     Splunk query to use
+  #
+  #   schedule: (string, default: "*/15 * * * *")
+  #     The cron-style schedule on which to run the alert
+  #
+  #   counttype: ("number of events" | "number of hosts" | "number of sources" | "always", default: "number of events")
+  #
+  #   comparison: ("greater than" | "less than" | "equal to" | "not equal to" | "drops by" | "rises by", default: "greater than")
+  #
+  #   quantity: (number, default: 0)
+  #     Alert will trigger when "counttype comparison quantity" is true, e.g. "number of events > 10"
+  #
+  #   time: (string, default: "-15m")
+  #     Events will be searched from this value until now. "rt" indicates "realtime".
+  #
+  #   severity: ([1-6], {{splunk_alert_default_severity}})
+  #     The severity of the alert. 1-debug, 2-info, 3-warn, 4-error, 5-severe, 6-fatal
+  #
+  #   digest_delay: (string, optional)
+  #     Whether to send email digests at most every "digest_delay" rather than for every alert. e.g. 15m
+  #
+  # NB: None of the string values can contain newlines except "message"
+
+SPLUNK_FIELD_EXTRACTIONS: []
+  # A list of dicts of the following form.
+  # source and sourcetype are mutually exclusive
+  # - sourcetype | source:
+  #   name:
+  #   regex:
+
+SPLUNK_ALERT_DEFAULT_SEVERITY: "3"
+
+
+SPLUNK_VOLUMES: []
+
+# Should include protocol & a trailing slash, e.g. http://splunk.mydomain.com/
+SPLUNK_HOSTNAME: splunk.example.com
+SPLUNK_SMTP_SERVER: smtp.example.com
+SPLUNK_SMTP_USERNAME: username
+SPLUNK_SMTP_PASSWORD: password
+SPLUNK_FROM_ADDRESS: no-reply@example.com
+SPLUNK_EMAIL_FOOTER: Generated by {{ SPLUNK_HOSTNAME }}
+
+
 splunk-server_role_name: splunk-server
 
 splunk_user: "splunk"
@@ -22,12 +86,7 @@ splunk_root: "/vol/splunk/storage"
 splunk_hot_dir: "{{ splunk_root }}/hot"
 splunk_thawed_dir: "{{ splunk_root }}/thawed"
 splunk_cold_dir: "{{ splunk_root }}/cold"
-splunk_frozen_dir: "{{ splunk_root }}/frozen" 
-
-SPLUNK_INDEXES: 
-  - "default"
-
-SPLUNK_VOLUMES: []
+splunk_frozen_dir: "{{ splunk_root }}/frozen"
 
 #
 # OS packages

playbooks/roles/splunk-server/tasks/main.yml

@@ -21,6 +21,11 @@
 #
 #
 
+- name: Validate field extractions
+  fail:
+    msg: Please define either "source" or "sourcetype", not both or neither
+  when: ('source' in item and 'sourcetype' in item) or ('source' not in item and 'sourcetype' not in item)
+  with_items: SPLUNK_FIELD_EXTRACTIONS
 
 - name: Create bucket directories
   file:
@@ -51,7 +56,7 @@
 
 - name: configure splunk buckets
   template:
-    src: "opt/splunk/etc/apps/search/local/indexes.conf"
+    src: "opt/splunk/etc/apps/search/local/indexes.conf.j2"
     dest: "/opt/splunk/etc/apps/search/local/indexes.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
@@ -60,6 +65,41 @@
     - "install"
     - "install:configuration"
 
+- name: configure splunk email
+  template:
+    src: opt/splunk/etc/system/local/alert_actions.conf.j2
+    dest: /opt/splunk/etc/system/local/alert_actions.conf
+    owner: "{{ splunk_user }}"
+    group: "{{ splunk_user }}"
+    mode: 0700
+  tags:
+    - install
+    - install:configuration
+
+- name: configure splunk searches
+  template:
+    src: "opt/splunk/etc/apps/search/local/savedsearches.conf.j2"
+    dest: "/opt/splunk/etc/apps/search/local/savedsearches.conf"
+    owner: "{{ splunk_user }}"
+    group: "{{ splunk_user }}"
+    mode: 0700
+  tags:
+    - "install"
+    - "install:configuration"
+  when: SPLUNK_ALERTS is defined
+
+- name: configure splunk field extractions
+  template:
+    src: opt/splunk/etc/apps/search/local/props.conf.j2
+    dest: /opt/splunk/etc/apps/search/local/props.conf
+    owner: "{{ splunk_user }}"
+    group: "{{ splunk_user }}"
+    mode: 0700
+  tags:
+    - install
+    - install:configuration
+  when: SPLUNK_FIELD_EXTRACTIONS is defined
+
 - name: restart splunk
   service: 
     name: splunk

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/indexes.conf → playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/indexes.conf.j2

-{% for name in SPLUNK_INDEXES -%}
+{% for name in SPLUNK_INDEXES %}
 [{{ name }}]
 coldPath = {{ splunk_cold_dir }}/{{ name }}/colddb
 homePath = {{ splunk_hot_dir }}/{{ name }}/db
 thawedPath = {{ splunk_thawed_dir }}/{{ name }}/thaweddb
 coldToFrozenDir = {{ splunk_frozen_dir }}/{{ name }}/frozendb
 
-{%- endfor %}
+{% endfor %}

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/props.conf.j2

+{% for extraction in SPLUNK_FIELD_EXTRACTIONS %}
+{% if 'source' in extraction %}
+[source::{{ extraction.source }}]
+{% elif 'sourcetype' in extraction %}
+[{{ extraction.sourcetype }}]
+{% endif %}
+EXTRACT-{{ extraction.name }} = {{ extraction.regex }}
+
+{% endfor %}

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/savedsearches.conf.j2

+{# The format of this file is partially documented here:
+   http://docs.splunk.com/Documentation/Splunk/6.1/admin/savedsearchesconf
+#}
+{% for search in SPLUNK_ALERTS %}
+[{{ search.name }}]
+{% if search.email is defined %}
+{# Default email options:
+   action.email.include.view_link = 1     Include a link to the alert
+   action.email.include.results_link = 1  Include a link to the results #}
+action.email = 1
+{# Include the search string #}
+action.email.include.search = 1
+{# Needed for the next option #}
+action.email.sendresults = 1
+{# Include the search results as a table in the body of the message #}
+action.email.inline = 1
+{# Include the alert trigger #}
+action.email.include.trigger = 1
+{# Include a timestamp of when the alert was triggered #}
+action.email.include.trigger_time = 1
+action.email.to = {{ search.email | join(', ') }}
+{# No idea what this does, it's undocumented #}
+action.email.reportServerEnabled = 0
+{# Use a namespaced subject. No idea what that means. #}
+action.email.useNSSubject = 1
+{% if search.message is defined %}
+{# Include a backslash before newlines to match splunk's wonky INI format #}
+action.email.message.alert = {{ search.message.split('\n') | join('\\\n')}}
+{% endif %}
+{% endif %}
+alert.severity = {{ search.severity | default(SPLUNK_ALERT_DEFAULT_SEVERITY) }}
+alert.track = 1
+counttype = {{ search.counttype | default("number of events") }}
+quantity = {{ search.quantity | default("0") }}
+relation = {{ search.comparison | default("greater than") }}
+enableSched = 1
+cron_schedule = {{ search.schedule | default("*/15 * * * *") }}
+dispatch.earliest_time = {{ search.time | default("-15m") }}
+dispatch.latest_time = {{ "rt" if search.time == "rt" else "now" }}
+alert.digest_mode = {{ "0" if search.time == "rt" else "1" }}
+{% if search.digest_delay is defined %}
+alert.suppress = 1
+alert.suppress.period = {{ search.digest_delay }}
+{% endif %}
+request.ui_dispatch_app = search
+request.ui_dispatch_view = search
+{% if search.description %}
+description = {{ search.description }}
+{% endif %}
+search = {{ search.search }}
+
+{% endfor %}

playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/alert_actions.conf.j2

+[email]
+mailserver = {{ SPLUNK_SMTP_SERVER }}
+pdf.header_left = none
+pdf.header_right = none
+auth_password = {{ SPLUNK_SMTP_PASSWORD }}
+auth_username = {{ SPLUNK_SMTP_USERNAME }}
+footer.text = {{ SPLUNK_EMAIL_FOOTER }}
+hostname = {{ SPLUNK_HOSTNAME }}
+from = {{ SPLUNK_FROM_ADDRESS }}
+pdf.footer_enabled = 0
+pdf.header_enabled = 0
+use_tls = 1
\ No newline at end of file