Merge pull request #3304 from edx/max/encrypt-splunk

Add ability to encrypt splunk traffic

playbooks/edx-east/splunk.yml

@@ -10,7 +10,5 @@
     - splunk-server
     - role: datadog
       when: COMMON_ENABLE_DATADOG
-    - role: splunkforwarder
-      when: COMMON_ENABLE_SPLUNKFORWARDER
     - role: newrelic
       when: COMMON_ENABLE_NEWRELIC

playbooks/roles/splunk-server/defaults/main.yml

@@ -84,17 +84,27 @@ SPLUNK_SMTP_PASSWORD: password
 SPLUNK_FROM_ADDRESS: no-reply@example.com
 SPLUNK_EMAIL_FOOTER: Generated by {{ SPLUNK_HOSTNAME }}
 
+# SSL settings. Either all or none of these must be defined.
+# For more details about setting up splunk with SSL, see
+# https://openedx.atlassian.net/wiki/display/EdxOps/viewpage.action?pageId=40174184
+SPLUNK_SSL_CERT: !!null
+SPLUNK_SSL_PASSWORD: !!null
+SPLUNK_SSL_ROOT_CA: !!null
 
 splunk-server_role_name: splunk-server
 
 splunk_user: "splunk"
 splunk_root: "/vol/splunk/storage"
+splunk_home: "/opt/splunk"
 
 splunk_hot_dir: "{{ splunk_root }}/hot"
 splunk_thawed_dir: "{{ splunk_root }}/thawed"
 splunk_cold_dir: "{{ splunk_root }}/cold"
 splunk_frozen_dir: "{{ splunk_root }}/frozen"
 
+splunk_ssl_port: 9995
+splunk_cert_path: 'etc/auth/edxcerts'
+
 #
 # OS packages
 #

playbooks/roles/splunk-server/tasks/main.yml

@@ -27,6 +27,36 @@
   when: ('source' in item and 'sourcetype' in item) or ('source' not in item and 'sourcetype' not in item)
   with_items: SPLUNK_FIELD_EXTRACTIONS
 
+- name: Make sure necessary dirs exist
+  file:
+    path: "{{ item }}"
+    owner: splunk
+    group: splunk
+    state: directory
+  with_items:
+    - "{{ splunk_home }}/{{ splunk_cert_path }}"
+    - "{{ splunk_home }}/etc/system/local"
+
+- name: Write SSL certs to disk
+  copy:
+    dest: "{{ splunk_home }}/{{ splunk_cert_path }}/{{ item.0 }}"
+    content: "{{ item.1 }}"
+    owner: splunk
+    group: splunk
+    mode: "0400"
+  when: "{{ SPLUNK_SSL_CERT is defined and SPLUNK_SSL_CERT | length > 0 }}"
+  with_together:
+    - [forwarder.pem, cacert.pem]
+    - ["{{ SPLUNK_SSL_CERT }}", "{{ SPLUNK_SSL_ROOT_CA }}"]
+
+- name: Create inputs configuration
+  template:
+    src: opt/splunk/etc/system/local/inputs.conf.j2
+    dest: "{{ splunk_home }}/etc/system/local/inputs.conf"
+    owner: splunk
+    group: splunk
+    mode: "0644"
+
 - name: Create bucket directories
   file:
     path: "{{ item }}"
@@ -49,7 +79,7 @@
     owner: "{{ splunk_user  }}"
     group: "{{ splunk_user }}"
   with_items:
-    - "/opt/splunk/etc/apps/search/local"
+    - "{{ splunk_home }}/etc/apps/search/local"
   tags:
     - "install"
     - "install:configuration"
@@ -57,7 +87,7 @@
 - name: configure splunk buckets
   template:
     src: "opt/splunk/etc/apps/search/local/indexes.conf.j2"
-    dest: "/opt/splunk/etc/apps/search/local/indexes.conf"
+    dest: "{{ splunk_home }}/etc/apps/search/local/indexes.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
     mode: 0700
@@ -68,7 +98,7 @@
 - name: configure splunk email
   template:
     src: opt/splunk/etc/system/local/alert_actions.conf.j2
-    dest: /opt/splunk/etc/system/local/alert_actions.conf
+    dest: "{{ splunk_home }}/etc/system/local/alert_actions.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
     mode: 0700
@@ -79,7 +109,7 @@
 - name: configure splunk searches
   template:
     src: "opt/splunk/etc/apps/search/local/savedsearches.conf.j2"
-    dest: "/opt/splunk/etc/apps/search/local/savedsearches.conf"
+    dest: "{{ splunk_home }}/etc/apps/search/local/savedsearches.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
     mode: 0700
@@ -91,7 +121,7 @@
 - name: configure splunk field extractions
   template:
     src: opt/splunk/etc/apps/search/local/props.conf.j2
-    dest: /opt/splunk/etc/apps/search/local/props.conf
+    dest: "{{ splunk_home }}/etc/apps/search/local/props.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
     mode: 0700
@@ -103,14 +133,14 @@
 - name: Make dashboards directory
   file:
     state: directory
-    path: /opt/splunk/etc/apps/search/local/data/ui/views
+    path: "{{ splunk_home }}/etc/apps/search/local/data/ui/views"
     owner: "{{ splunk_user  }}"
     group: "{{ splunk_user }}"
 
 - name: configure splunk dashboards
   template:
     src: "{{ item }}"
-    dest: /opt/splunk/etc/apps/search/local/data/ui/views/{{ item.rstrip('.j2') | basename }}
+    dest: "{{ splunk_home }}/etc/apps/search/local/data/ui/views/{{ item.rstrip('.j2') | basename }}"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
     mode: 0700

playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/inputs.conf.j2

+[default]
+host = {{ SPLUNK_HOSTNAME }}
+
+{% if SPLUNK_SSL_CERT %}
+[splunktcp-ssl:{{ splunk_ssl_port }}]
+compressed = true
+
+[SSL]
+serverCert = $SPLUNK_HOME/{{ splunk_cert_path }}/forwarder.pem
+password = {{ SPLUNK_SSL_PASSWORD }}
+requireClientCert = false
+rootCA = $SPLUNK_HOME/{{ splunk_cert_path }}/cacert.pem
+{% endif %}
\ No newline at end of file

playbooks/roles/splunkforwarder/defaults/main.yml

@@ -20,11 +20,33 @@ SPLUNKFORWARDER_PACKAGE_URL: !!null
 SPLUNKFORWARDER_DEB: !!null
 SPLUNKFORWARDER_PASSWORD: !!null
 
+# A list of dicts with the following keys:
+#   target_group: the name of the group
+#   server: the hostname/IP address of the splunk server
+#   default: whether this group is the default logging group
+# The following keys are for SSL configuration with the server in question.
+# Either all must be defined, or none.
+#   ssl_cert: the text of the SSL cert to use
+#   ssl_cert_password: the password of the SSL cert
+#   ssl_root_ca: the root CA cert that signed the SSL cert
+#   ssl_common_name: the common name (CN) on the SSL cert
 SPLUNKFORWARDER_SERVERS:
   - target_group: "default_output_server"
     server: "localhost:9997"
     default: true
 
+# For more details about setting up splunk with SSL, see
+# https://openedx.atlassian.net/wiki/display/EdxOps/viewpage.action?pageId=40174184
+
+############################ DANGER WILL ROBINSON #############################
+# Splunk server only supports a single SSL cert for all connections!          #
+# If you ever need to rotate the cert, you will have to either take downtime  #
+# or let new logs buffer on the forwarders until you update them.             #
+# When you do update the forwarders, you can't simply roll out new AMIs since #
+# there will be un-forwarded logs. Instead, you must run ansible against your #
+# entire fleet.                                                               #
+###############################################################################
+
 SPLUNKFORWARDER_LOG_ITEMS:
   - source: '{{ COMMON_LOG_DIR }}/lms'
     recursive: true
@@ -60,3 +82,4 @@ splunk_debian_pkgs:
 splunk_redhat_pkgs: []
 
 splunkforwarder_output_dir: '/opt/splunkforwarder/'
+splunkforwarder_ssl_cert_path: 'etc/auth/edxcerts'

playbooks/roles/splunkforwarder/tasks/main.yml

@@ -98,6 +98,36 @@
   when: download_deb.changed and create_boot_script.changed
   notify: restart splunkforwarder
 
+- name: Make sure necessary dirs exist
+  file:
+    path: "{{ item }}"
+    owner: splunk
+    group: splunk
+    state: directory
+  with_items:
+    - "{{ splunkforwarder_output_dir }}/{{ splunkforwarder_ssl_cert_path }}"
+    - /opt/splunkforwarder/etc/system/local
+
+- name: Write SSL certs to disk
+  copy:
+    dest: "{{ splunkforwarder_output_dir }}/{{ splunkforwarder_ssl_cert_path }}/{{ item.target_group }}.pem"
+    content: "{{ item.ssl_cert }}"
+    owner: splunk
+    group: splunk
+    mode: "0400"
+  when: "{{ item.ssl_cert is defined }}"
+  with_items: SPLUNKFORWARDER_SERVERS
+
+- name: Write root CA to disk
+  copy:
+    dest: "{{ splunkforwarder_output_dir }}/{{ splunkforwarder_ssl_cert_path }}/{{ item.target_group }}-rootca.pem"
+    content: "{{ item.ssl_root_ca }}"
+    owner: splunk
+    group: splunk
+    mode: "0400"
+  when: "{{ item.ssl_cert is defined }}"
+  with_items: SPLUNKFORWARDER_SERVERS
+
 - name: Create inputs and outputs configuration
   template:
     src: "opt/splunkforwarder/etc/system/local/{{ item }}.conf.j2"
@@ -108,4 +138,4 @@
   with_items:
     - inputs
     - outputs
-  notify: restart splunkforwarder
+  notify: restart splunkforwarder
\ No newline at end of file

playbooks/roles/splunkforwarder/templates/opt/splunkforwarder/etc/system/local/outputs.conf.j2

@@ -9,5 +9,13 @@ defaultGroup = {{ server.target_group }}
 {% for server in SPLUNKFORWARDER_SERVERS %}
 [tcpout:{{ server.target_group }}]
 server = {{ server.server }}
+compressed = true
+{% if server.ssl_cert is defined %}
+sslCertPath = $SPLUNK_HOME/{{ splunkforwarder_ssl_cert_path }}/{{ server.target_group }}.pem
+sslRootCAPath = $SPLUNK_HOME/{{ splunkforwarder_ssl_cert_path }}/{{ server.target_group }}-rootca.pem
+sslPassword = {{ server.ssl_cert_password }}
+sslVerifyServerCert = true
+sslCommonNameToCheck = {{ server.ssl_common_name }}
+{% endif %}
 
 {% endfor %}
\ No newline at end of file