Merge pull request #4218 from edx/nadeem/OPS-2121

Move splunk data from cold to frozen on s3 bucket

playbooks/roles/splunk-server/defaults/main.yml

@@ -112,3 +112,6 @@ splunk_cert_path: 'etc/auth/edxcerts'
 splunk-server_debian_pkgs: []
 
 splunk-server_redhat_pkgs: []
+
+#S3 bucket to store data rolled from splunk cold bucket
+SPLUNK_FROZEN_DB_S3_BUCKET: 'splunk-s3-frozen-bucket-name-here' 

playbooks/roles/splunk-server/tasks/main.yml

@@ -149,6 +149,17 @@
     - install
     - install:configuration
 
+- name: create splunk coldToFrozen script
+  template:
+    src: "opt/splunk/bin/coldToFrozenS3.j2"
+    dest: "{{ splunk_home }}/bin/coldToFrozenS3"
+    owner: "{{ splunk_user }}"
+    group: "{{ splunk_user }}"
+    mode: 0700
+  tags:
+    - "install"
+    - "install:configuration"
+
 - name: restart splunk
   service:
     name: splunk
@@ -157,4 +168,3 @@
     - "install"
     - "install:configuration"
     - "restart"
-

playbooks/roles/splunk-server/templates/opt/splunk/bin/coldToFrozenS3.j2

+#!/usr/bin/env bash
+
+s3_backup_bucket="{{ SPLUNK_FROZEN_DB_S3_BUCKET }}"
+
+if [ $# -lt 1 ]; then
+    echo "Usage: coldToFrozenS3 <splunk_bucket_to_frozen>"
+    exit 1
+fi
+
+colddb=$1
+if [[ $colddb == */ ]]; then
+    colddb=${colddb::-1}
+fi
+
+if [[ ! -d $colddb ]]; then
+    echo "Given splunk bucket is not a valid directory: $colddb"
+    exit 1
+elif [[ ! -d $colddb/rawdata ]]; then
+    echo "No rawadata directory, this not an index database: $colddb"
+    exit 1
+fi
+
+# We have splunk bucket path like this /SPLUNK/DB/PATH/$index/colddb/$splunkbucket
+# and want to move on s3 bucket in format as s3://$s3_backup_bucket/$index/frozendb/$splunkbucket
+
+read -a segments <<EOF
+    $(echo  $colddb | awk -F "/" '{print $(NF-2)" "$(NF)}')
+EOF
+index=${segments[0]}
+splunkbucket=${segments[1]}
+
+remotepath="s3://${s3_backup_bucket}/${index}/frozendb/${splunkbucket}"
+
+# Transfer splunk cold bucket to s3 
+# We want to timeout s3 upload and kill it if command running after 15 minutes 
+# as it will occupy splunk worker slots and will effect indexing
+
+timeout=900
+awscli=$(/usr/bin/timeout ${timeout} /usr/bin/aws s3 sync ${colddb} ${remotepath})
+
+retval=$?
+if [ $retval -ne 0 ]; then
+    echo "S3 upload failed to rolled data from splunk cold bucket" 2>&1
+fi

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/indexes.conf.j2

@@ -18,8 +18,8 @@ thawedPath = {{ index.thawedPath }}
 {% else %}
 thawedPath = {{ splunk_thawed_dir }}/{{ index.name }}/thaweddb
 {% endif %}
-{% if index.coldToFrozenDir is not defined %}
-coldToFrozenDir = {{ splunk_frozen_dir }}/{{ index.name }}/frozendb
+{% if index.coldToFrozenScript is defined %}
+coldToFrozenScript = {{ splunk_home }}/bin/coldToFrozenS3
 {% endif %}
 {% if index.disabled is defined %}
 disabled = {{ index.disabled }}