Merge pull request #3710 from edx/nadeem/rabbitmq-memory-alerting

rabbitmq memory alerting

playbooks/roles/rabbitmq/tasks/main.yml

@@ -52,11 +52,24 @@
     group: "{{ rabbitmq_group }}"
     mode: "0755"
 
-- name: Set up a cron job to run the script
+- name: Add RabbitMQ memory usage script
+  template:
+    src: "edx/app/rabbitmq/log-rabbitmq-memory.sh.j2"
+    dest: "{{ rabbitmq_app_dir }}/log-rabbitmq-memory.sh"
+    owner: "{{ rabbitmq_user }}"
+    group: "{{ rabbitmq_group }}"
+    mode: "0775"
+
+- name: Set up a cron job to run queue script
   cron:
     name: "log-queue-lenghts"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh >/dev/null 2>&1"
 
+- name: Set up a cron job to run the script
+  cron:
+    name: "log-rabbitmq-memory-usage"
+    job: "{{ rabbitmq_app_dir }}/log-rabbitmq-memory.sh >/dev/null 2>&1"
+
 - name: install logrotate configuration
   template:
     src: etc/logrotate.d/rabbitmq.j2

playbooks/roles/rabbitmq/templates/edx/app/rabbitmq/log-rabbitmq-memory.sh.j2

+#!/usr/bin/env bash
+set -x
+
+log_directory={{ rabbitmq_log_dir }}
+
+{% raw %}
+MemTotal=`grep 'MemTotal' /proc/meminfo | awk '{print $2}'`
+memusg=`/usr/sbin/rabbitmqctl status | grep total | awk -F',|}' -v date="$(date)" -v MemTotal="$MemTotal" 'NR==1{printf date"\tRabbitMQ Memory Usage:(%%)\t" ((($2/1024)/MemTotal)*100)}'`
+echo $memusg >> "$log_directory/rabbitmq.memory.log"
+{% endraw %}

playbooks/roles/splunkforwarder/defaults/main.yml

@@ -72,6 +72,10 @@ SPLUNKFORWARDER_LOG_ITEMS:
     recursive: true
     index: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}'
     sourcetype: 'nginx'
+  - source: '{{ COMMON_LOG_DIR }}/rabbitmq'
+    recursive: true
+    index: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}'
+    sourcetype: 'rabbitmq'
 
 #
 # OS packages