Log the length of all the queues.

playbooks/roles/rabbitmq/defaults/main.yml

 #Variables for rabbitmq
 ---
 
-rabbit_app_dir: "{{ COMMON_APP_DIR }}/rabbitmq"
-rabbit_data_dir: "{{ COMMON_DATA_DIR }}/rabbitmq"
-rabbit_log_dir: "{{ COMMON_LOG_DIR }}/rabbitmq"
-rabbit_cfg_dir: "{{ COMMON_CFG_DIR }}/rabbitmq"
+rabbitmq_app_dir: "{{ COMMON_APP_DIR }}/rabbitmq"
+rabbitmq_data_dir: "{{ COMMON_DATA_DIR }}/rabbitmq"
+rabbitmq_log_dir: "{{ COMMON_LOG_DIR }}/rabbitmq"
+rabbitmq_cfg_dir: "{{ COMMON_CFG_DIR }}/rabbitmq"
+
+rabbitmq_user: "rabbitmq"
+rabbitmq_group: "rabbitmq"
 
 # Environment specific vars
 RABBIT_ERLANG_COOKIE: 'DEFAULT_COOKIE'

playbooks/roles/rabbitmq/tasks/main.yml

@@ -36,6 +36,30 @@
 - name: send sigterm to any running rabbitmq processes
   shell: pkill -u rabbitmq || true
 
+# Create the rabbitmq directories
+- name: create rabbitmq edx directories
+  file:
+    path={{ item }}
+    owner={{ rabbitmq_user }}
+    mode=0755
+    state=directory
+  with_items:
+    - "{{ rabbitmq_app_dir }}"
+    - "{{ rabbitmq_log_dir }}"
+
+- name: add queue monitoring script
+  template:
+    src="edx/app/rabbitmq/log-rabbitmq-queues.sh.j2"
+    dest="{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh"
+    owner="{{ rabbitmq_user }}"
+    group="{{ rabbitmq_group }}"
+    mode=0755
+
+- name: set up a cron job to run the script
+  cron:
+    name="log-queue-lenghts"
+    job="{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh"
+
 # Defaulting to /var/lib/rabbitmq
 - name: create cookie directory
   file: >

playbooks/roles/rabbitmq/templates/edx/app/rabbitmq/log-rabbitmq-queues.sh.j2

+#!/usr/bin/env bash
+set -x
+
+vpc_name={{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}
+log_directory={{ rabbitmq_log_dir }}
+
+
+{% raw %}
+OLD_IFS=$IFS
+IFS=$'\n'
+vhosts=`/usr/sbin/rabbitmqctl list_vhosts | grep "^/"`
+for vhost in $vhosts; do
+  queues=`/usr/sbin/rabbitmqctl list_queues -p $vhost | awk 'NF==2{ print }'`
+  mkdir -p ${log_directory}/${vhost}
+  for queue in $queues; do
+    queue_name=`echo $queue | awk '{ print $1 }'`
+    echo $queue | sed 's/\s*/ /' | awk -v date="$(date)" -v vhost="$vhost" '{ print "date=\x27"date"\x27","vhost=\x27"vhost"\x27","queue=\x27"$1"\x27","length="$2}' >> ${log_directory}/${vhost}/${queue_name}.log
+  done
+done
+IFS=$OLD_IFS
+{% endraw %}