Merge pull request #1439 from edx/jarv/fix-log-sync

updates to syncing s3 logs and cron jobs

playbooks/roles/aws/defaults/main.yml

@@ -27,12 +27,12 @@ AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
 # You should be overriding the environment and deployment vars
 # Order of precedence is left to right for exclude and include options
 AWS_S3_LOG_PATHS:
+  - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/tracking"
+    path: "{{ COMMON_LOG_DIR }}/tracking/*"
   - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/application"
     path: "{{ COMMON_LOG_DIR }}/!(*tracking*)"
   - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/system"
     path: "/var/log/*"
-  - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/tracking"
-    path: "{{ COMMON_LOG_DIR }}/tracking/*"
 
 #
 # vars are namespace with the module name.

playbooks/roles/aws/tasks/main.yml

@@ -84,18 +84,19 @@
     dest={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
   when: AWS_S3_LOGS
 
-- name: run s3 log sync script on shutdown
-  file: >
-    state=link
-    src={{ COMMON_BIN_DIR }}/send-logs-to-s3
-    path=/etc/rc0.d/S00send-logs-to-s3
+- name: run s3 log sync script on supervisor shutdown
+  template: >
+    src=etc/init/sync-on-stop.conf.j2
+    dest=/etc/init/sync-on-stop.conf
+    owner=root group=root mode=0644
   when: AWS_S3_LOGS
 
   # cron job runs the aws s3 sync script
+  # this job will log its output to /var/log/aws
 - name: cronjob for s3 log sync
-  cron: >
-    name="cronjob for s3 log sync"
-    user=root
-    minute=0
-    job={{ aws_s3_sync_script }}
+  cron:
+    name: "cronjob for s3 log sync"
+    user: root
+    minute: 0
+    job: "{{ aws_s3_sync_script }} > /dev/null 2>&1"
   when: AWS_S3_LOGS

playbooks/roles/aws/templates/etc/init/sync-on-stop.conf.j2

+start on stopped supervisor
+description     "sync s3 logs on supervisor shutdown"
+script
+        /bin/bash {{ aws_s3_sync_script }} 
+end script

playbooks/roles/aws/templates/send-logs-to-s3.j2

@@ -13,6 +13,10 @@ fi
 exec > >(tee "{{ aws_s3_logfile }}")
 exec 2>&1
 
+# s3cmd sync requires a valid home
+# directory
+export HOME=/
+
 shopt -s extglob
 
 usage() {

playbooks/roles/rabbitmq/tasks/main.yml

@@ -57,8 +57,8 @@
 
 - name: set up a cron job to run the script
   cron:
-    name="log-queue-lenghts"
-    job="{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh"
+    name: "log-queue-lenghts"
+    job: "{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh >/dev/null 2>&1"
 
 # Defaulting to /var/lib/rabbitmq
 - name: create cookie directory