optionally instruct logrotate to rotate logs to s3 buckets

playbooks/roles/common/defaults/main.yml

@@ -25,7 +25,22 @@ COMMON_HOSTNAME: !!null
 COMMON_CUSTOM_DHCLIENT_CONFIG: false
 # uncomment and specifity your domains.
 # COMMON_DHCLIENT_DNS_SEARCH: ["ec2.internal","example.com"]
-  
+
+# Rotate logs to S3
+# Only for when edX is running in AWS since it organizes
+# logs by security group.
+# The two buckets defined below MUST exist prior to enabling
+# this feature
+COMMON_S3_LOGS: false
+# If there are any issues with the s3 sync an error
+# log will be sent to the following address.
+# This relies on your server being able to send mail
+COMMON_S3_LOGS_NOTIFY_MAIL: dummy@example.com
+# Separate buckets for tracking logs and everything else
+# You should be overriding the environment and deployment vars
+COMMON_S3_LOGS_APP_BUCKET: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
+COMMON_S3_LOGS_TRACKING_BUCKET: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-tracking-logs"
+
 common_debian_pkgs:
   - ntp
   - ack-grep

playbooks/roles/common/tasks/main.yml

@@ -55,6 +55,10 @@
 - name: Install logrotate configuration for edX
   template: dest=/etc/logrotate.d/edx-services src=edx_logrotate.j2 owner=root group=root mode=644
 
+  # This is in common to keep all logrotation config
+  # in the same role
+- name: Install logrotate configuration for tracking file
+  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
 
 - name: update /etc/hosts
   template: src=hosts.j2 dest=/etc/hosts

playbooks/roles/common/templates/edx_logrotate.j2

@@ -9,4 +9,10 @@
   daily
   rotate 90
   size 1M
+  {% if COMMON_S3_LOGS -%}
+  sharedscripts
+  postrotate
+    bash /edx/bin/send-logs-to-s3 "/edx/var/log/!(*tracking*)"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+  endscript
+  {% endif -%}
 }

playbooks/roles/common/templates/edx_logrotate_rsyslog.j2

+/var/log/syslog
+{
+    rotate 7
+    daily
+    missingok
+    notifempty
+    delaycompress
+    compress
+    postrotate
+        reload rsyslog >/dev/null 2>&1 || true
+        {% if COMMON_S3_LOGS -%}
+        bash /edx/bin/send-logs-to-s3 "/var/log/syslog*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        {% endif -%}
+    endscript
+}
+
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/auth.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/cron.log
+/var/log/debug
+/var/log/messages
+{
+    rotate 4
+    weekly
+    missingok
+    notifempty
+    compress
+    delaycompress
+    sharedscripts
+    postrotate
+        reload rsyslog >/dev/null 2>&1 || true
+        {% if COMMON_S3_LOGS -%}
+        # wildcard expansion is fine, brace expansion won't work
+        # http://stackoverflow.com/questions/369145/how-to-glob-variables-in-bash-script
+        bash /edx/bin/send-logs-to-s3 "/var/log/mail*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/daemon*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/kern*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/auth*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/user*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/lpr*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/cron*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/debug*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        bash /edx/bin/send-logs-to-s3 "/var/log/messages*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
+        {% endif -%}
+    endscript
+}

playbooks/roles/edxapp/templates/edx_logrotate_tracking_log.j2 → playbooks/roles/common/templates/edx_logrotate_tracking_log.j2

@@ -8,4 +8,11 @@
   daily
   rotate 365000
   size 1M
+  {% if COMMON_S3_LOGS -%}
+  sharedscripts
+  postrotate
+    bash /edx/bin/send-logs-to-s3 "/edx/var/log/*tracking*"  s3://{{ COMMON_S3_LOGS_TRACKING_BUCKET }}
+  endscript
+  {% endif -%}
+
 }

playbooks/roles/common/templates/send-logs-to-s3.j2

+#!/bin/bash
+#
+#  This script can be called from logrotate
+#  to sync logs to s3
+
+if (( $EUID != 0 )); then
+    echo "Please run as the root user"
+    exit 1
+fi
+
+exec > >(tee /var/log/s3-log-sync.log) 
+exec 2>&1
+
+shopt -s extglob
+
+NOTIFY_EMAIL={{ COMMON_S3_LOGS_NOTIFY_MAIL }}
+S3CMD=/var/tmp/s3cmd-1.5.0-alpha2/s3cmd
+
+usage() {
+
+  cat<<EO
+ 
+  A wrapper of s3cmd sync that will sync files to
+  an s3 bucket, will send mail to {{ COMMON_S3_LOGS_NOTIFY_MAIL }} 
+  on failures.
+ 
+  Usage: $PROG <path-to-sync> <s3-bucket-path> 
+            -v    add verbosity (set -x)
+            -n    echo what will be done
+            -h    this
+
+     <s3-path> - path to s3 bucket and folder for sync
+     <path-to-sync> - path on the filesystem to sync
+
+  Example:
+      /edx/bin/send-logs-to-s3 -v /edx/var/log/lms  s3://edx-ec2-app-logs
+EO
+}
+
+while getopts "vhn" opt; do
+  case $opt in
+    v)
+      set -x
+      shift
+      ;;
+    h)
+      usage
+      exit 0
+      ;;
+    n)
+      noop="echo Would have run: "
+      shift
+  esac
+done
+
+if [[ -z $1 || -z $2 ]]; then
+    echo
+    echo "ERROR: You must specify an s3 url and a path"
+    usage
+    exit 1
+fi
+
+
+# Ensure that curl, s3cmd and mail are
+# installed prior to execution
+
+command -v mail >/dev/null 2>&1 || {
+  apt-get install -y mailutils
+}
+
+command -v $S3CMD >/dev/null 2>&1 || {
+  cd /var/tmp
+  curl -o s3cmd-1.5.0-alpha2.tar.gz http://files.edx.org/s3cmd/s3cmd-1.5.0-alpha2.tar.gz
+  tar zxvf s3cmd-1.5.0-alpha2.tar.gz
+}
+
+command -v curl >/dev/null 2>&1 || {
+  apt-get install -y curl
+}
+
+# grab the first security group for the instance
+# which will be used as a directory name in the s3
+# bucket
+
+# If there are any errors from this point
+# send mail to $NOTIFY_EMAIL
+
+set -e
+
+sec_grp=unset
+instance_id=unset
+s3_path=unset
+
+onerror() {
+  if [[ -z $noop ]]; then
+    echo "ERROR: syncing $s3_path on $instance_id"
+    cat /var/log/s3-log-sync.log | mail -s "Error syncing $s3_path on $instance_id" $NOTIFY_EMAIL
+  else
+    echo "Error syncing $s3_path on $instance_id"
+  fi
+}
+
+trap onerror ERR SIGHUP SIGINT SIGTERM 
+
+# first security group is used as the directory name in the bucket
+sec_grp=$(curl -s http://169.254.169.254/latest/meta-data/security-groups | head -1)
+instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+ip=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
+
+s3_path="${2}/$sec_grp/"
+
+$noop $S3CMD sync $1 ${2}/$sec_grp/${instance_id}-${ip}/

playbooks/roles/edxapp/tasks/main.yml

@@ -2,14 +2,6 @@
 #  - group_vars/all
 #  - common/tasks/main.yml
 ---
-
-
-- name: Install logrotate configuration for tracking file
-  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
 - name: create application user
   user: >
     name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"

util/jenkins/ansible-provision.sh

@@ -129,6 +129,8 @@ discern_version: $discern_version
 rabbitmq_ip: "127.0.0.1"
 rabbitmq_refresh: True
 COMMON_HOSTNAME: edx-server
+COMMON_DEPLOYMENT: edx
+COMMON_ENVIRONMENT: sandbox
 EDXAPP_STATIC_URL_BASE: $static_url_base
 
 # Settings for Grade downloads
@@ -136,6 +138,9 @@ EDXAPP_GRADE_STORAGE_TYPE: 's3'
 EDXAPP_GRADE_BUCKET: 'edx-grades'
 EDXAPP_GRADE_ROOT_PATH: 'sandbox'
 
+# send logs to s3
+COMMON_S3_LOGS: true
+COMMON_S3_LOGS_NOTIFY_MAIL: jarv@edx.org
 EOF
 
 if [[ $basic_auth == "true" ]]; then