sync logs on rotation instead of by a cron job

playbooks/edx-east/lifecycle_inventory.py

@@ -64,10 +64,10 @@ class LifecycleInventory():
             for instance in group.instances:
 
                 private_ip_address = instances[instance.instance_id].private_ip_address
-
-                inventory[group.name].append(private_ip_address)
-                inventory[group.name + "_" + instance.lifecycle_state].append(private_ip_address)
-                inventory[instance.lifecycle_state.replace(":","_")].append(private_ip_address)
+                if private_ip_address:
+                    inventory[group.name].append(private_ip_address)
+                    inventory[group.name + "_" + instance.lifecycle_state].append(private_ip_address)
+                    inventory[instance.lifecycle_state.replace(":","_")].append(private_ip_address)
 
         print json.dumps(inventory, sort_keys=True, indent=2)
 
@@ -77,8 +77,8 @@ if __name__=="__main__":
     parser.add_argument('-p', '--profile', help='The aws profile to use when connecting.')
     parser.add_argument('-l', '--list', help='Ansible passes this, we ignore it.', action='store_true', default=True)
     args = parser.parse_args()
-    
+
     LifecycleInventory(args.profile).run()
 
 
-    
+

playbooks/edx-east/retire_host.yml

-# ansible-playbook -i ./lifecycle_inventory.py ./retire_host.yml 
+# ansible-playbook -i ./lifecycle_inventory.py ./retire_host.yml
 # -e@/vars/env.yml --limit Terminating_Wait
 
 #
@@ -19,35 +19,15 @@
   sudo: True
   gather_facts: False
   tasks:
-    - name: Force a log rotation
-      command: /usr/sbin/logrotate -f /etc/logrotate.d/{{ item }}
-      with_items:
-        - "apport"
-        - "apt"
-        - "aptitude"
-        - "dpkg"
-        - "hourly"
-        - "landscape-client"
-        - "newrelic-sysmond"
-        - "nginx"
-        - "nginx-access"
-        - "nginx-error"
-        - "ppp"
-        - "rsyslog"
-        - "ufw"
-        - "unattended-upgrades"
-        - "upstart"
-    - name: Force a log rotation
+    # We only force a rotation of edx logs.
+    # Forced rotation of system logfiles will only
+    # work if there hasn't already been a previous rotation
+    - name: Force a log rotation which will call the log sync
       command: /usr/sbin/logrotate -f /etc/logrotate.d/hourly/{{ item }}
       with_items:
         - "tracking.log"
         - "edx-services"
-    - name: Terminate existing s3 log sync
-      command: /usr/bin/pkill send-logs-to-s3 || true
-    - name: Send logs to s3
-      command: /edx/bin/send-logs-to-s3
 
-      
 - name: Run minos verification
   hosts: Terminating_Wait
   sudo: True

playbooks/roles/aws/defaults/main.yml

@@ -23,16 +23,6 @@ AWS_S3_LOGS: false
 # This relies on your server being able to send mail
 AWS_S3_LOGS_NOTIFY_EMAIL: dummy@example.com
 AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
-# Separate buckets for tracking logs and everything else
-# You should be overriding the environment and deployment vars
-# Order of precedence is left to right for exclude and include options
-AWS_S3_LOG_PATHS:
-  - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/tracking"
-    path: "{{ COMMON_LOG_DIR }}/tracking/*"
-  - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/application"
-    path: "{{ COMMON_LOG_DIR }}/!(*tracking*)"
-  - bucket: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}/logs/system"
-    path: "/var/log/*"
 
 #
 # vars are namespace with the module name.
@@ -45,7 +35,7 @@ aws_s3_logfile: "{{ aws_log_dir }}/s3-log-sync.log"
 aws_log_dir: "{{ COMMON_LOG_DIR }}/aws"
 aws_region: "us-east-1"
 # default path to the aws binary
-s3cmd_cmd: "{{ COMMON_BIN_DIR }}/s3cmd"
+aws_s3cmd: "{{ COMMON_BIN_DIR }}/s3cmd"
 aws_cmd: "/usr/local/bin/aws"
 #
 # OS packages

playbooks/roles/aws/tasks/main.yml

@@ -68,7 +68,7 @@
 - name: create symlink for s3cmd
   file: >
     src={{ aws_app_dir }}/{{ aws_s3cmd_version }}/s3cmd
-    dest={{ COMMON_BIN_DIR }}/s3cmd
+    dest={{ aws_s3cmd }}
     state=link
 
 - name: create s3 log sync script
@@ -84,7 +84,7 @@
     dest={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
   when: AWS_S3_LOGS
 
-- name: run s3 log sync script on supervisor shutdown
+- name: force logrotate on supervisor stop
   template: >
     src=etc/init/sync-on-stop.conf.j2
     dest=/etc/init/sync-on-stop.conf
@@ -99,4 +99,5 @@
     user: root
     minute: 0
     job: "{{ aws_s3_sync_script }} > /dev/null 2>&1"
+    state: absent
   when: AWS_S3_LOGS

playbooks/roles/aws/templates/etc/init/sync-on-stop.conf.j2

 start on stopped supervisor
-description     "sync s3 logs on supervisor shutdown"
+description     "sync tracking logs on supervisor shutdown"
 script
-        /bin/bash {{ aws_s3_sync_script }} 
+        /usr/sbin/logrotate -f /etc/logrotate.d/hourly/tracking.log
+        /usr/sbin/logrotate -f /etc/logrotate.d/hourly/edx-services
 end script

playbooks/roles/aws/templates/send-logs-to-s3.j2

@@ -4,6 +4,7 @@
 #
 #  This script can be called from logrotate
 #  to sync logs to s3
+# 
 
 if (( $EUID != 0 )); then
     echo "Please run as the root user"
@@ -31,10 +32,12 @@ usage() {
             -v    add verbosity (set -x)
             -n    echo what will be done
             -h    this
+            -d    directory to sync
+            -b    bucket path to sync to
 EO
 }
 
-while getopts "vhn" opt; do
+while getopts "vhnb:d:" opt; do
   case $opt in
     v)
       set -x
@@ -48,9 +51,21 @@ while getopts "vhn" opt; do
       noop="echo Would have run: "
       shift
       ;;
+    d)
+      directory=$OPTARG
+      ;;
+    b)
+      bucket_path=$OPTARG
+      ;;
   esac
 done
 
+if [[ -z $bucket_path || -z $directory ]]; then
+  echo "ERROR: You must provide a directory and a bucket to sync!"
+  usage
+  exit 1
+fi
+
 # grab the first security group for the instance
 # which will be used as a directory name in the s3
 # bucket
@@ -93,6 +108,4 @@ availability_zone=$(ec2metadata --availability-zone)
 region=${availability_zone:0:${{lb}}#availability_zone{{rb}} - 1}
 
 s3_path="${2}/$sec_grp/"
-{% for item in AWS_S3_LOG_PATHS -%}
-$noop {{ s3cmd_cmd }} sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/"
-{% endfor %}
+$noop {{ aws_s3cmd }} sync $directory "s3://${bucket_path}/${sec_grp}/${instance_id}-${ip}/"

playbooks/roles/common/defaults/main.yml

@@ -7,7 +7,11 @@
 COMMON_ENABLE_BASIC_AUTH: False
 COMMON_HTPASSWD_USER: edx
 COMMON_HTPASSWD_PASS: edx
-
+# Turn on syncing logs on rotation for edx
+# application and tracking logs, must also
+# have the AWS role installed
+COMMON_AWS_SYNC: False
+COMMON_AWS_SYNC_BUCKET: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}"
 COMMON_BASE_DIR: /edx
 COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var"
 COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app"

playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2

@@ -12,5 +12,12 @@
   size 1M
   postrotate
     /usr/bin/killall -HUP rsyslogd
+    {% if COMMON_AWS_SYNC -%}
+    for dir in `ls {{ COMMON_LOG_DIR }}`; do
+        if echo $dir | grep -qv tracking; then
+            {{ aws_s3_sync_script }} -d {{ COMMON_LOG_DIR }}/$dir -b "{{ COMMON_AWS_SYNC_BUCKET }}/logs/application"
+        fi
+    done
+    {% endif -%}
   endscript
 }

playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2

@@ -10,5 +10,8 @@
   size 1M
   postrotate
     /usr/bin/killall -HUP rsyslogd
+    {% if COMMON_AWS_SYNC -%}
+    {{ aws_s3_sync_script }} -d "{{ COMMON_LOG_DIR }}/tracking/*" -b "{{ COMMON_AWS_SYNC_BUCKET }}/logs/tracking"
+    {% endif -%}
   endscript
 }