Merge pull request #800 from edx/jarv/add-cron-exclude

adding option to the sync script to skip .log files

playbooks/roles/aws/defaults/main.yml

@@ -29,7 +29,7 @@ AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
 AWS_S3_LOG_PATHS:
   - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
     path: "{{ COMMON_LOG_DIR }}/"
-    extra_args: "--exclude '*tracking*' --exclude {{ aws_s3_logfile }}"
+    extra_args: "--exclude '*tracking*'"
   - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
     path: "/var/log/"
     extra_args: ""

playbooks/roles/aws/tasks/main.yml

@@ -118,10 +118,15 @@
     path=/etc/rc0.d/S00send-logs-to-s3
   when: AWS_S3_LOGS
 
+  # cron job runs the aws s3 sync script
+  # with the -c option which skips over
+  # .log files. This is because aws s3 sync
+  # has issues syncing files that are written
+  # to during the sync
 - name: cronjob for s3 log sync
   cron: >
     name="cronjob for s3 log sync"
     user=root
     minute=0
-    job={{ aws_s3_sync_script }}
+    job="{{ aws_s3_sync_script }} -c"
   when: AWS_S3_LOGS

playbooks/roles/aws/templates/send-logs-to-s3.j2

@@ -32,6 +32,7 @@ usage() {
             -v    add verbosity (set -x)
             -n    echo what will be done
             -h    this
+            -c    run as a cron which will exclude .log files
 EO
 }
 
@@ -48,6 +49,9 @@ while getopts "vhn" opt; do
     n)
       noop="echo Would have run: "
       shift
+    c)
+      cron_exclude="--exclude '*log'"
+      shift
   esac
 done
 
@@ -94,5 +98,5 @@ region=${availability_zone:0:${{lb}}#availability_zone{{rb}} - 1}
 
 s3_path="${2}/$sec_grp/"
 {% for item in AWS_S3_LOG_PATHS -%}
-$noop $AWS_CMD s3 sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/" --region $region {{ item['extra_args'] }}
+$noop $AWS_CMD s3 sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/" --region $region {{ item['extra_args'] }} $cron_exclude
 {% endfor %}