switching to a single bash script instead of logrotate

playbooks/roles/common/defaults/main.yml

@@ -39,9 +39,16 @@ COMMON_S3_LOGS: false
 COMMON_S3_LOGS_NOTIFY_MAIL: dummy@example.com
 # Separate buckets for tracking logs and everything else
 # You should be overriding the environment and deployment vars
-COMMON_S3_LOGS_APP_BUCKET: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
-COMMON_S3_LOGS_TRACKING_BUCKET: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-tracking-logs"
-
+COMMON_S3_LOG_PATHS:
+  - bucket: "{{ COMMON_S3_LOGS_APP_BUCKET }}"
+    paths:
+    # globs are ok, do not use brace expansion
+    # everything except tracking logs
+    - "{{ COMMON_LOG_DIR }}/!(*tracking*)"
+    - "/var/log/*"
+  - bucket: "{{ COMMON_S3_LOGS_TRACKING_BUCKET }}"
+    paths:
+    - "{{ COMMON_LOG_DIR }}/*tracking*"
 common_debian_pkgs:
   - ntp
   - ack-grep

playbooks/roles/common/templates/edx_logrotate.j2

@@ -9,10 +9,4 @@
   daily
   rotate 90
   size 1M
-  {% if COMMON_S3_LOGS -%}
-  sharedscripts
-  postrotate
-    {% include s3-application.j2 %}
-  endscript
-  {% endif -%}
 }

playbooks/roles/common/templates/edx_logrotate_rsyslog.j2

@@ -8,9 +8,6 @@
     compress
     postrotate
         reload rsyslog >/dev/null 2>&1 || true
-        {% if COMMON_S3_LOGS -%}
-        bash /edx/bin/send-logs-to-s3 "/var/log/syslog*"  s3://{{ COMMON_S3_LOGS_APP_BUCKET }}
-        {% endif -%}
     endscript
 }
 

playbooks/roles/common/templates/edx_logrotate_tracking_log.j2

@@ -8,11 +8,4 @@
   daily
   rotate 365000
   size 1M
-  {% if COMMON_S3_LOGS -%}
-  sharedscripts
-  postrotate
-    {% include s3-tracking.j2 %}
-  endscript
-  {% endif -%}
-
 }

playbooks/roles/common/templates/send-logs-to-s3.j2

@@ -108,5 +108,8 @@ instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
 ip=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 
 s3_path="${2}/$sec_grp/"
-
-$noop $S3CMD sync $1 ${2}/$sec_grp/${instance_id}-${ip}/
+{% for bucket in COMMON_S3_LOG_PATHS -%}
+  {% for path in bucket.paths -%}
+$noop $S3CMD sync {{ path }} {{ bucket }}/$sec_grp/${instance_id}-${ip}/
+  {% endfor %}
+{% endfor %}