Merge pull request #811 from edx/jarv/add-back-s3cmd

Jarv/add back s3cmd

playbooks/roles/aws/defaults/main.yml

@@ -28,14 +28,11 @@ AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
 # Order of precedence is left to right for exclude and include options
 AWS_S3_LOG_PATHS:
   - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
-    path: "{{ COMMON_LOG_DIR }}/"
-    extra_args: "--exclude '*tracking*'"
+    path: "{{ COMMON_LOG_DIR }}/!(*tracking*)"
   - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
-    path: "/var/log/"
-    extra_args: ""
+    path: "/var/log/*"
   - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-tracking-logs"
-    path: "{{ COMMON_LOG_DIR }}/"
-    extra_args: "--exclude '*' --include '*tracking*'"
+    path: "{{ COMMON_LOG_DIR }}/*tracking*"
 
 # set this to true dump all extra vars
 # this is currently broken when extra vars
@@ -55,7 +52,7 @@ aws_s3_logfile: "{{ aws_log_dir }}/s3-log-sync.log"
 aws_log_dir: "{{ COMMON_LOG_DIR }}/aws"
 
 # default path to the aws binary
-aws_cmd: "/usr/local/bin/aws"
+aws_cmd: "{{ COMMON_BIN_DIR }}/s3cmd"
 #
 # OS packages
 #
@@ -68,3 +65,5 @@ aws_pip_pkgs:
   - awscli
 
 aws_redhat_pkgs: []
+aws_s3cmd_version: s3cmd-1.5.0-beta1
+aws_s3cmd_url: "http://files.edx.org/s3cmd/{{ aws_s3cmd_version }}.tar.gz"

playbooks/roles/aws/tasks/main.yml

@@ -21,6 +21,18 @@
 #
 #
 
+- name: create data and app directories
+  file: >
+    path={{ item }}
+    state=directory
+    owner=root
+    group=root
+    mode=0700
+  with_items:
+    - "{{ aws_data_dir }}"
+    - "{{ aws_app_dir }}"
+    - "{{ aws_log_dir }}"
+
 - name: install system packages
   apt: >
     pkg={{','.join(aws_debian_pkgs)}}
@@ -33,17 +45,22 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   with_items: aws_pip_pkgs
 
-- name: create data and app directories
+- name: get s3cmd
+  get_url: >
+    url={{ aws_s3cmd_url }}
+    dest={{ aws_data_dir }}/
+
+- name: untar s3cmd
+  shell: >
+    tar xf {{ aws_data_dir }}/{{ aws_s3cmd_version }}.tar.gz
+    creates={{ aws_app_dir }}/{{ aws_s3cmd_version }}/s3cmd
+    chdir={{ aws_app_dir }}
+
+- name: create symlink for s3cmd
   file: >
-    path={{ item }}
-    state=directory
-    owner=root
-    group=root
-    mode=0700
-  with_items:
-    - "{{ aws_data_dir }}"
-    - "{{ aws_app_dir }}"
-    - "{{ aws_log_dir }}"
+    src={{ aws_app_dir }}/{{ aws_s3cmd_version }}/s3cmd
+    dest={{ COMMON_BIN_DIR }}/s3cmd
+    state=link
 
 # The sync script and config file are now symlinks
 # Remove them if they are regular files
@@ -122,14 +139,10 @@
   when: AWS_S3_LOGS
 
   # cron job runs the aws s3 sync script
-  # with the -c option which skips over
-  # .log files. This is because aws s3 sync
-  # has issues syncing files that are written
-  # to during the sync
 - name: cronjob for s3 log sync
   cron: >
     name="cronjob for s3 log sync"
     user=root
     minute=0
-    job="{{ aws_s3_sync_script }} -c"
+    job={{ aws_s3_sync_script }}
   when: AWS_S3_LOGS

playbooks/roles/aws/templates/send-logs-to-s3.j2

@@ -13,7 +13,7 @@ fi
 S3_LOGFILE="{{ aws_s3_logfile }}"
 NOTIFY_EMAIL={{ AWS_S3_LOGS_NOTIFY_EMAIL }}
 FROM_EMAIL={{ AWS_S3_LOGS_FROM_EMAIL }}
-AWS_CMD="{{ aws_cmd }} --debug"
+AWS_CMD={{ aws_cmd }}
 
 exec > >(tee $S3_LOGFILE) 
 exec 2>&1
@@ -32,11 +32,10 @@ usage() {
             -v    add verbosity (set -x)
             -n    echo what will be done
             -h    this
-            -c    run as a cron which will exclude .log files
 EO
 }
 
-while getopts "vhnc" opt; do
+while getopts "vhn" opt; do
   case $opt in
     v)
       set -x
@@ -50,10 +49,6 @@ while getopts "vhnc" opt; do
       noop="echo Would have run: "
       shift
       ;;
-    c)
-      cron_exclude=\-\-exclude\ '*log'
-      shift
-      ;;
   esac
 done
 
@@ -100,5 +95,5 @@ region=${availability_zone:0:${{lb}}#availability_zone{{rb}} - 1}
 
 s3_path="${2}/$sec_grp/"
 {% for item in AWS_S3_LOG_PATHS -%}
-$noop $AWS_CMD s3 sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/" --region $region {{ item['extra_args'] }} $cron_exclude
+$noop $AWS_CMD sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/"
 {% endfor %}