Merge pull request #2816 from open-craft/smarnach/aws-s3cmd-improvements

Improvements for s3cmd in the "aws" role

playbooks/roles/aws/defaults/main.yml

@@ -16,13 +16,17 @@
 # logs by security group.
 # !! The buckets defined below MUST exist prior to enabling !!
 # this feature and the instance IAM role must have write permissions
-# to the buckets
+# to the buckets, or you must specify the access and secret keys below.
 AWS_S3_LOGS: false
 # If there are any issues with the s3 sync an error
 # log will be sent to the following address.
 # This relies on your server being able to send mail
 AWS_S3_LOGS_NOTIFY_EMAIL: dummy@example.com
 AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
+# Credentials for S3 access in case the instance role doesn't have write
+# permissions to S3
+AWS_S3_LOGS_ACCESS_KEY_ID: ""
+AWS_S3_LOGS_SECRET_KEY: ""
 
 #
 # vars are namespace with the module name.
@@ -50,7 +54,7 @@ aws_s3_sync_script: "{{ aws_dirs.home.path }}/send-logs-to-s3"
 aws_s3_logfile: "{{ aws_dirs.logs.path }}/s3-log-sync.log"
 aws_region: "us-east-1"
 # default path to the aws binary
-aws_s3cmd: "{{ COMMON_BIN_DIR }}/s3cmd"
+aws_s3cmd: "/usr/local/bin/s3cmd"
 aws_cmd: "/usr/local/bin/aws"
 #
 # OS packages
@@ -63,7 +67,6 @@ aws_pip_pkgs:
   - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
   - awscli==1.4.2
   - boto=="{{ common_boto_version }}"
+  - s3cmd==1.6.1
 
 aws_redhat_pkgs: []
-aws_s3cmd_version: s3cmd-1.5.0-beta1
-aws_s3cmd_url: "http://files.edx.org/s3cmd/{{ aws_s3cmd_version }}.tar.gz"

playbooks/roles/aws/tasks/main.yml

@@ -70,23 +70,6 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   with_items: aws_pip_pkgs
 
-- name: get s3cmd
-  get_url: >
-    url={{ aws_s3cmd_url }}
-    dest={{ aws_dirs.data.path }}/
-
-- name: untar s3cmd
-  shell: >
-    tar xf {{ aws_dirs.data.path }}/{{ aws_s3cmd_version }}.tar.gz
-    creates={{ aws_dirs.data.path }}/{{ aws_s3cmd_version }}/s3cmd
-    chdir={{ aws_dirs.home.path }}
-
-- name: create symlink for s3cmd
-  file: >
-    src={{ aws_dirs.home.path }}/{{ aws_s3cmd_version }}/s3cmd
-    dest={{ aws_s3cmd }}
-    state=link
-
 - name: create s3 log sync script
   template: >
     dest={{ aws_s3_sync_script }}

playbooks/roles/aws/templates/send-logs-to-s3.j2

@@ -116,5 +116,11 @@ availability_zone=$(ec2metadata --availability-zone)
 # region isn't available via the metadata service
 region=${availability_zone:0:${{ lb }}#availability_zone{{ rb }} - 1}
 
+{% if AWS_S3_LOGS_ACCESS_KEY_ID %}
+auth_opts="--access_key {{ AWS_S3_LOGS_ACCESS_KEY_ID }} --secret_key {{ AWS_S3_LOGS_SECRET_KEY }}"
+{% else %}
+auth_opts=""
+{% endif %}
+
 s3_path="${2}/$sec_grp/"
-$noop {{ aws_s3cmd }} --multipart-chunk-size-mb 5120 --disable-multipart sync $directory "s3://${bucket_path}/${sec_grp}/${instance_id}-${ip}/"
+$noop {{ aws_s3cmd }} $auth_opts --multipart-chunk-size-mb 5120 --disable-multipart sync $directory "s3://${bucket_path}/${sec_grp}/${instance_id}-${ip}/"