moving s3 log sync to the aws role

playbooks/edx-east/aws.yml

+- name: Deploy aws
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
+  roles:
+    - aws

playbooks/roles/aws/defaults/main.yml

@@ -10,16 +10,43 @@
 ##
 # Defaults for role aws
 #
-
+#
+# Rotate logs to S3
+# Only for when edX is running in AWS since it organizes
+# logs by security group.
+# !! The buckets defined below MUST exist prior to enabling !!
+# this feature and the instance IAM role must have write permissions
+# to the buckets
+AWS_S3_LOGS: false
+# If there are any issues with the s3 sync an error
+# log will be sent to the following address.
+# This relies on your server being able to send mail
+AWS_S3_LOGS_NOTIFY_EMAIL: dummy@example.com
+AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
+# Separate buckets for tracking logs and everything else
+# You should be overriding the environment and deployment vars
+# Order of precedence is left to right for exclude and include options
+AWS_S3_LOG_PATHS:
+  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
+    path: "{{ COMMON_LOG_DIR }}/"
+    extra_args: "--exclude '*tracking*'"
+  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
+    path: "/var/log/"
+    extra_args: ""
+  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-tracking-logs"
+    path: "{{ COMMON_LOG_DIR }}/"
+    extra_args: "--exclude '*' --include '*tracking*'"
 #
 # vars are namespace with the module name.
 #
 aws_role_name: aws
 aws_data_dir: "{{ COMMON_DATA_DIR }}/aws"
 aws_app_dir: "{{ COMMON_APP_DIR }}/aws"
-aws_var_file: "{{ aws_data_dir }}/server-vars.yml"
+aws_var_file: "{{ aws_app_dir }}/server-vars.yml"
 aws_s3_sync_script: "{{ aws_app_dir }}/send-logs-to-s3"
 
+# default path to the aws binary
+aws_cmd: "/usr/local/bin/aws"
 #
 # OS packages
 #

playbooks/roles/aws/tasks/main.yml

@@ -33,13 +33,32 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   with_items: aws_pip_pkgs
 
-- name: create data directory
+- name: create data and app directories
   file: >
-    path={{ aws_data_dir }}
+    path={{ item }}
     state=directory
     owner=root
     group=root
     mode=0700
+  with_items:
+    - "{{ aws_data_dir }}"
+    - "{{ aws_app_dir }}"
+
+# The sync script and config file are now symlinks
+# Remove them if they are regular files
+# This can be removed once we don't need to worry
+# about backwards compatibility.
+- stat: path={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
+  register: sync_script
+
+- stat: path={{ COMMON_CFG_DIR}}/{{ aws_var_file|basename }}
+  register: var_file
+
+- file: path={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }} state=absent
+  when: sync_script.stat.exists and sync_script.stat.isreg
+
+- file: path={{ COMMON_CFG_DIR}}/{{ aws_var_file|basename }} state=absent
+  when: var_file.stat.exists and var_file.stat.isreg
 
 - name: dump all vars to yaml
   template: src=dumpall.yml.j2 dest={{ aws_var_file }} mode=0600
@@ -47,7 +66,7 @@
 - name: create symlink for config file
   file: >
     src={{ aws_var_file }}
-    dest={{ COMMON_BIN_DIR }}/{{ aws_var_file|basename }}
+    dest={{ COMMON_CFG_DIR }}/{{ aws_var_file|basename }}
     state=link
 
 - name: clean up var file, removing all version vars and internal ansible vars
@@ -87,8 +106,8 @@
 - name: create symlink for s3 log sync script
   file: >
     state=link
-    src={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
-    path={{ aws_s3_sync_script }}
+    src={{ aws_s3_sync_script }}
+    dest={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
   when: AWS_S3_LOGS
 
 - name: run s3 log sync script on shutdown

playbooks/roles/certs/tasks/deploy.yml

@@ -26,11 +26,15 @@
     owner={{ certs_user }} mode=750
   notify: restart certs
 
+- stat: path={{ CERTS_LOCAL_GIT_IDENTITY }}
+  register: certs_identity
+
 - name: install read-only ssh key for the certs repo
   copy: >
     src={{ CERTS_LOCAL_GIT_IDENTITY }} dest={{ certs_git_identity }}
     force=yes owner={{ certs_user }} mode=0600
   notify: restart certs
+  when: certs_identity.stat.exists
 
 - name: checkout certificates repo into {{ certs_code_dir }}
   git: dest={{ certs_code_dir }} repo={{ certs_repo }} version={{ certs_version }}
@@ -38,10 +42,12 @@
   environment:
     GIT_SSH: "{{ certs_git_ssh }}"
   notify: restart certs
+  when: certs_identity.stat.exists
 
 - name: remove read-only ssh key for the certs repo
   file: path={{ certs_git_identity }} state=absent
   notify: restart certs
+  when: certs_identity.stat.exists
 
 - name : install python requirements
   pip: requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present

playbooks/roles/common/defaults/main.yml

@@ -25,33 +25,10 @@ COMMON_HOSTNAME: !!null
 COMMON_CUSTOM_DHCLIENT_CONFIG: false
 # uncomment and specifity your domains.
 # COMMON_DHCLIENT_DNS_SEARCH: ["ec2.internal","example.com"]
-  
+
 
 COMMON_MOTD_TEMPLATE: "motd.tail.j2"
 
-# Rotate logs to S3
-# Only for when edX is running in AWS since it organizes
-# logs by security group.
-# !! The buckets defined below MUST exist prior to enabling !!
-# this feature and the instance IAM role must have write permissions
-# to the buckets
-COMMON_S3_LOGS: false
-# If there are any issues with the s3 sync an error
-# log will be sent to the following address.
-# This relies on your server being able to send mail
-COMMON_S3_LOGS_NOTIFY_MAIL: dummy@example.com
-# Separate buckets for tracking logs and everything else
-# You should be overriding the environment and deployment vars
-COMMON_S3_LOG_PATHS:
-  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
-    paths:
-    # globs are ok, do not use brace expansion
-    # everything except tracking logs
-    - "{{ COMMON_LOG_DIR }}/!(*tracking*)"
-    - "/var/log/*"
-  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-tracking-logs"
-    paths:
-    - "{{ COMMON_LOG_DIR }}/*tracking*"
 common_debian_pkgs:
   - ntp
   - ack-grep

playbooks/roles/common/templates/send-logs-to-s3.j2

-#!/bin/bash
-#
-#  This script can be called from logrotate
-#  to sync logs to s3
-
-if (( $EUID != 0 )); then
-    echo "Please run as the root user"
-    exit 1
-fi
-
-# s3cmd needs $HOME defined even though it
-# doesn't use a config file
-export HOME=/
-
-exec > >(tee /var/log/s3-log-sync.log) 
-exec 2>&1
-
-shopt -s extglob
-
-NOTIFY_EMAIL={{ COMMON_S3_LOGS_NOTIFY_MAIL }}
-S3CMD=/var/tmp/s3cmd-1.5.0-alpha2/s3cmd
-
-usage() {
-
-  cat<<EO
- 
-  A wrapper of s3cmd sync that will sync files to
-  an s3 bucket, will send mail to {{ COMMON_S3_LOGS_NOTIFY_MAIL }} 
-  on failures.
- 
-  Usage: $PROG
-            -v    add verbosity (set -x)
-            -n    echo what will be done
-            -h    this
-EO
-}
-
-while getopts "vhn" opt; do
-  case $opt in
-    v)
-      set -x
-      shift
-      ;;
-    h)
-      usage
-      exit 0
-      ;;
-    n)
-      noop="echo Would have run: "
-      shift
-  esac
-done
-
-
-# Ensure that curl, s3cmd and mail are
-# installed prior to execution
-
-command -v mail >/dev/null 2>&1 || {
-  apt-get install -y mailutils
-}
-
-command -v $S3CMD >/dev/null 2>&1 || {
-  cd /var/tmp
-  curl -o s3cmd-1.5.0-alpha2.tar.gz http://files.edx.org/s3cmd/s3cmd-1.5.0-alpha2.tar.gz
-  tar zxvf s3cmd-1.5.0-alpha2.tar.gz
-}
-
-command -v curl >/dev/null 2>&1 || {
-  apt-get install -y curl
-}
-
-# grab the first security group for the instance
-# which will be used as a directory name in the s3
-# bucket
-
-# If there are any errors from this point
-# send mail to $NOTIFY_EMAIL
-
-set -e
-
-sec_grp=unset
-instance_id=unset
-s3_path=unset
-
-onerror() {
-  if [[ -z $noop ]]; then
-    echo "ERROR: syncing $s3_path on $instance_id"
-    cat /var/log/s3-log-sync.log | mail -s "Error syncing $s3_path on $instance_id" $NOTIFY_EMAIL
-  else
-    echo "Error syncing $s3_path on $instance_id"
-  fi
-}
-
-trap onerror ERR SIGHUP SIGINT SIGTERM 
-
-# first security group is used as the directory name in the bucket
-sec_grp=$(ec2metadata --security-groups | head -1)
-instance_id=$(ec2metadata --instance-id)
-ip=$(ec2metadata --local-ipv4)
-
-s3_path="${2}/$sec_grp/"
-{% for item in COMMON_S3_LOG_PATHS -%}
-{% for path in item['paths'] -%}
-$noop $S3CMD sync {{ path }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/"
-{% endfor %}
-{% endfor %}

playbooks/roles/edx_ansible/tasks/deploy.yml

@@ -29,5 +29,3 @@
     src={{ edx_ansible_venv_bin }}/ansible-playbook
     dest={{ COMMON_BIN_DIR }}/ansible-playbook
     state=link
-
-

playbooks/roles/nginx/tasks/main.yml

@@ -75,20 +75,40 @@
     path={{ nginx_log_dir}} state=directory
     owner={{ common_web_user }} group={{ common_web_user }}
 
+# Check to see if the ssl cert/key exists before copying.
+# This extra check is done to prevent failures when
+# ansible-playbook is run locally
+- stat: path={{ NGINX_SSL_CERTIFICATE }}
+  register: ssl_cert
+
+- stat: path={{ NGINX_SSL_KEY }}
+  register: ssl_key
+
 - name: copy ssl cert
   copy: >
     src={{ NGINX_SSL_CERTIFICATE }}
-    dest=/etc/ssl/certs/{{ item|basename }}
+    dest=/etc/ssl/certs/
     owner=root group=root mode=0644
-  when: NGINX_ENABLE_SSL and NGINX_SSL_CERTIFICATE != 'ssl-cert-snakeoil.pem'
+  when: ssl_cert.stat.exists and NGINX_ENABLE_SSL and NGINX_SSL_CERTIFICATE != 'ssl-cert-snakeoil.pem'
 
 - name: copy ssl key
   copy: >
     src={{ NGINX_SSL_KEY }}
-    dest=/etc/ssl/private/{{ item|basename }}
+    dest=/etc/ssl/private/
     owner=root group=root mode=0640
-  when: NGINX_ENABLE_SSL and NGINX_SSL_KEY != 'ssl-cert-snakeoil.key'
+  when: ssl_key.stat.exists and NGINX_ENABLE_SSL and NGINX_SSL_KEY != 'ssl-cert-snakeoil.key'
+
+# removing default link
+- name: Removing default nginx config and restart (enabled)
+  file: path={{ nginx_sites_enabled_dir }}/default state=absent
+  notify: reload nginx
 
+# Note that nginx logs to /var/log until it reads its configuration, so /etc/logrotate.d/nginx is still good
+
+- name: Set up nginx access log rotation
+  template: >
+    dest=/etc/logrotate.d/nginx-access src=edx_logrotate_nginx_access.j2
+    owner=root group=root mode=644
 
 # removing default link
 - name: Removing default nginx config and restart (enabled)

playbooks/roles/xserver/tasks/deploy.yml

@@ -23,17 +23,25 @@
   sudo_user: "{{ xserver_user }}"
   notify: restart xserver
 
+# Check to see if the identity file exists before copying.
+# This extra check is done to prevent failures when
+# ansible-playbook is run locally
+- stat: path={{ XSERVER_LOCAL_GIT_IDENTITY }}
+  register: xserver_identity
+
 - name: install read-only ssh key for the content repo that is required for grading
   copy: >
     src={{ XSERVER_LOCAL_GIT_IDENTITY }} dest={{ xserver_git_identity }}
     owner={{ xserver_user }} group={{ xserver_user }} mode=0600
   notify: restart xserver
+  when: xserver_identity.stat.exists
 
 - name: upload ssh script
   template: >
     src=git_ssh.sh.j2 dest=/tmp/git_ssh.sh
     owner={{ xserver_user }} mode=750
   notify: restart xserver
+  when: xserver_identity.stat.exists
 
 - name: checkout grader code
   git: dest={{ XSERVER_GRADER_DIR }} repo={{ XSERVER_GRADER_SOURCE }} version={{ xserver_grader_version }}
@@ -41,10 +49,12 @@
     GIT_SSH: /tmp/git_ssh.sh
   notify: restart xserver
   sudo_user: "{{ xserver_user }}"
+  when: xserver_identity.stat.exists
 
 - name: remove read-only ssh key for the content repo
   file: path={{ xserver_git_identity }} state=absent
   notify: restart xserver
+  when: xserver_identity.stat.exists
 
   # call supervisorctl update. this reloads
   # the supervisorctl config and restarts

util/jenkins/ansible-provision.sh

@@ -138,8 +138,9 @@ EDXAPP_GRADE_BUCKET: 'edx-grades'
 EDXAPP_GRADE_ROOT_PATH: 'sandbox'
 
 # send logs to s3
-COMMON_S3_LOGS: true
-COMMON_S3_LOGS_NOTIFY_MAIL: devops@edx.org
+AWS_S3_LOGS: true
+AWS_S3_LOGS_NOTIFY_EMAIL: devops+sandbox-log-sync@edx.org
+AWS_S3_LOGS_FROM_EMAIL: devops@edx.org
 EOF
 
 if [[ $basic_auth == "true" ]]; then