Merge pull request #783 from edx/jarv/s3-logrotate

sync application and tracking logs to s3

playbooks/edx-east/aws.yml

+- name: Deploy aws
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars_files:
+    - ["{{ secure_vars }}", "dummy.yml"]
+  roles:
+    - aws

playbooks/edx-east/edx_continuous_integration.yml

@@ -7,6 +7,7 @@
     migrate_db: "yes"
     openid_workaround: True
   roles:
+    - aws
     - role: nginx
       nginx_sites:
       - cms

playbooks/roles/aws/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role aws
+#
+#
+# Rotate logs to S3
+# Only for when edX is running in AWS since it organizes
+# logs by security group.
+# !! The buckets defined below MUST exist prior to enabling !!
+# this feature and the instance IAM role must have write permissions
+# to the buckets
+AWS_S3_LOGS: false
+# If there are any issues with the s3 sync an error
+# log will be sent to the following address.
+# This relies on your server being able to send mail
+AWS_S3_LOGS_NOTIFY_EMAIL: dummy@example.com
+AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
+# Separate buckets for tracking logs and everything else
+# You should be overriding the environment and deployment vars
+# Order of precedence is left to right for exclude and include options
+AWS_S3_LOG_PATHS:
+  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
+    path: "{{ COMMON_LOG_DIR }}/"
+    extra_args: "--exclude '*tracking*' --exclude {{ aws_s3_logfile }}"
+  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-app-logs"
+    path: "/var/log/"
+    extra_args: ""
+  - bucket: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-tracking-logs"
+    path: "{{ COMMON_LOG_DIR }}/"
+    extra_args: "--exclude '*' --include '*tracking*'"
+#
+# vars are namespace with the module name.
+#
+aws_role_name: aws
+aws_data_dir: "{{ COMMON_DATA_DIR }}/aws"
+aws_app_dir: "{{ COMMON_APP_DIR }}/aws"
+aws_var_file: "{{ aws_app_dir }}/server-vars.yml"
+aws_s3_sync_script: "{{ aws_app_dir }}/send-logs-to-s3"
+aws_s3_logfile: "{{ aws_log_dir }}/s3-log-sync.log"
+aws_log_dir: "{{ COMMON_LOG_DIR }}/aws"
+
+# default path to the aws binary
+aws_cmd: "/usr/local/bin/aws"
+#
+# OS packages
+#
+
+aws_debian_pkgs:
+  - python-setuptools
+
+aws_pip_pkgs:
+  - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
+  - awscli
+
+aws_redhat_pkgs: []

playbooks/roles/aws/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role aws
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+dependencies:
+  - common

playbooks/roles/aws/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role aws
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- name: install system packages
+  apt: >
+    pkg={{','.join(aws_debian_pkgs)}}
+    state=present
+    update_cache=yes
+
+- name: install aws python packages
+  pip: >
+    name="{{ item }}" state=present
+    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+  with_items: aws_pip_pkgs
+
+- name: create data and app directories
+  file: >
+    path={{ item }}
+    state=directory
+    owner=root
+    group=root
+    mode=0700
+  with_items:
+    - "{{ aws_data_dir }}"
+    - "{{ aws_app_dir }}"
+    - "{{ aws_log_dir }}"
+
+# The sync script and config file are now symlinks
+# Remove them if they are regular files
+# This can be removed once we don't need to worry
+# about backwards compatibility.
+- stat: path={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
+  register: sync_script
+
+- stat: path={{ COMMON_CFG_DIR}}/{{ aws_var_file|basename }}
+  register: var_file
+
+- file: path={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }} state=absent
+  when: sync_script.stat.exists and sync_script.stat.isreg
+
+- file: path={{ COMMON_CFG_DIR}}/{{ aws_var_file|basename }} state=absent
+  when: var_file.stat.exists and var_file.stat.isreg
+
+- name: dump all vars to yaml
+  template: src=dumpall.yml.j2 dest={{ aws_var_file }} mode=0600
+
+- name: create symlink for config file
+  file: >
+    src={{ aws_var_file }}
+    dest={{ COMMON_CFG_DIR }}/{{ aws_var_file|basename }}
+    state=link
+
+- name: clean up var file, removing all version vars and internal ansible vars
+  shell: sed -i -e "/{{item}}/d" {{ aws_var_file }}
+  with_items:
+    # deploy versions
+    - "^edx_platform_version:"
+    - "^edx_platform_commit:"
+    - "^xqueue_version:"
+    - "^forum_version:"
+    - "^xserver_version:"
+    - "^discern_ease_version:"
+    - "^ora_ease_version:"
+    - "^discern_version:"
+    - "^ora_version:"
+    - "^configuration_version:"
+    - "^ease_version:"
+    - "^certs_version:"
+    # other misc vars
+    - "^tags:"
+    - "^_original_file:"
+    - "^register:"
+    - "^item:"
+    - "^failed_when:"
+    - "^changed_when:"
+    - "^delegate_to:"
+    - "^ansible_ssh_private_key_file:"
+    - "^always_run:"
+
+
+- name: create s3 log sync script
+  template: >
+    dest={{ aws_s3_sync_script }}
+    src=send-logs-to-s3.j2 mode=0755 owner=root group=root
+  when: AWS_S3_LOGS
+
+- name: create symlink for s3 log sync script
+  file: >
+    state=link
+    src={{ aws_s3_sync_script }}
+    dest={{ COMMON_BIN_DIR }}/{{ aws_s3_sync_script|basename }}
+  when: AWS_S3_LOGS
+
+- name: run s3 log sync script on shutdown
+  file: >
+    state=link
+    src={{ COMMON_BIN_DIR }}/send-logs-to-s3
+    path=/etc/rc0.d/S00send-logs-to-s3
+  when: AWS_S3_LOGS
+
+- name: cronjob for s3 log sync
+  cron: >
+    name="cronjob for s3 log sync"
+    user=root
+    minute=0
+    job={{ aws_s3_sync_script }}
+  when: AWS_S3_LOGS

playbooks/roles/aws/templates/send-logs-to-s3.j2

+{% set lb = '{' %}
+{% set rb = '}' %}
+#!/bin/bash
+#
+#  This script can be called from logrotate
+#  to sync logs to s3
+
+if (( $EUID != 0 )); then
+    echo "Please run as the root user"
+    exit 1
+fi
+
+S3_LOGFILE="{{ aws_s3_logfile }}"
+NOTIFY_EMAIL={{ AWS_S3_LOGS_NOTIFY_EMAIL }}
+FROM_EMAIL={{ AWS_S3_LOGS_FROM_EMAIL }}
+AWS_CMD="{{ aws_cmd }} --debug"
+
+exec > >(tee $S3_LOGFILE) 
+exec 2>&1
+
+shopt -s extglob
+
+usage() {
+
+  cat<<EO
+ 
+  A wrapper of s3cmd sync that will sync files to
+  an s3 bucket, will send mail to {{ AWS_S3_LOGS_NOTIFY_EMAIL }} 
+  on failures.
+ 
+  Usage: $PROG
+            -v    add verbosity (set -x)
+            -n    echo what will be done
+            -h    this
+EO
+}
+
+while getopts "vhn" opt; do
+  case $opt in
+    v)
+      set -x
+      shift
+      ;;
+    h)
+      usage
+      exit 0
+      ;;
+    n)
+      noop="echo Would have run: "
+      shift
+  esac
+done
+
+# grab the first security group for the instance
+# which will be used as a directory name in the s3
+# bucket
+
+# If there are any errors from this point
+# send mail to $NOTIFY_EMAIL
+
+set -e
+
+sec_grp=unset
+instance_id=unset
+s3_path=unset
+
+onerror() {
+  if [[ -z $noop ]]; then
+    message_file=/var/tmp/message-$$.json
+    message_string="Error syncing $s3_path: inst_id=$instance_id ip=$ip region=$region"
+    if [[ -r $S3_LOGFILE ]]; then
+      python -c "import json; d={'Subject':{'Data':'$message_string'},'Body':{'Text':{'Data':open('$S3_LOGFILE').read()}}};print json.dumps(d)" > $message_file
+    else
+      cat << EOF > $message_file
+      {"Subject": { "Data": "$message_string" }, "Body": { "Text": { "Data": "!! ERROR !! no logfile" } } }
+EOF
+    fi
+    echo "ERROR: syncing $s3_path on $instance_id"
+    $AWS_CMD ses send-email --from $FROM_EMAIL --to $NOTIFY_EMAIL --message file://$message_file --region $region
+  else
+    echo "Error syncing $s3_path on $instance_id"
+  fi
+}
+
+trap onerror ERR SIGHUP SIGINT SIGTERM 
+
+# first security group is used as the directory name in the bucket
+sec_grp=$(ec2metadata --security-groups | head -1)
+instance_id=$(ec2metadata --instance-id)
+ip=$(ec2metadata --local-ipv4)
+availability_zone=$(ec2metadata --availability-zone)
+# region isn't available via the metadata service
+region=${availability_zone:0:${{lb}}#availability_zone{{rb}} - 1}
+
+s3_path="${2}/$sec_grp/"
+{% for item in AWS_S3_LOG_PATHS -%}
+$noop $AWS_CMD s3 sync {{ item['path'] }} "s3://{{ item['bucket'] }}/$sec_grp/${instance_id}-${ip}/" --region $region {{ item['extra_args'] }}
+{% endfor %}

playbooks/roles/certs/tasks/deploy.yml

@@ -26,11 +26,15 @@
     owner={{ certs_user }} mode=750
   notify: restart certs
 
+- stat: path={{ CERTS_LOCAL_GIT_IDENTITY }}
+  register: certs_identity
+
 - name: install read-only ssh key for the certs repo
   copy: >
     src={{ CERTS_LOCAL_GIT_IDENTITY }} dest={{ certs_git_identity }}
     force=yes owner={{ certs_user }} mode=0600
   notify: restart certs
+  when: certs_identity.stat.exists
 
 - name: checkout certificates repo into {{ certs_code_dir }}
   git: dest={{ certs_code_dir }} repo={{ certs_repo }} version={{ certs_version }}
@@ -38,10 +42,12 @@
   environment:
     GIT_SSH: "{{ certs_git_ssh }}"
   notify: restart certs
+  when: certs_identity.stat.exists
 
 - name: remove read-only ssh key for the certs repo
   file: path={{ certs_git_identity }} state=absent
   notify: restart certs
+  when: certs_identity.stat.exists
 
 - name : install python requirements
   pip: requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present

playbooks/roles/common/defaults/main.yml

@@ -26,6 +26,7 @@ COMMON_CUSTOM_DHCLIENT_CONFIG: false
 # uncomment and specifity your domains.
 # COMMON_DHCLIENT_DNS_SEARCH: ["ec2.internal","example.com"]
 
+
 COMMON_MOTD_TEMPLATE: "motd.tail.j2"
 
 common_debian_pkgs:

playbooks/roles/common/tasks/main.yml

@@ -53,8 +53,12 @@
 
 
 - name: Install logrotate configuration for edX
-  template: dest=/etc/logrotate.d/edx-services src=edx_logrotate.j2 owner=root group=root mode=644
+  template: dest=/etc/logrotate.d/edx-services src=etc/logrotate.d/edx_logrotate.j2 owner=root group=root mode=644
 
+  # This is in common to keep all logrotation config
+  # in the same role
+- name: Install logrotate configuration for tracking file
+  template: dest=/etc/logrotate.d/tracking.log src=etc/logrotate.d/edx_logrotate_tracking_log.j2 owner=root group=root mode=644
 
 - name: update /etc/hosts
   template: src=hosts.j2 dest=/etc/hosts

playbooks/roles/common/templates/etc/logrotate.d/edx_logrotate.j2

+{{ COMMON_LOG_DIR }}/*/edx.log {
+  create
+  compress
+  copytruncate
+  delaycompress
+  dateext
+  missingok
+  notifempty
+  daily
+  rotate 90
+  size 1M
+}

playbooks/roles/common/templates/etc/logrotate.d/edx_logrotate_tracking_log.j2

+{{ COMMON_LOG_DIR }}/tracking.log {
+  create
+  compress
+  delaycompress
+  dateext
+  missingok
+  notifempty
+  daily
+  rotate 365000
+  size 1M
+}

playbooks/roles/edx_ansible/tasks/deploy.yml

@@ -18,31 +18,14 @@
     dest={{ COMMON_BIN_DIR }}/update
     state=link
 
-- name: dump all vars to yaml
-  template: src=dumpall.yml.j2 dest={{ edx_ansible_var_file }} mode=0600
-
-- name: clean up var file, removing all version vars
-  shell: sed -i -e "/{{item}}/d" {{ edx_ansible_var_file }}
-  with_items:
-    # deploy versions
-    - "^edx_platform_version:"
-    - "^edx_platform_commit:"
-    - "^xqueue_version:"
-    - "^forum_version:"
-    - "^xserver_version:"
-    - "^discern_ease_version:"
-    - "^ora_ease_version:"
-    - "^discern_version:"
-    - "^ora_version:"
-    - "^configuration_version:"
-    - "^ease_version:"
-    - "^certs_version:"
-    # other misc vars
-    - "^tags:"
-    - "^_original_file:"
-
 - name: create a symlink for var file
   file: >
     src={{ edx_ansible_var_file }}
     dest={{ COMMON_CFG_DIR }}/{{ edx_ansible_var_file|basename }}
     state=link
+
+- name: create a symlink for ansible-playbook
+  file: >
+    src={{ edx_ansible_venv_bin }}/ansible-playbook
+    dest={{ COMMON_BIN_DIR }}/ansible-playbook
+    state=link

playbooks/roles/edxapp/tasks/main.yml

@@ -2,14 +2,6 @@
 #  - group_vars/all
 #  - common/tasks/main.yml
 ---
-
-
-- name: Install logrotate configuration for tracking file
-  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
 - name: create application user
   user: >
     name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"

playbooks/roles/nginx/tasks/main.yml

@@ -75,20 +75,40 @@
     path={{ nginx_log_dir}} state=directory
     owner={{ common_web_user }} group={{ common_web_user }}
 
+# Check to see if the ssl cert/key exists before copying.
+# This extra check is done to prevent failures when
+# ansible-playbook is run locally
+- stat: path={{ NGINX_SSL_CERTIFICATE }}
+  register: ssl_cert
+
+- stat: path={{ NGINX_SSL_KEY }}
+  register: ssl_key
+
 - name: copy ssl cert
   copy: >
     src={{ NGINX_SSL_CERTIFICATE }}
-    dest=/etc/ssl/certs/{{ item|basename }}
+    dest=/etc/ssl/certs/
     owner=root group=root mode=0644
-  when: NGINX_ENABLE_SSL and NGINX_SSL_CERTIFICATE != 'ssl-cert-snakeoil.pem'
+  when: ssl_cert.stat.exists and NGINX_ENABLE_SSL and NGINX_SSL_CERTIFICATE != 'ssl-cert-snakeoil.pem'
 
 - name: copy ssl key
   copy: >
     src={{ NGINX_SSL_KEY }}
-    dest=/etc/ssl/private/{{ item|basename }}
+    dest=/etc/ssl/private/
     owner=root group=root mode=0640
-  when: NGINX_ENABLE_SSL and NGINX_SSL_KEY != 'ssl-cert-snakeoil.key'
+  when: ssl_key.stat.exists and NGINX_ENABLE_SSL and NGINX_SSL_KEY != 'ssl-cert-snakeoil.key'
+
+# removing default link
+- name: Removing default nginx config and restart (enabled)
+  file: path={{ nginx_sites_enabled_dir }}/default state=absent
+  notify: reload nginx
 
+# Note that nginx logs to /var/log until it reads its configuration, so /etc/logrotate.d/nginx is still good
+
+- name: Set up nginx access log rotation
+  template: >
+    dest=/etc/logrotate.d/nginx-access src=edx_logrotate_nginx_access.j2
+    owner=root group=root mode=644
 
 # removing default link
 - name: Removing default nginx config and restart (enabled)

playbooks/roles/xserver/tasks/deploy.yml

@@ -23,17 +23,25 @@
   sudo_user: "{{ xserver_user }}"
   notify: restart xserver
 
+# Check to see if the identity file exists before copying.
+# This extra check is done to prevent failures when
+# ansible-playbook is run locally
+- stat: path={{ XSERVER_LOCAL_GIT_IDENTITY }}
+  register: xserver_identity
+
 - name: install read-only ssh key for the content repo that is required for grading
   copy: >
     src={{ XSERVER_LOCAL_GIT_IDENTITY }} dest={{ xserver_git_identity }}
     owner={{ xserver_user }} group={{ xserver_user }} mode=0600
   notify: restart xserver
+  when: xserver_identity.stat.exists
 
 - name: upload ssh script
   template: >
     src=git_ssh.sh.j2 dest=/tmp/git_ssh.sh
     owner={{ xserver_user }} mode=750
   notify: restart xserver
+  when: xserver_identity.stat.exists
 
 - name: checkout grader code
   git: dest={{ XSERVER_GRADER_DIR }} repo={{ XSERVER_GRADER_SOURCE }} version={{ xserver_grader_version }}
@@ -41,10 +49,12 @@
     GIT_SSH: /tmp/git_ssh.sh
   notify: restart xserver
   sudo_user: "{{ xserver_user }}"
+  when: xserver_identity.stat.exists
 
 - name: remove read-only ssh key for the content repo
   file: path={{ xserver_git_identity }} state=absent
   notify: restart xserver
+  when: xserver_identity.stat.exists
 
   # call supervisorctl update. this reloads
   # the supervisorctl config and restarts

playbooks/run_role.yml

+---
+# Creates a new ansible role
+# Usage:
+#   ansible-playbook ./create_role.yml -i "hostname," -e role=my_awesome_role
+#
+- hosts: all
+  sudo: True
+  gather_facts: False
+  roles:
+    - "{{role}}"

util/jenkins/ansible-provision.sh

@@ -107,7 +107,6 @@ EDXAPP_LMS_NGINX_PORT: 80
 EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
 EDXAPP_CMS_NGINX_PORT: 80
 EDXAPP_SITE_NAME: ${deploy_host}
-COMMON_PYPI_MIRROR_URL: 'https://pypi.edx.org/root/pypi/+simple/'
 XSERVER_GRADER_DIR: "/edx/var/xserver/data/content-mit-600x~2012_Fall"
 XSERVER_GRADER_SOURCE: "git@github.com:/MITx/6.00x.git"
 XSERVER_LOCAL_GIT_IDENTITY: /var/lib/jenkins/git-identity-edx-pull
@@ -129,6 +128,8 @@ discern_version: $discern_version
 rabbitmq_ip: "127.0.0.1"
 rabbitmq_refresh: True
 COMMON_HOSTNAME: edx-server
+COMMON_DEPLOYMENT: edx
+COMMON_ENVIRONMENT: sandbox
 EDXAPP_STATIC_URL_BASE: $static_url_base
 
 # Settings for Grade downloads
@@ -136,6 +137,10 @@ EDXAPP_GRADE_STORAGE_TYPE: 's3'
 EDXAPP_GRADE_BUCKET: 'edx-grades'
 EDXAPP_GRADE_ROOT_PATH: 'sandbox'
 
+# send logs to s3
+AWS_S3_LOGS: true
+AWS_S3_LOGS_NOTIFY_EMAIL: devops+sandbox-log-sync@edx.org
+AWS_S3_LOGS_FROM_EMAIL: devops@edx.org
 EOF
 
 if [[ $basic_auth == "true" ]]; then