Configure rsyslog to catch logs from edx

* Touches directories and files into existence expected during setup for
  LMS and CMS
* Configures rsyslog to catch local0 and local1 from edx, and route into
  the appropriately named files in the appropriately named
  subdirectories
* Permissions updates to ensure rsyslog can write everywhere it's
  supposed to
* Restarts rsyslog after reconfiguration is complete

playbooks/group_vars/all

 ---
 app_base_dir: /opt/wwc
+log_base_dir: /mnt/logs
 venv_dir: /opt/edx
 
 # these pathes are relative to the playbook dir
@@ -7,4 +8,4 @@ venv_dir: /opt/edx
 secure_dir: 'secure_example'
 # this indicates the path to site-specific (with precedence)
 # things like nginx template files
-local_dir:  '../../ansible_local'
\ No newline at end of file
+local_dir:  '../../ansible_local'

playbooks/roles/cms/tasks/main.yml

@@ -19,5 +19,12 @@
   
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=cms-backend
 
+- name: Create CMS log target directory
+  file: path={{log_base_dir}}/cms state=directory owner=syslog group=adm mode=2770
+  tags:
+  - cms 
+  - cms-env
+  - logging
+
 # Creates LMS upstart file
 - include: ../../gunicorn/tasks/upstart.yml service_variant=cms

playbooks/roles/cms/vars/main.yml

@@ -26,7 +26,7 @@ cms_env_config:
       'KEY_FUNCTION':  'util.memcache.safe_key'
       'KEY_PREFIX': 'cms.edx.org'
       'LOCATION': [ "deploycache-large.foo-bar.amazonaws.com:11211" ]
-  'LOG_DIR': '/mnt/logs/edx'
+  'LOG_DIR': '{{log_base_dir}}/edx'
   'LOGGING_ENV': 'cms-dev'
   'SITE_NAME': 'studio.cms-dev.m.edx.org'
   'SYSLOG_SERVER': 'syslog.a.m.i4x.org'

playbooks/roles/common/handlers/main.yml

+---
+- name: restart rsyslogd
+  service: name=rsyslog state=restarted
+  sudo: True

playbooks/roles/common/tasks/main.yml

@@ -16,38 +16,44 @@
   tags:
   - pre_install
   
+- name: Install role-independent useful system packages
+  # do this before log dir setup; rsyslog package guarantees syslog user present
+  apt: pkg={{item}} install_recommends=yes state=present update_cache=yes
+  with_items:
+  - ack-grep
+  - lynx-cur
+  - mosh
+  - rsyslog
+  - screen
+  - tree
+  tags:
+  - pre_install
+
 - name: Create log directory
-  file: path=/mnt/logs state=directory mode=2770 group=adm owner=root
+  file: path=$log_base_dir state=directory mode=2770 group=adm owner=syslog
   tags:
   - pre_install
+
 - name: Create aliases to the log directory
-  file: state=link src=/mnt/logs path=$app_base_dir/log
+  file: state=link src=$log_base_dir path=$app_base_dir/log
   tags:
   - pre_install
-- name: Touch the edx log file into place 
-  command: touch -a /mnt/logs/edx.log
+  - logging
+
+- name: Touch edx log file into place 
+  # This is done for the benefit of the rake commands, which expect it
+  command: touch -a {{log_base_dir}}/edx.log
   tags:
   - pre_install
+  - logging
+
 - name: Set permissions on edx log file
-  file: path=/mnt/logs/edx.log owner=www-data group=adm mode=775
-  tags:
-  - pre_install
-- name: Update apt cache
-  apt: update_cache=yes
+  # This is done for the benefit of the rake commands, which expect it
+  file: path={{log_base_dir}}/edx.log owner=syslog group=adm mode=770
   tags:
   - pre_install
+  - logging
 
 - include: create_venv.yml
-
-- name: Install role-independent useful system packages
-  apt: pkg={{item}} install_recommends=yes state=present
-  with_items:
-  - ack-grep
-  - lynx-cur
-  - mosh
-  - most
-  - screen
-  - tree
-  tags:
-  - pre_install
+- include: rsyslog.yml
 

playbooks/roles/common/tasks/rsyslog.yml

+---
+- name: Install rsyslog configuration for edX
+  template: dest=/etc/rsyslog.d/99-edx src=edx_rsyslog.j2 owner=root group=root mode=644
+  notify: restart rsyslogd
+  tags:
+  - logging

playbooks/roles/common/templates/edx_rsyslog.j2

+# custom edx syslog configuration
+# Put in place and templatized by ansible 
+#
+# Cliffs notes version: ansible uses local0 and local1, so they have to be
+# plumbed through appropriately.
+ 
+#############
+# Change some global configuration
+#############
+# don't escape newlines
+$EscapeControlCharactersOnReceive off
+$SystemLogRateLimitInterval 0
+$RepeatedMsgReduction off
+$MaxMessageSize 32768
+ 
+#############
+# Override default auth config so we can ignore local0 and local1 also
+#############
+auth,authpriv.*                                         /var/log/auth.log
+*.*;auth,authpriv.none,local0.none,local1.none          -/var/log/syslog
+ 
+# According to the docs for rsyslog, "syslogtag" is the "TAG" from 
+# the message which in the case of tracking logs is interpreted to 
+# be everything before the first whitespace character. 
+# This is why we include "syslogtag."
+# Maybe one day this will be answered:
+# - http://stackoverflow.com/questions/10449447/how-to-avoid-syslogtag-from-rsyslog-template
+$template tracking,"%syslogtag%%msg%\n"
+ 
+# looks for [service_name=<name>] in the beginning of the log message,
+# if it exists the log will go into {{log_base_dir}}/<name>/edx.log, otherwise
+# it will go into {{log_base_dir}}/edx.log
+$template DynaFile,"{{log_base_dir}}/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
+ 
+local0.*                        -?DynaFile
+local1.*                        {{log_base_dir}}/tracking.log;tracking

playbooks/roles/lms/tasks/main.yml

@@ -25,6 +25,13 @@
   - lms
   - lms-env
 
+- name: Create lms log target directory
+  file: path={{log_base_dir}}/lms state=directory owner=syslog group=adm mode=2770
+  tags:
+  - lms
+  - lms-env
+  - logging
+
 # Install ssh keys for ubuntu account to be able to check out from mitx
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
 # {{ secure_dir }} is relative to the top-level playbooks dir so there is some

playbooks/roles/nginx/tasks/main.yml

@@ -29,3 +29,8 @@
   service: name=nginx state=started
   tags:
   - nginx
+- name: Create nginx log file location (just in case)
+  file: path={{log_base_dir}}/nginx state=directory owner=syslog group=adm mode=2770
+  tags:
+  - nginx
+  - logging