adding newrelic plugin for logwatching

playbooks/roles/newrelic/defaults/main.yml

@@ -21,6 +21,10 @@ NEWRELIC_DEBIAN_KEY_ID: '548C16BF'
 NEWRELIC_DEBIAN_KEY_URL: 'https://download.newrelic.com/{{ NEWRELIC_DEBIAN_KEY_ID }}.gpg'
 NEWRELIC_LICENSE_KEY: "SPECIFY_KEY_HERE"
 
+# Set this to newrelic logwatch
+# agent template files to enable
+# the logwatch agent
+NEWRELIC_LOGWATCH: []
 #
 # OS packages
 #
@@ -30,3 +34,12 @@ newrelic_debian_pkgs:
 
 newrelic_redhat_pkgs:
   - newrelic-sysmond
+
+newrelic_debian_plugin_pkgs:
+  - ruby-bundler
+  - rubygems
+
+newrelic_logwatch_repo: https://github.com/railsware/newrelic_platform_plugins
+newrelic_logwatch_repo_dir: /opt/newrelic_platform_plugins
+newrelic_logwatch_dir: "{{ newrelic_logwatch_repo_dir }}/newrelic_logwatcher_agent"
+newrelic_logwatch_version: "8edd6d214e462b27fdd07d41712eb7b4fff2f7d8"

playbooks/roles/newrelic/handlers/main.yml

+---
+- name: restart newrelic-logwatch-agent
+  service: name=newrelic-logwatch-agent state=restarted

playbooks/roles/newrelic/tasks/main.yml

@@ -62,3 +62,12 @@
 
 - name: ensure started and enabled
   service: name=newrelic-sysmond state=restarted enabled=yes
+
+  # tags=deploy here so that if we are deploying
+  # an application update on a server the config
+  # for the plugin will be updated with the appropriate
+  # edp
+- include: newrelic-logwatcher.yml tags=deploy
+  when:
+  - NEWRELIC_LOGWATCH
+  - ansible_distribution == 'Ubuntu'

playbooks/roles/newrelic/tasks/newrelic-logwatcher.yml

+---
+# This task file is for the newrelic logwatcher plugin
+# which is an agent that watches logfiles on the system
+# for string matches
+
+- name: Install newrelic plugin related packages
+  apt: pkg={{ item }} install_recommends=yes state=present
+  with_items: newrelic_debian_plugin_pkgs
+
+
+- name: check out the newrelic logwatcher plugin
+  git: >
+    dest={{ newrelic_logwatch_repo_dir }}
+    repo={{ newrelic_logwatch_repo }} version={{ newrelic_logwatch_version }}
+    accept_hostkey=yes
+
+- name: bundle install
+  shell: >
+    chdir={{ newrelic_logwatch_dir }}
+    creates=/var/lib/gems/1.8/gems/newrelic_plugin-1.0.2/
+    bundle install
+  notify: restart newrelic-logwatch-agent
+
+- name: create agent configuration
+  template: >
+    src=opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/newrelic_plugin.yml.j2
+    dest={{ newrelic_logwatch_dir }}/config/newrelic_plugin.yml
+  notify: restart newrelic-logwatch-agent
+
+- template:
+    owner: root
+    src: etc/init/newrelic-logwatch-agent.conf.j2
+    dest: /etc/init/newrelic-logwatch-agent.conf
+  notify: restart newrelic-logwatch-agent

playbooks/roles/newrelic/templates/etc/init/newrelic-logwatch-agent.conf.j2

+description  "newrelic nginx agent"
+start on runlevel [2345]
+stop on runlevel [016]
+respawn
+chdir {{ newrelic_logwatch_dir }} 
+exec ruby newrelic_logwatcher_agent.rb

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-503.j2

+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') %}
+{% endif %}
+
+  {{ server_name }}-nginx-503:
+    # Full path to the the log file
+    log_path: {{ nginx_log_dir|default('/edx/var/log/nginx') }}/error.log
+    # Returns the number of matches for this term. Use Linux Regex formatting.
+    term: "limiting requests"
+    # Provide any options to pass to grep when running.
+    # For example, to count non-matching lines, enter 'v'.
+    # Use the abbreviated format ('v' and not 'invert-match').
+    grep_options:

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-certs-errors.j2

+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') %}
+{% endif %}
+
+  {{ server_name }}-certs:
+    # Full path to the the log file
+    log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/certs/edx.log
+    # Returns the number of matches for this term. Use Linux Regex formatting.
+    term: " ERROR "
+    # Provide any options to pass to grep when running.
+    # For example, to count non-matching lines, enter 'v'.
+    # Use the abbreviated format ('v' and not 'invert-match').
+    grep_options:

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-cms-errors.j2

+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') %}
+{% endif %}
+
+  {{ server_name }}-cms:
+    # Full path to the the log file
+    log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/cms/edx.log
+    # Returns the number of matches for this term. Use Linux Regex formatting.
+    term: " ERROR "
+    # Provide any options to pass to grep when running.
+    # For example, to count non-matching lines, enter 'v'.
+    # Use the abbreviated format ('v' and not 'invert-match').
+    grep_options:

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-lms-errors.j2

+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') %}
+{% endif %}
+
+  {{ server_name }}-lms:
+    # Full path to the the log file
+    log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/lms/edx.log
+    # Returns the number of matches for this term. Use Linux Regex formatting.
+    term: " ERROR "
+    # Provide any options to pass to grep when running.
+    # For example, to count non-matching lines, enter 'v'.
+    # Use the abbreviated format ('v' and not 'invert-match').
+    grep_options:

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/logwatch-xqueue-errors.j2

+{% if SANDBOX_USERNAME is defined %}
+{% set server_name = SANDBOX_USERNAME %}
+{% else %}
+{% set server_name = COMMON_ENVIRONMENT|default('unknown-env') + '-' + COMMON_DEPLOYMENT|default('unknown-deployment') %}
+{% endif %}
+
+  {{ server_name }}-xqueue:
+    # Full path to the the log file
+    log_path: {{ COMMON_LOG_DIR|default('/edx/var/log') }}/xqueue/edx.log
+    # Returns the number of matches for this term. Use Linux Regex formatting.
+    term: " ERROR "
+    # Provide any options to pass to grep when running.
+    # For example, to count non-matching lines, enter 'v'.
+    # Use the abbreviated format ('v' and not 'invert-match').
+    grep_options:

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/newrelic_logwatcher_agent/config/newrelic_plugin.yml.j2

+#
+#
+# This is a sample newrelic_plugin.yml file. Please move this file
+# to the following location if it is not already there:
+#
+#     ./config/newrelic_plugin.yml
+#
+# Where the current directory is the directory where your main program resides and is your current
+# directory when you run the main program.
+#
+# Please make sure to update the license_key information with the license key for your New Relic
+# account.
+#
+#
+newrelic:
+  #
+  # Update with your New Relic account license key:
+  #
+  license_key: '{{ NEWRELIC_LICENSE_KEY }}'
+  #
+  # Set to '1' for verbose output, remove for normal output.
+  # All output goes to stdout/stderr.
+  #
+  # verbose: 1
+#
+# Agent Configuration:
+#
+agents:
+{% for agent in NEWRELIC_LOGWATCH %}
+
+{% include agent %}
+
+{% endfor %}

util/jenkins/ansible-provision.sh

@@ -174,6 +174,7 @@ EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms
 EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms
 XQUEUE_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue
 FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
+SANDBOX_USERNAME: $github_username
 EOF
 fi