Merge pull request #1671 from edx/benp/new-relic-s3-plugin

new relic s3 plugin

playbooks/roles/newrelic/defaults/main.yml

@@ -43,3 +43,13 @@ newrelic_logwatch_repo: https://github.com/railsware/newrelic_platform_plugins
 newrelic_logwatch_repo_dir: /opt/newrelic_platform_plugins
 newrelic_logwatch_dir: "{{ newrelic_logwatch_repo_dir }}/newrelic_logwatcher_agent"
 newrelic_logwatch_version: "8edd6d214e462b27fdd07d41712eb7b4fff2f7d8"
+
+# s3_watcher
+newrelic_plugin_user: "{{ common_web_user }}"
+newrelic_s3watcher_repo: https://github.com/edx/analytics-gasket
+newrelic_s3watcher_version: "e33cf920aa172707b788834e176350f423c7c38d"
+newrelic_s3watcher_repo_dir: /edx/app/s3watcher
+newrelic_s3watcher_dir: "{{ newrelic_s3watcher_repo_dir }}/newrelic/s3-newrelic-plugin"
+
+rbenv_root_dir: "/opt/rbenv"
+

playbooks/roles/newrelic/meta/main.yml

+---
+dependencies:
+  - common
+  - supervisor
+  - role: rbenv
+    common_web_user: "root"
+    rbenv_user: "root"
+    rbenv_dir: "{{ rbenv_root_dir }}"
+    rbenv_ruby_version: "1.9.3-p547"

playbooks/roles/newrelic/tasks/main.yml

@@ -71,3 +71,8 @@
   when:
   - NEWRELIC_LOGWATCH
   - ansible_distribution == 'Ubuntu'
+
+- include: s3-watcher.yml tags=deploy
+  when:
+  - NEWRELIC_S3_WATCHER
+  - ansible_distribution == 'Ubuntu'

playbooks/roles/newrelic/tasks/s3-watcher.yml

+---
+# This task file is for the newrelic s3watcher plugin
+# which is an agent that obtains metrics from S3 for
+# a given bucket
+
+- name: Install newrelic plugin related packages
+  apt: pkg={{ item }} install_recommends=yes state=present
+  with_items: newrelic_debian_plugin_pkgs
+
+- name: Install s3cmd
+  apt: pkg="s3cmd" install_recommends=yes state=present
+
+- name: create {{ newrelic_s3watcher_repo_dir }}
+  file: >
+    path={{ newrelic_s3watcher_repo_dir }} state=directory
+    owner="{{ newrelic_plugin_user }}" group="{{ common_web_group }}"
+    mode=0755
+
+- name: check out the s3 newrelic plugin
+  git: >
+    dest={{ newrelic_s3watcher_repo_dir }}
+    repo={{ newrelic_s3watcher_repo }} version={{ newrelic_s3watcher_version }}
+    accept_hostkey=yes
+  sudo_user: "{{ newrelic_plugin_user }}"
+
+- name: create bundle install shell script
+  template:
+    owner: "{{ newrelic_plugin_user }}"
+    src: s3-watcher-bundle-install.sh.j2
+    dest: "{{ newrelic_s3watcher_repo_dir }}/s3-watcher-bundle-install.sh"
+    mode: 0755
+  sudo_user: "{{ newrelic_plugin_user }}"
+
+- name: execute bundle install via rbenv
+  shell: ./s3-watcher-bundle-install.sh chdir={{ newrelic_s3watcher_repo_dir }}
+
+- name: create agent configuration
+  template: >
+    src=opt/newrelic_platform_plugins/s3_watcher/config/newrelic_plugins.yml.j2
+    dest={{ newrelic_s3watcher_dir }}/config/newrelic_plugin.yml
+  sudo_user: "{{ newrelic_plugin_user }}"
+
+- name: set agent as executable
+  file: >
+    path={{ newrelic_s3watcher_dir }}/newrelic_s3_agent
+    mode=0755
+
+- name: create supervisor wrapper
+  template:
+    src: s3-watcher-supervisor.sh.j2
+    dest: "{{ newrelic_s3watcher_dir }}/s3-watcher-supervisor.sh"
+    mode: 0755
+    owner: "{{ newrelic_plugin_user }}"
+  sudo_user: "{{ newrelic_plugin_user }}"
+
+- name: create supervisor config
+  template:
+    src: s3watcher.conf.j2
+    dest: "{{ supervisor_cfg_dir }}/s3watcher.conf"
+    owner: "{{ supervisor_user }}"
+    mode: 644
+  sudo_user: "{{ supervisor_user }}"
+
+- name: update supervisor configuration
+  shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  register: supervisor_update
+  changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
+
+- name: ensure s3watcher is started
+  supervisorctl_local: >
+    name=s3watcher
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=started
+

playbooks/roles/newrelic/templates/opt/newrelic_platform_plugins/s3_watcher/config/newrelic_plugins.yml.j2

+# Please make sure to update the license_key information with the license key for your New Relic
+# account.
+#
+newrelic:
+  #
+  # Update with your New Relic account license key:
+  #
+  license_key: {{ NEWRELIC_LICENSE_KEY }}
+  #
+  # Set to '1' for verbose output, remove for normal output.
+  # All output goes to stdout/stderr.
+  #
+  # verbose: 1
+
+  # Proxy configuration:
+  #proxy:
+  #  address: localhost
+  #  port: 8080
+  #  user: nil
+  #  password: nil
+
+# Agent Configuration:
+#
+agents:
+  s3:
+    s3_access_key_id: {{ S3_ACCESS_KEY_ID }}
+    s3_secret_access_key: {{ S3_SECRET_ACCESS_KEY }}
+    s3_bucket_name: {{ S3_BUCKET_NAME }}
+    s3_region: {{ AWS_REGION }}
+    s3_cmd_config_file: {{ S3_CONFIG_FILENAME }}

playbooks/roles/newrelic/templates/s3-watcher-bundle-install.sh.j2

+#!/usr/bin/env bash
+
+source {{ rbenv_root_dir }}/ruby_env
+
+cd {{ newrelic_s3watcher_dir }}
+
+bundle install

playbooks/roles/newrelic/templates/s3-watcher-supervisor.sh.j2

+#!/usr/bin/env bash
+
+source {{ rbenv_root_dir }}/ruby_env
+
+cd {{ newrelic_s3watcher_dir }}
+
+./newrelic_s3_agent

playbooks/roles/newrelic/templates/s3watcher.conf.j2

+[program:s3watcher]
+command={{ newrelic_s3watcher_dir }}/s3-watcher-supervisor.sh
+priority=999
+user={{ newrelic_plugin_user }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true
+stopsignal=QUIT