Add a playbook for installing and configuring Graphite.

This playbook will install carbon-cache and graphite-api to provide a
minimalistic starting point for metrics collection.

It currently installs all components from their upstream source (albeit
with defaults that point to tags, and not master/development branches).
It uses systemd, and so is tailored specifically to Ubuntu 16.04.

It currently does not support any of the advanced configuration such as
multiple carbon-cache process + carbon-relay.  It is also lacking many
variables to fully tweak the various configuration files, although
there's support for fully configuring the storage schemas.

There is a small "master" playbook -- edx-monitoring -- for running the
role, and in the future, supporting the culmination of other playbooks
to build out an all-in-one monitoring server: Graphite, Grafana, etc.

docker/build/graphite/Dockerfile

+FROM edxops/xenial-common:latest
+MAINTAINER edxops
+
+USER root
+ADD . /edx/app/edx_ansible/edx_ansible
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
+RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook edx-monitoring.yml -c local \
+   -i '127.0.0.1,'

playbooks/edx-monitoring.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - role: python
+      tags:
+        - install
+        - install:system-requirements
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - graphite
+    - role: nginx
+      nginx_sites:
+      - graphite

playbooks/roles/config-encoders/.gitignore

+*.pyc

playbooks/roles/config-encoders/meta/main.yml

+---
+dependencies: []

playbooks/roles/graphite/defaults/main.yml

+---
+GRAPHITE_CARBON_GIT_URL: "https://github.com/graphite-project/carbon.git"
+GRAPHITE_WHISPER_GIT_URL: "https://github.com/graphite-project/whisper.git"
+GRAPHITE_API_GIT_URL: "https://github.com/brutasse/graphite-api.git"
+
+GRAPHITE_CARBON_VERSION: "816ac631efae33c7b47ecbe79ca9e1f907e5efe8"
+GRAPHITE_WHISPER_VERSION: "0.9.15"
+GRAPHITE_API_VERSION: "1.1.3"
+
+graphite_root: "/opt/graphite"
+graphite_storage_root: "/var/opt/graphite"
+graphite_common_dirs:
+  root:
+    path: "{{ graphite_root }}"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+  src:
+    path: "{{ graphite_root }}/src"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+  conf:
+    path: "{{ graphite_root }}/conf"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+  run:
+    path: "{{ graphite_root }}/run"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0775"
+  storage:
+    path: "{{ graphite_storage_root }}"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0775"
+  log:
+    path: "/var/log/carbon"
+    owner: "{{ carbon_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+
+graphite_common_deb_deps:
+  - libcairo2-dev
+  - libffi-dev
+  - pkg-config
+  - fontconfig
+
+carbon_user: "carbon"
+graphite_user: "graphite"
+graphite_group: "graphite"
+
+GRAPHITE_NGINX_PORT: 6000
+GRAPHITE_GUNICORN_WORKERS: 3
+
+CARBON_LISTEN_ADDR: 0.0.0.0
+
+CARBON_CONF:
+  cache:
+    USER: "{{ carbon_user }}"
+    STORAGE_DIR: "{{ graphite_storage_root }}"
+    CACHE_WRITE_STRATEGY: max
+    MAX_CACHE_SIZE: inf
+    USE_FLOW_CONTROL:  True
+    WHISPER_FALLOCATE_CREATE: True
+    MAX_CREATES_PER_MINUTE: 100
+    MAX_UPDATES_PER_SECOND: 1000
+    LINE_RECEIVER_INTERFACE: "{{ CARBON_LISTEN_ADDR }}"
+    PICKLE_RECEIVER_INTERFACE: "{{ CARBON_LISTEN_ADDR }}"
+    USE_INSECURE_UNPICKLER: False
+    CACHE_QUERY_INTERFACE: "{{ CARBON_LISTEN_ADDR }}"
+    LOG_CACHE_HITS: False
+    LOG_CACHE_QUEUE_SORTS: True
+    LOG_LISTENER_CONNECTIONS: True
+    LOG_UPDATES: False
+    ENABLE_LOGROTATION: True
+    WHISPER_AUTOFLUSH: False
+
+  "cache:1":
+    LINE_RECEIVER_PORT: 2003
+    PICKLE_RECEIVER_PORT: 2004
+    CACHE_QUERY_PORT: 7002
+
+CARBON_STORAGE_SCHEMAS:
+  carbon:
+    description: "Catches all of Carbon's internal metrics"
+    pattern: "carbon.*"
+    retentions: "10s:1y"
+  default:
+    pattern: ".*"
+    retentions: "60s:1y"
+
+CARBON_STORAGE_AGGREGATIONS:
+  min:
+    pattern: "\\.min$"
+    xFilesFactor: "0.1"
+    aggregationMethod: "min"
+  max:
+    pattern: "\\.max$"
+    xFilesFactor: "0.1"
+    aggregationMethod: "max"
+  sum:
+    pattern: "\\.count$"
+    xFilesFactor: "0"
+    aggregationMethod: "sum"
+  default_average:
+    pattern: ".*"
+    xFilesFactor: "0.5"
+    aggregationMethod: "average"
+
+GRAPHITE_API_CONF:
+  search_index: "{{ graphite_root }}/api/index"
+  whisper:
+    directories:
+      - "{{ graphite_storage_root }}"

playbooks/roles/graphite/meta/main.yml

+---
+dependencies:
+  - role: common
+    tags:
+      - always
+  - config-encoders

playbooks/roles/graphite/tasks/carbon.yml

+---
+- name: checkout carbon
+  git:
+    repo: "{{ GRAPHITE_CARBON_GIT_URL }}"
+    dest: "{{ graphite_root }}/src/carbon"
+    version: "{{ GRAPHITE_CARBON_VERSION }}"
+  tags:
+    - install
+    - install:code
+
+- name: install carbon dependencies
+  pip:
+    virtualenv: "{{ graphite_root }}"
+    requirements: "{{ graphite_root }}/src/carbon/requirements.txt"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install carbon
+  command: "{{ graphite_root }}/bin/python setup.py install"
+  args:
+    chdir: "{{ graphite_root }}/src/carbon"
+    creates: "{{ graphite_root }}/bin/carbon-cache.py"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: configure carbon
+  template:
+    src: "carbon/conf/{{ item }}.conf.j2"
+    dest: "{{ graphite_root }}/conf/{{ item }}.conf"
+  with_items:
+    - carbon
+    - storage-schemas
+    - storage-aggregation
+  tags:
+    - install
+    - install:app-configuration
+
+- name: configure carbon service definition
+  template:
+    src: carbon/systemd/carbon-cache.service.j2
+    dest: /etc/systemd/system/carbon-cache.service
+  tags:
+    - install
+    - install:configuration
+
+- name: reload systemd configuration
+  command: systemctl daemon-reload
+  tags:
+    - install
+    - install:configuration
+
+- name: enable carbon
+  service:
+    name: carbon-cache
+    enabled: yes
+    state: started
+  tags:
+    - install
+    - install:configuration
+
+- name: stop carbon
+  service:
+    name: carbon-cache
+    state: stopped
+  tags:
+    - manage:stop
+
+- name: restart carbon
+  service:
+    name: carbon-cache
+    state: restarted
+  tags:
+    - manage:start
+    - manage:restart

playbooks/roles/graphite/tasks/graphite-api.yml

+---
+- name: checkout graphite-api
+  git:
+    repo: "{{ GRAPHITE_API_GIT_URL }}"
+    dest: "{{ graphite_root }}/src/api"
+    version: "{{ GRAPHITE_API_VERSION }}"
+  tags:
+    - install
+    - install:code
+
+- name: install graphite-api dependencies
+  pip:
+    virtualenv: "{{ graphite_root }}"
+    requirements: "{{ graphite_root }}/src/api/requirements.txt"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install graphite-api
+  command: "{{ graphite_root }}/bin/python setup.py install"
+  args:
+    chdir: "{{ graphite_root }}/src/api"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install gunicorn
+  pip:
+    virtualenv: "{{ graphite_root }}"
+    name: gunicorn
+  tags:
+    - install
+    - install:app-requirements
+
+- name: configure graphite-api
+  template:
+    src: graphite-api/conf/graphite-api.yml.j2
+    dest: "{{ graphite_root }}/conf/graphite-api.yml"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+  tags:
+    - install
+    - install:app-configuration
+
+- name: configure graphite-api service definitions
+  template:
+    src: "graphite-api/systemd/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+  with_items:
+    - graphite-api.socket
+    - graphite-api.service
+  tags:
+    - install
+    - install:configuration
+
+- name: reload systemd configuration
+  command: systemctl daemon-reload
+  tags:
+    - install
+    - install:configuration
+
+- name: enable graphite-api
+  service:
+    name: "{{ item }}"
+    enabled: yes
+  with_items:
+    - graphite-api.socket
+    - graphite-api.service
+
+- name: stop graphite-api
+  service:
+    name: graphite-api
+    state: stopped
+  tags:
+    - manage:stop
+
+- name: restart graphite-api
+  service:
+    name: graphite-api
+    state: restarted
+  tags:
+    - manage:start
+    - manage:restart

playbooks/roles/graphite/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Tasks for role graphite
+#
+# Overview:
+#
+# This task is to install "Graphite", or more specifically,
+# its constituent parts: carbon and graphite-api.
+#
+# For our needs at edX, we do not need the full-fledged
+# graphite-web, instead opting to use the light graphite-api,
+# which can be pointed to by other tools such as Grafana.
+#
+# Dependencies:
+# - common
+# - config-encoders
+#
+
+- name: create Graphite user group
+  group:
+    name: "{{ graphite_group }}"
+    state: present
+  tags:
+    - install
+    - install:base
+
+- name: create service account for Graphite
+  user:
+    name: "{{ graphite_user }}"
+    system: yes
+    home: "{{ graphite_root }}"
+    shell: /bin/false
+    group: "{{ graphite_group }}"
+    state: present
+  tags:
+    - install
+    - install:base
+
+- name: create service account for Carbon
+  user:
+    name: "{{ carbon_user }}"
+    system: yes
+    home: "{{ graphite_root }}"
+    shell: /bin/false
+    group: "{{ graphite_group }}"
+    state: present
+  tags:
+    - install
+    - install:base
+
+- name: create common Graphite directories
+  file:
+    path: "{{ item.value.path }}"
+    state: directory
+    owner: "{{ item.value.owner }}"
+    group: "{{ item.value.group }}"
+    mode: "{{ item.value.mode }}"
+  with_dict: "{{ graphite_common_dirs }}"
+  tags:
+    - install
+    - install:base
+
+- name: install Graphite dependencies
+  apt:
+    pkg: "{{ item }}"
+    state: present
+  with_items: "{{ graphite_common_deb_deps }}"
+  tags:
+    - install
+    - install:system-requirements
+
+- name: initialize virtualenv for Graphite
+  command: "virtualenv {{ graphite_root }}"
+  args:
+    creates: "{{ graphite_root }}/bin/activate"
+  tags:
+    - install
+    - install:base
+
+- include: whisper.yml
+- include: carbon.yml
+- include: graphite-api.yml

playbooks/roles/graphite/tasks/whisper.yml

+---
+- name: checkout Whisper
+  git:
+    repo: "{{ GRAPHITE_WHISPER_GIT_URL }}"
+    dest: "{{ graphite_root }}/src/whisper"
+    version: "{{ GRAPHITE_WHISPER_VERSION }}"
+  tags:
+    - install
+    - install:code
+
+- name: install Whisper
+  command: "{{ graphite_root }}/bin/python setup.py install"
+  args:
+    chdir: "{{ graphite_root }}/src/whisper"
+    creates: "{{ graphite_root }}/bin/whisper-create.py"
+  tags:
+    - install
+    - install:app-requirements

playbooks/roles/graphite/templates/carbon/conf/carbon.conf.j2

+{{ CARBON_CONF | encode_ini }}

playbooks/roles/graphite/templates/carbon/conf/storage-aggregation.conf.j2

+# Aggregation methods for whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds
+#
+#  [name]
+#  pattern = <regex>
+#  xFilesFactor = <float between 0 and 1>
+#  aggregationMethod = <average|sum|last|max|min>
+#
+#  name: Arbitrary unique name for the rule
+#  pattern: Regex pattern to match against the metric name
+#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
+#  aggregationMethod: function to apply to data points for aggregation
+
+{{ CARBON_STORAGE_AGGREGATIONS | encode_ini }}

playbooks/roles/graphite/templates/carbon/conf/storage-schemas.conf.j2

+# Schema definitions for Whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds.
+
+{{ CARBON_STORAGE_SCHEMAS | encode_ini }}

playbooks/roles/graphite/templates/carbon/systemd/carbon-cache.service.j2

+[Unit]
+Description=carbon-cache
+After=network.target
+
+[Service]
+Type=forking
+StandardOutput=syslog
+StandardError=syslog
+User={{ carbon_user }}
+Group={{ graphite_group }}
+ExecStart={{ graphite_root }}/bin/carbon-cache.py --config={{ graphite_root }}/conf/carbon.conf --pidfile={{ graphite_root }}/run/carbon-cache.pid --logdir=/var/log/carbon/ start
+ExecReload=/bin/kill -USR1 $MAINPID
+ExecStop={{ graphite_root }}/bin/carbon-cache.py --pidfile={{ graphite_root }}/run/carbon-cache.pid stop
+PIDFile={{ graphite_root }}/run/carbon-cache.pid
+PrivateTmp=true
+LimitNOFILE=128000
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/graphite/templates/graphite-api/conf/graphite-api.yml.j2

+{{ GRAPHITE_API_CONF | encode_yaml }}

playbooks/roles/graphite/templates/graphite-api/systemd/graphite-api.service.j2

+[Unit]
+Description=graphite-api service
+After=network.target
+Requires=graphite-api.socket
+
+[Service]
+StandardOutput=syslog
+StandardError=syslog
+User={{ graphite_user }}
+Group={{ graphite_group }}
+WorkingDirectory={{ graphite_root }}
+Environment=GRAPHITE_API_CONFIG={{ graphite_root }}/conf/graphite-api.yml
+ExecStart={{ graphite_root }}/bin/gunicorn -w {{ GRAPHITE_GUNICORN_WORKERS }} graphite_api.app:app
+Restart=on-failure
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+LimitNOFILE=128000
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/graphite/templates/graphite-api/systemd/graphite-api.socket.j2

+[Unit]
+Description=graphite-api socket
+
+[Socket]
+SocketUser={{ graphite_user }}
+SocketGroup={{ graphite_group }}
+ListenStream={{ graphite_root }}/run/graphite-api.sock
+
+[Install]
+WantedBy=sockets.target

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/graphite.j2

+#
+# {{ ansible_managed }}
+#
+{# This prevents the injected comment from eating the server
+   directive.  There's probably a better way of doing this,
+   but I don't know it currently.
+#}
+{% raw %}
+
+{% endraw %}
+
+{%- if "graphite" in nginx_default_sites -%}
+  {%- set default_site = "default_server" -%}
+{%- else -%}
+  {%- set default_site = "" -%}
+{%- endif -%}
+
+upstream graphite_app_server {
+  server unix:{{ graphite_root }}/run/graphite-api.sock fail_timeout=0;
+}
+
+server {
+
+  server_name graphite.*;
+  listen {{ GRAPHITE_NGINX_PORT }} {{ default_site }};
+  client_max_body_size 1M;
+  keepalive_timeout 5;
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  location @proxy_to_app {
+    {% if NGINX_SET_X_FORWARDED_HEADERS %}
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    {% else %}
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    {% endif %}
+
+    # newrelic-specific header records the time when nginx handles a request.
+    proxy_set_header X-Queue-Start "t=${msec}";
+
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://graphite_app_server;
+  }
+}