Kibana and logstash roles

playbooks/roles/kibana/defaults/main.yml

+---
+kibana_app_dir: /opt/kibana
+kibana_file: kibana-3.0.0milestone4.tar.gz
+kibana_url: "https://download.elasticsearch.org/kibana/kibana/{{ kibana_file }}"
+kibana_server_name: log.example.com
+kibana_valid_users:
+  - jimbo

playbooks/roles/kibana/handlers/main.yml

+---
+- name: nginx | restart nginx
+  service: name=nginx state=restarted
+
+- name: nginx | reload nginx
+  service: name=nginx state=reloaded

playbooks/roles/kibana/tasks/main.yml

+# requires:
+# - oraclejdk
+# - elasticsearch
+# - nginx
+---
+
+- name: kibana | Ensure app apt dependencies are installed
+  apt: pkg={{ item }} state=installed
+  with_items:
+    - python-software-properties
+    - git
+    - nginx
+
+- name: kibana | Ensure {{ kibana_app_dir }} exists
+  file: path={{ kibana_app_dir }} state=directory owner=root group=root mode=0755
+
+- name: kibana | Ensure subdirectories exist
+  file: path={{ kibana_app_dir }}/{{ item }} owner=root group=root mode=0755 state=directory
+  with_items:
+    - htdocs
+    - share
+
+- name: kibana | ensure we have the specified kibana release
+  get_url: url={{ kibana_url }} dest={{ kibana_app_dir }}/share/{{ kibana_file }}
+
+- name: kibana | extract
+  shell: >
+    chdir={{ kibana_app_dir }}/share
+    tar -xzvf {{ kibana_app_dir }}/share/{{ kibana_file }}
+    creates={{ kibana_app_dir }}/share/{{ kibana_file|replace('.tar.gz','') }}
+
+- name: kibana | install
+  shell: >
+    chdir={{ kibana_app_dir }}/share/{{ kibana_file|replace('.tar.gz','') }}
+    cp -R * {{ kibana_app_dir }}/htdocs/
+
+- name: kibana | copy config
+  template: src=config.js.j2 dest={{ kibana_app_dir }}/htdocs/config.js

playbooks/roles/kibana/templates/config.js.j2

+/**
+ * These is the app's configuration, If you need to configure
+ * the default dashboard, please see dashboards/default
+ */
+define(['settings'],
+function (Settings) {
+  
+
+  return new Settings({
+
+    /**
+     * URL to your elasticsearch server. You almost certainly don't
+     * want 'http://localhost:9200' here. Even if Kibana and ES are on
+     * the same host
+     *
+     * By default this will attempt to reach ES at the same host you have
+     * elasticsearch installed on. You probably want to set it to the FQDN of your
+     * elasticsearch host
+     * @type {String}
+     */
+    //elasticsearch: "http://"+window.location.hostname+":9200",
+	  elasticsearch: "https://{{ kibana_server_name }}/e",
+
+    /**
+     * The default ES index to use for storing Kibana specific object
+     * such as stored dashboards
+     * @type {String}
+     */
+    kibana_index: "kibana-int",
+
+    /**
+     * Panel modules available. Panels will only be loaded when they are defined in the
+     * dashboard, but this list is used in the "add panel" interface.
+     * @type {Array}
+     */
+    panel_names: [
+      'histogram',
+      'map',
+      'pie',
+      'table',
+      'filtering',
+      'timepicker',
+      'text',
+      'fields',
+      'hits',
+      'dashcontrol',
+      'column',
+      'derivequeries',
+      'trends',
+      'bettermap',
+      'query',
+      'terms'
+    ]
+  });
+});

playbooks/roles/kibana/templates/kibana.j2

+server {
+  listen 80;
+  server_name {{ kibana_server_name }};
+  root {{ kibana_app_dir }}/htdocs;
+
+  # Set image format types to expire in a very long time
+  location ~* ^.+\.(jpg|jpeg|gif|png|ico)$ {
+      access_log off;
+      expires max;
+  }
+
+  # Set css and js to expire in a very long time
+  location ~* ^.+\.(css|js)$ {
+      access_log off;
+      expires max;
+  }
+
+  # Catchall for everything else
+  location / {
+    root {{ kibana_app_dir }};
+    index index.html;
+    expires 1d;
+    try_files $uri/ $uri;
+    if (-f $request_filename) {
+      break;
+    }
+  }
+}

playbooks/roles/logstash/defaults/main.yml

+---
+logstash_app_dir: /opt/logstash
+logstash_log_dir: /var/log/logstash
+logstash_data_dir: /opt/syslog/file_logs
+logstash_syslog_port: 514
+logstash_file: logstash-1.2.2-flatjar.jar
+logstash_url: "https://download.elasticsearch.org/logstash/logstash/{{ logstash_file }}"

playbooks/roles/logstash/files/template_logstash.json

+{
+    "template": "logstash-*",
+    "settings" : {
+        "number_of_shards" : 1,
+        "number_of_replicas" : 0,
+        "index" : {
+            "query" : { "default_field" : "message" },
+            "store" : { "compress" : { "stored" : true, "tv": true } }
+        }
+    },
+    "mappings": {
+        "_default_": { 
+            "_all": { "enabled": false },
+            "_source": { "compress": true },
+            "dynamic_templates": [
+                {
+                    "string_template" : { 
+                        "match" : "*",
+                        "mapping": { "type": "string", "index": "not_analyzed" },
+                        "match_mapping_type" : "string"
+                     } 
+                 }
+             ],
+             "properties" : {
+                "@fields": { "type": "object", "dynamic": true, "path": "full" }, 
+                "@message" : { "type" : "string", "index" : "analyzed" },
+                "@source" : { "type" : "string", "index" : "not_analyzed" },
+                "@source_host" : { "type" : "string", "index" : "not_analyzed" },
+                "@source_path" : { "type" : "string", "index" : "not_analyzed" },
+                "@tags": { "type": "string", "index" : "not_analyzed" }, 
+                "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
+                "@type" : { "type" : "string", "index" : "not_analyzed" }
+            }
+        }
+    }
+}

playbooks/roles/logstash/handlers/main.yml

+---
+- name: restart logstash
+  service: name=logstash state=restarted
+

playbooks/roles/logstash/tasks/main.yml

+# requires:
+# - oraclejdk
+# - elasticsearch
+---
+
+- name: logstash | Ensure app apt dependencies are installed
+  apt: pkg={{ item }} state=installed
+  with_items:
+    - redis-server
+
+- name: logstash | Ensure {{ logstash_app_dir }} exists
+  file: path={{ logstash_app_dir }} state=directory owner=root group=root mode=0755
+
+- name: logstash | Ensure subdirectories exist
+  file: path={{ logstash_app_dir }}/{{ item }} owner=root group=root mode=0755 state=directory
+  with_items:
+    - bin
+    - etc
+    - share
+
+- name: logstash | ensure logstash config is in place
+  template: src=logstash.conf.j2 dest={{ logstash_app_dir }}/etc/logstash.conf owner=root group=root mode=0644
+  notify: restart logstash
+
+- name: logstash | ensure logstash upstart job is in place
+  template: src=logstash.upstart.conf.j2 dest=/etc/init/logstash.conf owner=root group=root mode=0755
+
+- name: logstash | ensure logstash has a logging dir at {{ logstash_log_dir }}
+  file: path={{ logstash_log_dir }} owner=root group=root mode=0755 state=directory
+
+- name: logstash | ensure we have the specified logstash release
+  get_url: url={{ logstash_url }} dest={{ logstash_app_dir }}/share/{{ logstash_file }}
+
+- name: logstash | ensure symlink with no version exists at /opt/logstash/share/logstash.jar
+  file: src={{ logstash_app_dir }}/share/${logstash_file} dest={{ logstash_app_dir }}/share/logstash.jar state=link
+
+- name: logstash | start logstash
+  action: service name=logstash state=started enabled=yes
+
+- name: logstash | Ensure we are running
+  wait_for: port={{ logstash_syslog_port }} host=localhost timeout=60
+
+- name: logstash | Copy logstash es index template
+  copy: src=template_logstash.json dest=/etc/elasticsearch/template_logstash.json
+
+- name: logstash | Enable logstash es index template
+  shell: chdir=/etc/elasticsearch executable=/bin/bash curl -XPUT 'http://localhost:9200/_template/template_logstash' -d @template_logstash.json
+
+# TODO: add logrotate for cycling the file logs

playbooks/roles/logstash/templates/logstash.conf.j2

+input {
+  tcp {
+  	port => {{ logstash_syslog_port }}
+    type => syslog
+  }
+  udp {
+    port => {{ logstash_syslog_port }}
+    type => syslog
+  }
+}
+
+filter {
+  if [type] == "syslog" {
+    grok {
+      match => { "message" => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{@source_host}" ]
+    }
+    syslog_pri { }
+    date {
+      match => { "syslog_timestamp" => [ "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ] }
+    }
+    if !("_grokparsefailure" in [tags]) {
+      mutate {
+        replace => [ "@source_host", "%{syslog_hostname}" ]
+        replace => [ "@message", "%{syslog_message}" ]
+      }
+    }
+    mutate {
+      remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
+    }
+  }
+}
+
+output {
+  # Example just to output to elasticsearch
+  elasticsearch { }
+  # And gzip for each host and program
+  file {
+  	   path => '{{ logstash_data_dir }}/%{@source_host}/all.%{+yyyyMMdd}.gz'
+	   gzip => true
+  }
+  # Should add option for S3 as well.
+}

playbooks/roles/logstash/templates/logstash.upstart.conf.j2

+# logstash-indexer.conf# logstash - indexer instance
+#
+
+description     "logstash indexer instance"
+
+start on virtual-filesystems
+stop on runlevel [06]
+
+respawn
+respawn limit 5 30
+limit nofile 65550 65550
+
+env HOME=/opt/logstash
+env JAVA_OPTS='-Xms512m -Xmx512m'
+env PATH=$PATH:/usr/lib/jvm/{{ oraclejdk_base }}/bin
+
+chdir {{ logstash_app_dir }}
+setuid root
+console log
+
+# for versions 1.1.1 - 1.1.4 the internal web service crashes when touched
+# and the current workaround is to just not run it and run Kibana instead
+
+script
+        exec java -jar {{ logstash_app_dir }}/share/logstash.jar agent -f {{ logstash_app_dir }}/etc/logstash.conf --log {{ logstash_log_dir }}/logstash-indexer.out
+end script