Updated to be in line with current standards and use basic auth

playbooks/roles/kibana/defaults/main.yml

 ---
-kibana_app_dir: /opt/kibana
-kibana_file: kibana-3.0.0milestone4.tar.gz
+KIBANA_SERVER_NAME: log.example.com
+KIBANA_NGINX_PORT: 80
+KIBANA_SSL_NGINX_PORT: 443
+
+kibana_app_dir: /edx/app/kibana
+kibana_file: kibana-3.0.0.tar.gz
 kibana_url: "https://download.elasticsearch.org/kibana/kibana/{{ kibana_file }}"
-kibana_server_name: log.example.com
-kibana_valid_users:
-  - jimbo

playbooks/roles/kibana/handlers/main.yml

 ---
-- name: nginx | restart nginx
+- name: restart nginx
   service: name=nginx state=restarted
 
-- name: nginx | reload nginx
+- name: reload nginx
   service: name=nginx state=reloaded

playbooks/roles/kibana/tasks/main.yml

@@ -4,35 +4,36 @@
 # - nginx
 ---
 
-- name: kibana | Ensure app apt dependencies are installed
+- name: Ensure app apt dependencies are installed
   apt: pkg={{ item }} state=installed
   with_items:
     - python-software-properties
     - git
     - nginx
 
-- name: kibana | Ensure {{ kibana_app_dir }} exists
+- name: Ensure {{ kibana_app_dir }} exists
   file: path={{ kibana_app_dir }} state=directory owner=root group=root mode=0755
 
-- name: kibana | Ensure subdirectories exist
+- name: Ensure subdirectories exist
   file: path={{ kibana_app_dir }}/{{ item }} owner=root group=root mode=0755 state=directory
   with_items:
     - htdocs
     - share
 
-- name: kibana | ensure we have the specified kibana release
+- name: ensure we have the specified kibana release
   get_url: url={{ kibana_url }} dest={{ kibana_app_dir }}/share/{{ kibana_file }}
 
-- name: kibana | extract
+- name: extract
   shell: >
     chdir={{ kibana_app_dir }}/share
     tar -xzvf {{ kibana_app_dir }}/share/{{ kibana_file }}
     creates={{ kibana_app_dir }}/share/{{ kibana_file|replace('.tar.gz','') }}
 
-- name: kibana | install
+- name: install
   shell: >
     chdir={{ kibana_app_dir }}/share/{{ kibana_file|replace('.tar.gz','') }}
     cp -R * {{ kibana_app_dir }}/htdocs/
 
-- name: kibana | copy config
+- name: copy config
   template: src=config.js.j2 dest={{ kibana_app_dir }}/htdocs/config.js
+

playbooks/roles/logstash/defaults/main.yml

 ---
-logstash_app_dir: /opt/logstash
-logstash_log_dir: /var/log/logstash
-logstash_data_dir: /opt/syslog/file_logs
+logstash_app_dir: /edx/app/logstash
+logstash_log_dir: /edx/var/log/logstash
+logstash_data_dir: /edx/var/logstash/file_logs
 logstash_syslog_port: 514
-logstash_file: logstash-1.2.2-flatjar.jar
+logstash_file: logstash-1.3.3-flatjar.jar
 logstash_url: "https://download.elasticsearch.org/logstash/logstash/{{ logstash_file }}"

playbooks/roles/logstash/tasks/main.yml

@@ -3,47 +3,47 @@
 # - elasticsearch
 ---
 
-- name: logstash | Ensure app apt dependencies are installed
+- name: Ensure app apt dependencies are installed
   apt: pkg={{ item }} state=installed
   with_items:
     - redis-server
 
-- name: logstash | Ensure {{ logstash_app_dir }} exists
+- name: Ensure {{ logstash_app_dir }} exists
   file: path={{ logstash_app_dir }} state=directory owner=root group=root mode=0755
 
-- name: logstash | Ensure subdirectories exist
+- name: Ensure subdirectories exist
   file: path={{ logstash_app_dir }}/{{ item }} owner=root group=root mode=0755 state=directory
   with_items:
     - bin
     - etc
     - share
 
-- name: logstash | ensure logstash config is in place
+- name: ensure logstash config is in place
   template: src=logstash.conf.j2 dest={{ logstash_app_dir }}/etc/logstash.conf owner=root group=root mode=0644
   notify: restart logstash
 
-- name: logstash | ensure logstash upstart job is in place
+- name: ensure logstash upstart job is in place
   template: src=logstash.upstart.conf.j2 dest=/etc/init/logstash.conf owner=root group=root mode=0755
 
-- name: logstash | ensure logstash has a logging dir at {{ logstash_log_dir }}
+- name: ensure logstash has a logging dir at {{ logstash_log_dir }}
   file: path={{ logstash_log_dir }} owner=root group=root mode=0755 state=directory
 
-- name: logstash | ensure we have the specified logstash release
+- name: ensure we have the specified logstash release
   get_url: url={{ logstash_url }} dest={{ logstash_app_dir }}/share/{{ logstash_file }}
 
-- name: logstash | ensure symlink with no version exists at /opt/logstash/share/logstash.jar
+- name: ensure symlink with no version exists at /opt/logstash/share/logstash.jar
   file: src={{ logstash_app_dir }}/share/${logstash_file} dest={{ logstash_app_dir }}/share/logstash.jar state=link
 
-- name: logstash | start logstash
+- name: start logstash
   action: service name=logstash state=started enabled=yes
 
-- name: logstash | Ensure we are running
+- name: Ensure we are running
   wait_for: port={{ logstash_syslog_port }} host=localhost timeout=60
 
-- name: logstash | Copy logstash es index template
+- name: Copy logstash es index template
   copy: src=template_logstash.json dest=/etc/elasticsearch/template_logstash.json
 
-- name: logstash | Enable logstash es index template
+- name: Enable logstash es index template
   shell: chdir=/etc/elasticsearch executable=/bin/bash curl -XPUT 'http://localhost:9200/_template/template_logstash' -d @template_logstash.json
 
-# TODO: add logrotate for cycling the file logs
+# TODO: Add index rotations for kibana
\ No newline at end of file

playbooks/roles/logstash/templates/logstash.conf.j2

@@ -18,7 +18,7 @@ filter {
     }
     syslog_pri { }
     date {
-      match => { "syslog_timestamp" => [ "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ] }
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
     }
     if !("_grokparsefailure" in [tags]) {
       mutate {

playbooks/roles/nginx/templates/kibana.j2

@@ -5,23 +5,27 @@ upstream elasticsearch_server {
 server {
   # Kibana server, templated by ansible
 
-  listen 443;
+  {% if NGINX_ENABLE_SSL %}
 
-  server_name {{ kibana_server_name }};
+  listen {{KIBANA_NGINX_PORT}} {{default_site}};
+  listen {{KIBANA_SSL_NGINX_PORT}} {{default_site}} ssl;
 
-  root {{ kibana_app_dir }}/htdocs;
+  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+
+  {% else %}
+  listen {{KIBANA_NGINX_PORT}} {{default_site}};
+  {% endif %}
 
-  access_log {{ COMMON_LOG_DIR }}/nginx/kibana.access.log ssl_combined;
-  error_log {{ COMMON_LOG_DIR }}/nginx/kibana.error.log error;
+  server_name {{ KIBANA_SERVER_NAME }};
 
+  root {{ kibana_app_dir }}/htdocs;
 
-  # Authentication restrictions to server
-  set $idok "no";
-  {%- for user in kibana_valid_users -%}
-  if ($ssl_client_s_dn ~ "/emailAddress={{ user }}@MIT.EDU") { set $idok "yes"; }
+  access_log {{ nginx_log_dir }}/kibana.access.log;
+  error_log {{ nginx_log_dir }}/kibana.error.log error;
 
-  {%- endfor -%}
-  if ($idok !~ "yes") {  return 403; }
+  # Access restriction
+  {% include "basic-auth.j2" %}
 
   # Set image format types to expire in a very long time
   location ~* ^.+\.(jpg|jpeg|gif|png|ico)$ {
@@ -38,14 +42,11 @@ server {
   # Elastic Search
   location /e {
 	  rewrite /e/(.*) /$1 break;
-	  proxy_pass http://elasticsearch_server;
 	  proxy_set_header X-Real-IP $remote_addr;
-	  proxy_set_header SSL_CLIENT_S_DN $ssl_client_s_dn;
-	  proxy_set_header X-Forwarded-Protocol https;	# sets HTTP_X_FORWARDED_PROTOCOL
- 	  proxy_set_header HTTP_X_FORWARDED_PROTO https;	# sets HTTP_X_FORWARDED_PROTOCOL
-	  proxy_ssl_session_reuse off;	
-	  proxy_redirect off;
+	  proxy_set_header X-Forwarded-For $http_x_forwarded_for;
 
+	  proxy_redirect off;
+	  proxy_pass http://elasticsearch_server;
     }
 
   # Kibana