Added elasticsearch index rotation and optimization scripts and cron tasks

playbooks/roles/logstash/defaults/main.yml

 ---
+LOGSTASH_DAYS_TO_KEEP: 30
+LOGSTASH_ROTATE: true
 logstash_app_dir: /edx/app/logstash
 logstash_log_dir: /edx/var/log/logstash
 logstash_data_dir: /edx/var/logstash/file_logs
 logstash_syslog_port: 514
 logstash_file: logstash-1.3.3-flatjar.jar
 logstash_url: "https://download.elasticsearch.org/logstash/logstash/{{ logstash_file }}"
+logstash_python_requirements:
+  - pyes==0.19.0
+logstash_scripts_repo: https://github.com/crashdump/logstash-elasticsearch-scripts
+logstash_rotate_cron:
+  hour: 5
+  minute: 42
+logstash_optimize_cron:
+  hour: 6
+  minute: 15

playbooks/roles/logstash/tasks/main.yml

@@ -46,4 +46,29 @@
 - name: Enable logstash es index template
   shell: chdir=/etc/elasticsearch executable=/bin/bash curl -XPUT 'http://localhost:9200/_template/template_logstash' -d @template_logstash.json
 
-# TODO: Add index rotations for kibana
+
+- name: Install python requirements
+  pip: name={{ item }} state=present
+  with_items: logstash_python_requirements
+
+- name: Checkout logstash rotation scripts
+  git: repo={{ logstash_scripts_repo }} dest={{ logstash_app_dir }}/share/logstash-elasticsearch-scripts
+  when: LOGSTASH_ROTATE|bool
+
+- name: Setup cron to run rotation
+  cron: >
+    user=root
+    name="Elasticsearch logstash index rotation"
+    hour={{ logstash_rotate_cron.hour }}
+    minute={{ logstash_rotate_cron.minute }}
+    job="/usr/bin/python {{ logstash_app_dir }}/share/logstash-elasticsearch-scripts/logstash_index_cleaner.py -d {{ LOGSTASH_DAYS_TO_KEEP }} > {{ logstash_log_dir }}/rotation_cron"
+  when: LOGSTASH_ROTATE|bool
+
+- name: Setup cron to run rotation
+  cron: >
+    user=root
+    name="Elasticsearch logstash index optimization"
+    hour={{ logstash_optimize_cron.hour }}
+    minute={{ logstash_optimize_cron.minute }}
+    job="/usr/bin/python {{ logstash_app_dir }}/share/logstash-elasticsearch-scripts/logstash_index_optimize.py -d {{ LOGSTASH_DAYS_TO_KEEP }} > {{ logstash_log_dir }}/optimize_cron"
+  when: LOGSTASH_ROTATE|bool