Merge branch 'master' into jarv/provision

cloudformation_templates/edx-reference-architecture.json

@@ -1602,6 +1602,12 @@
           },
           {
             "IpProtocol":"tcp",
+            "FromPort":"9997",
+            "ToPort":"9997",
+            "CidrIp":"0.0.0.0/0"
+          },
+          {
+            "IpProtocol":"tcp",
             "FromPort":"10016",
             "ToPort":"10016",
             "CidrIp":"0.0.0.0/0"
@@ -1636,6 +1642,12 @@
           },
           {
             "IpProtocol":"tcp",
+            "FromPort":"9997",
+            "ToPort":"9997",
+            "CidrIp":"0.0.0.0/0"
+          },
+          {
+            "IpProtocol":"tcp",
             "FromPort":"9418",
             "ToPort":"9418",
             "CidrIp":"0.0.0.0/0"

playbooks/edx-east/edx_feanilsandbox.yml

@@ -10,10 +10,9 @@
     - datadog
     - nginx
     - role: 'edxapp'
-      EDXAPP_LMS_NGINX_PORT: 80
-      EDXAPP_CMS_NGINX_PORT: 80
       edxapp_lms_env: 'lms.envs.load_test'
-      edx_platform_commit: 'master'
+      edx_platform_commit: 'release'
+    - splunkforwarder
 - hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_worker
   sudo: True
   vars_files:
@@ -27,7 +26,8 @@
     - role: 'edxapp'
       edxapp_lms_env: 'lms.envs.load_test'
       celery_worker: True
-      edx_platform_commit: 'master'
+      edx_platform_commit: 'release'
+    - splunkforwarder
 - hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_xserver
   sudo: True
   vars_files:
@@ -37,6 +37,7 @@
     - common
     - nginx
     - xserver
+    - splunkforwarder
 - hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_rabbitmq
   serial: 1
   sudo: True
@@ -46,6 +47,7 @@
   roles:
     - common
     - rabbitmq
+    - splunkforwarder
 - hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_xqueue
   sudo: True
   vars_files:
@@ -55,3 +57,4 @@
     - common
     - nginx
     - xqueue
+    - splunkforwarder

playbooks/group_vars/all

@@ -8,6 +8,9 @@ log_base_dir: "{{ storage_base_dir }}/logs"
 venv_dir: /opt/edx
 os_name: ubuntu
 
+ENV_NAME: 'default_env'
+ENV_TYPE: 'default_type'
+
 # these pathes are relative to the playbook dir
 # directory for secret settings (keys, etc)
 secure_dir: 'secure_example'

playbooks/roles/common/tasks/main.yml

@@ -51,13 +51,13 @@
   - update
 
 - name: common | Create log directory
-  file: path=$log_base_dir state=directory mode=2770 group=adm owner=syslog
+  file: path={{log_base_dir}} state=directory mode=2755 group=adm owner=syslog
   tags:
   - pre_install
   - update
 
 - name: common | Create alias from app_base_dir to the log_base_dir
-  file: state=link src=$log_base_dir path=$app_base_dir/log
+  file: state=link src={{log_base_dir}} path={{app_base_dir}}/log
   tags:
   - pre_install
   - logging

playbooks/roles/edxapp/tasks/cms.yml

@@ -24,7 +24,7 @@
   when: celery_worker is not defined
 
 - name: Create CMS log target directory
-  file: path={{log_base_dir}}/cms state=directory owner=syslog group=adm mode=2770
+  file: path={{log_base_dir}}/cms state=directory owner=syslog group=syslog mode=2750
   tags:
   - cms 
   - cms-env

playbooks/roles/edxapp/tasks/lms-preview.yml

@@ -16,7 +16,7 @@
   - lms-preview-env
 
 - name: Create lms-preview log target directory
-  file: path={{log_base_dir}}/lms-preview state=directory owner=syslog group=adm mode=2770
+  file: path={{log_base_dir}}/lms-preview state=directory owner=syslog group=syslog mode=2750
   tags:
   - lms-preview
   - lms-preview-env

playbooks/roles/edxapp/tasks/lms.yml

@@ -14,7 +14,7 @@
   - update
 
 - name: Create lms log target directory
-  file: path={{log_base_dir}}/lms state=directory owner=syslog group=adm mode=2770
+  file: path={{log_base_dir}}/lms state=directory owner=syslog group=syslog mode=2750
   tags:
   - lms
   - lms-env

playbooks/roles/nginx/tasks/main.yml

@@ -29,7 +29,7 @@
   - update
 
 - name: nginx | Create nginx log file location (just in case)
-  file: path={{log_base_dir}}/nginx state=directory owner=syslog group=adm mode=2770
+  file: path={{log_base_dir}}/nginx state=directory owner=syslog group=syslog mode=2770 recurse=yes
   tags:
   - nginx
   - logging

playbooks/roles/splunkforwarder/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Vars for role splunk
+# 
+
+#
+# vars are namespace with the module name.
+#
+splunk_role_name: 'splunk'
+
+SPLUNKFORWARDER_SERVER: 'localhost:9997'
+
+SPLUNKFORWARDER_PACKAGE_LOCATION: !!null
+SPLUNKFORWARDER_DEB: !!null
+SPLUNKFORWARDER_PASSWORD: !!null
+
+SPLUNKFORWARDER_LOG_ITEMS:
+  - directory: '{{log_base_dir}}'
+    recursive: true
+    index: '{{ENV_TYPE}}-{{ENV_NAME}}'
+    sourcetype: 'edx'
+  - directory: '/var/log'
+    recursive: true
+    index: '{{ENV_TYPE}}-{{ENV_NAME}}'
+    sourcetype: 'syslog'
+  - directory: '{{log_base_dir}}/nginx'
+    recursive: true
+    index: '{{ENV_TYPE}}-{{ENV_NAME}}'
+    sourcetype: 'nginx'
+
+#
+# OS packages
+#
+splunk_debian_pkgs:
+  - gdebi
+
+splunk_redhat_pkgs: []
+
+splunkforwarder_output_dir: '/opt/splunkforwarder/'

playbooks/roles/splunkforwarder/handlers/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role splunk
+# 
+# Overview:
+# 
+#
+
+# Restart Splunk
+- name: splunkforwarder | restart splunkforwarder
+  service: name=splunk state=restarted

playbooks/roles/splunkforwarder/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role splunk
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+# Install Splunk Forwarder
+- name: splunkforwarder| install splunkforwarder specific system packages
+  apt: pkg={{','.join(splunk_debian_pkgs)}} state=present
+  tags:
+  - splunk
+  - install
+  - update
+
+- name: splunkforwarder | download the splunk deb
+  get_url: >
+    dest="/tmp/{{SPLUNKFORWARDER_DEB}}"
+    url="{{SPLUNKFORWARDER_PACKAGE_LOCATION}}{{SPLUNKFORWARDER_DEB}}"
+  register: download_deb
+
+- name: splunkforwarder | install splunk forwarder
+  shell: gdebi -nq /tmp/{{SPLUNKFORWARDER_DEB}}
+  when: download_deb.changed
+
+# Create splunk user
+- name: splunkforwarder | create splunk user
+  user: name=splunk group=splunk createhome=no state=present append=yes groups=syslog
+  when: download_deb.changed
+
+# Need to start splunk manually so that it can create various files
+# and directories that aren't created till the first run and are needed
+# to run some of the below commands.
+- name: splunkforwarder | start splunk manually
+  shell: >
+    {{splunkforwarder_output_dir}}/bin/splunk start --accept-license --answer-yes --no-prompt
+      creates={{splunkforwarder_output_dir}}/var/lib/splunk
+  when: download_deb.changed
+  register: started_manually
+
+- name: splunkforwarder | stop splunk manually
+  shell: >
+    {{splunkforwarder_output_dir}}/bin/splunk stop --accept-license --answer-yes --no-prompt
+  when: download_deb.changed and started_manually.changed
+
+- name: splunkforwarder | create boot script
+  shell: >
+    {{splunkforwarder_output_dir}}/bin/splunk enable boot-start -user splunk --accept-license --answer-yes --no-prompt
+      creates=/etc/init.d/splunk 
+  register: create_boot_script
+  when: download_deb.changed
+  notify: splunkforwarder | restart splunkforwarder
+
+# Update credentials
+- name: splunkforwarder | update admin pasword
+  shell: "{{splunkforwarder_output_dir}}/bin/splunk edit user admin -password {{SPLUNKFORWARDER_PASSWORD}} -auth admin:changeme --accept-license --answer-yes --no-prompt"
+  when: download_deb.changed
+  notify: splunkforwarder | restart splunkforwarder
+
+- name: splunkforwarder | add chkconfig to init script
+  shell: 'sed -i -e "s/\/bin\/sh/\/bin\/sh\n# chkconfig: 235 98 55/" /etc/init.d/splunk'
+  when: download_deb.changed and create_boot_script.changed
+  notify: splunkforwarder | restart splunkforwarder
+
+# Ensure permissions on splunk content
+- name: splunkforwarder | ensure splunk forder permissions
+  file: path={{splunkforwarder_output_dir}} state=directory recurse=yes owner=splunk group=splunk
+  when: download_deb.changed
+  notify: splunkforwarder | restart splunkforwarder
+
+# Drop template files.
+- name: splunkforwarder | drop input configuration
+  template:
+    src=opt/splunkforwarder/etc/system/local/inputs.conf.j2
+    dest=/opt/splunkforwarder/etc/system/local/inputs.conf
+    owner=splunk
+    group=splunk
+    mode=644
+  notify: splunkforwarder | restart splunkforwarder
+
+- name: splunkforwarder | create outputs config file
+  template:
+    src=opt/splunkforwarder/etc/system/local/outputs.conf.j2
+    dest=/opt/splunkforwarder/etc/system/local/outputs.conf
+    owner=splunk
+    group=splunk
+    mode=644
+  notify: splunkforwarder | restart splunkforwarder

playbooks/roles/splunkforwarder/templates/opt/splunkforwarder/etc/system/local/inputs.conf.j2

+[default]
+host = {{ansible_hostname}}
+
+{% for loggable in SPLUNKFORWARDER_LOG_ITEMS%}
+[monitor://{{loggable.directory}}]
+recursive = {{loggable.recursive|default(false)}}
+{% if loggable.sourcetype is defined %}
+sourcetype = {{loggable.sourcetype}}
+{% endif %}
+{% if loggable.index is defined %}
+index = {{loggable.index}}
+{% endif %}
+{% endfor %}

playbooks/roles/splunkforwarder/templates/opt/splunkforwarder/etc/system/local/outputs.conf.j2

+[tcpout]
+defaultGroup = default_output_server
+
+[tcpout:default_output_server]
+server = {{SPLUNKFORWARDER_SERVER}}
+
+[tcpout-server://{{SPLUNKFORWARDER_SERVER}}]
+