Merge pull request #1925 from edx/e0d/ad_hoc_reporting

E0d/ad hoc reporting

playbooks/edx-east/bastion.yml → playbooks/edx-east/ad_hoc_reporting.yml

-- name: Deploy bastion
+- name: Deploy Ad Hoc Reporting Scripts 
   hosts: all
   sudo: True
   gather_facts: True
@@ -6,4 +6,4 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
-    - bastion
+    - ad_hoc_reporting 

playbooks/roles/bastion/defaults/main.yml → playbooks/roles/ad_hoc_reporting/defaults/main.yml

@@ -9,36 +9,35 @@
 #
 ##
 #
-# Defaults for role bastion
+# Defaults for role ad_hoc_reporting
 #
 
 # These users are given access
-# to the databases from the bastion
-# box, it needs to be a subset of the
+# to the databases from ad hoc reporting environment,
+# it needs to be a subset of the
 # users created on the box which is
-# COMMON_USER_INFO + BASTION_USER_INFO
+# COMMON_USER_INFO + AD_HOC_REPORTING_USER_INFO
 
-BASTION_REPLICA_USERS: []
+AD_HOC_REPORTING_USERS: []
 
-# These users are created on the bastion
-# server.
-BASTION_USER_INFO: []
+# These users are created on the ad_hoc_reporting environment
+AD_HOC_REPORTING_USER_INFO: []
 #
 # vars are namespace with the module name.
 #
-bastion_role_name: bastion
+ad_hoc_reporting_role_name: ad_hoc_reporting
 
 #
 # OS packages
 #
 
-bastion_debian_pkgs:
+ad_hoc_reporting_debian_pkgs:
   # for running ansible mysql module
   - mysql-client-core-5.5
   - libmysqlclient-dev
   # for connecting to mongo
   - mongodb-clients
 
-bastion_pip_pkgs:
+ad_hoc_reporting_pip_pkgs:
   # for running ansible mysql
   - mysql-python

playbooks/roles/bastion/meta/main.yml → playbooks/roles/ad_hoc_reporting/meta/main.yml

@@ -12,5 +12,5 @@
 #
 dependencies:
   - role: user
-    user_info: "{{ BASTION_USER_INFO }}"
+    user_info: "{{ AD_HOC_REPORTING_USER_INFO }}"
   - aws

playbooks/roles/ad_hoc_reporting/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Creates scripts and users to enable ad-hoc reporting using MySQL
+# read replicas and MongoDB secondary nodes.
+#
+# Overview:
+# Creates users and scripts for ad-hoc reporting environments from your
+# ansible var files.  You would run this role as follows
+#
+# ansible-playbook -i 'reporting.example.com,' ./ad_hoc_reporting.yml -e@/var/path/common.yml -e@/vars/path/environnment-deployment.yml
+#
+# Dependencies:
+#   - aws
+#   - user
+
+- name: install system packages
+  apt: >
+    pkg={{ item }}
+    state=present
+  with_items: ad_hoc_reporting_debian_pkgs
+
+- name: install python packages
+  pip: >
+    name="{{ item }}" state=present
+    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+  with_items: ad_hoc_reporting_pip_pkgs
+
+- name: create directories
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner=root
+    group=root
+    mode=0755
+  with_items:
+    - /edx/bin
+  
+# These templates rely on there being a global
+# read_only mysql user, you must override the default
+# in order for these templates to be written out
+- name: install common mysql replica scripts
+  template: >
+    src=edx/bin/mysql.sh.j2
+    dest=/edx/bin/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ item.script_name }}
+    mode=0755 owner=root group=root
+  with_items:
+    - db_host: "{{ EDXAPP_MYSQL_REPLICA_HOST }}"
+      db_name: "{{ EDXAPP_MYSQL_DB_NAME }}"
+      script_name: edxapp-mysql.sh
+  when: COMMON_MYSQL_READ_ONLY_PASS is defined
+
+- name: install xqueue mysql replica scripts
+  template: >
+    src=edx/bin/mysql.sh.j2
+    dest=/edx/bin/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ item.script_name }}
+    mode=0755 owner=root group=root
+  with_items:
+    - db_host: "{{ XQUEUE_MYSQL_REPLICA_HOST }}"
+      db_name: "{{ XQUEUE_MYSQL_DB_NAME }}"
+      script_name: xqueue-mysql.sh
+  when: COMMON_MYSQL_READ_ONLY_PASS is defined and XQUEUE_MYSQL_DB_HOST is defined
+  
+# These templates rely on there being a global
+# read_only mongo user, you must override the default
+# in order for these templates to be written out
+- name: install mongodb replica scripts
+  template: >
+    src=edx/bin/mongo.sh.j2
+    dest=/edx/bin/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ item.script_name }}
+    mode=0755 owner=root group=root
+  with_items:
+    - db_hosts: "{{ EDXAPP_MONGO_HOSTS }}"
+      db_name: "{{ EDXAPP_MONGO_DB_NAME }}"
+      db_port: "{{ EDXAPP_MONGO_PORT }}"
+      script_name: edxapp-mongo.sh
+    - db_hosts: "{{ FORUM_MONGO_HOSTS }}"
+      db_name: "{{ FORUM_MONGO_DATABASE }}"
+      db_port: "{{ FORUM_MONGO_PORT }}"
+      script_name: forum-mongo.sh
+  when: COMMON_MONGO_READ_ONLY_PASS is defined

playbooks/roles/ad_hoc_reporting/templates/edx/bin/mongo.sh.j2

+#!/usr/bin/env bash
+
+db_hosts={{ ",".join(item.db_hosts) }}
+from_port={{ item.db_port }}
+
+
+for host in ${db_hosts//,/ }; do
+    if [[ $(mongo ${host}:{{ item.db_port }}/{{ item.db_name }} -u {{ COMMON_MONGO_READ_ONLY_USER }} -p"{{ COMMON_MONGO_READ_ONLY_PASS }}"  --eval "printjson(db.isMaster())") =~ secondary\"\ \:\ true ]]; then
+        replica=$host
+    fi
+done
+
+if [[ -z $replica ]]; then
+    echo "No replica found for $from_db_hosts!"
+    exit 1
+fi
+
+mongo ${replica}:{{ item.db_port }}/{{ item.db_name }} -u {{ COMMON_MONGO_READ_ONLY_USER }} -p"{{ COMMON_MONGO_READ_ONLY_PASS }}"

playbooks/roles/ad_hoc_reporting/templates/edx/bin/mysql.sh.j2

+#!/usr/bin/env bash
+mysql -u {{ COMMON_MYSQL_READ_ONLY_USER }} -h {{ item.db_host }} -p"{{ COMMON_MYSQL_READ_ONLY_PASS }}" {{ item.db_name }}

playbooks/roles/bastion/tasks/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Tasks for role bastion
-#
-# Overview:
-#
-#
-# Dependencies:
-#   - common
-#
-- name: install system packages
-  apt: >
-    pkg={{','.join(bastion_debian_pkgs)}}
-    state=present
-
-- name: install bastion python packages
-  pip: >
-    name="{{ item }}" state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
-  with_items: bastion_pip_pkgs
-
-# These templates rely on there being a global
-# read_only mysql user, you must override the default
-# in order for these templates to be written out
-- template: >
-    src=mysql.sh.j2
-    dest=/home/{{ item[0] }}/{{ item[1].script_name }}
-    mode=0700 owner={{ item[0] }} group=root
-  with_nested:
-    - "{{ BASTION_REPLICA_USERS }}"
-    -
-      - db_host: "{{ EDXAPP_MYSQL_HOST }}"
-        db_name: "{{ EDXAPP_MYSQL_DB_NAME }}"
-        script_name: edxapp-rds.sh
-      - db_host: "{{ XQUEUE_MYSQL_HOST }}"
-        db_name: "{{ XQUEUE_MYSQL_DB_NAME }}"
-        script_name: xqueue-rds.sh
-      - db_host: "{{ ORA_MYSQL_HOST }}"
-        db_name: "{{ ORA_MYSQL_DB_NAME }}"
-        script_name: ora-rds.sh
-  when: COMMON_MYSQL_READ_ONLY_PASS is defined
-
-# These templates rely on there being a global
-# read_only mongo user, you must override the default
-# in order for these templates to be written out
-- template: >
-    src=mongo.sh.j2
-    dest=/home/{{ item[0] }}/{{ item[1].script_name }}
-    mode=0700 owner={{ item[0] }} group=root
-  with_nested:
-    - "{{ BASTION_REPLICA_USERS }}"
-    -
-      - db_host: "{{ EDXAPP_MONGO_HOSTS[1] }}"
-        db_name: "{{ EDXAPP_MONGO_DB_NAME }}"
-        db_port: "{{ EDXAPP_MONGO_PORT }}"
-        script_name: edxapp-mongo.sh
-      - db_host: "{{ FORUM_MONGO_HOSTS[1] }}"
-        db_name: "{{ FORUM_MONGO_DATABASE }}"
-        db_port: "{{ FORUM_MONGO_PORT }}"
-        script_name: forum-mongo.sh
-  when: COMMON_MONGO_READ_ONLY_PASS is defined

playbooks/roles/bastion/templates/mongo.sh.j2

-#!/usr/bin/env bash
-mongo {{ item[1].db_host }}:{{ item[1].db_port }}/{{ item[1].db_name }} -u {{ COMMON_MONGO_READ_ONLY_USER }} -p"{{ COMMON_MONGO_READ_ONLY_PASS }}"

playbooks/roles/bastion/templates/mysql.sh.j2

-#!/usr/bin/env bash
-mysql -u {{ COMMON_MYSQL_READ_ONLY_USER }} -h {{ item[1].db_host }} -p"{{ COMMON_MYSQL_READ_ONLY_PASS }}" {{ item[1].db_name }}

playbooks/roles/security/defaults/main.yml

@@ -30,5 +30,6 @@ SECURITY_UPGRADE_ON_ANSIBLE: false
 security_debian_pkgs: 
   - aptitude
   - unattended-upgrades
+  - gcc
 
 security_redhat_pkgs: []