Merge pull request #852 from edx/jarv/add-bastion-db-access

adding bastion role and task to setup access to the dbs from it

playbooks/edx-east/bastion.yml

+- name: Deploy bastion
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - bastion

playbooks/edx-east/create_all_user_types.yml

@@ -15,7 +15,7 @@
       when: user is not defined
     - name: give access with no sudo
       set_fact:
-        user_info:
+        CUSTOM_USER_INFO:
           - name: "{{ user }}"
             github: true
           - name: test-admin-user
@@ -28,4 +28,5 @@
             type: restricted
             sudoers_template: 99-analytics-manage-cmds.j2
   roles:
-    - user
+    - role: user
+      user_info: "{{ CUSTOM_USER_INFO }}"

playbooks/edx-east/create_dbs.yml

@@ -33,21 +33,8 @@
   vars_prompt:
     # passwords use vars_prompt so they aren't in the
     # bash history
-    - name: "edxapp_db_root_pass"
-      prompt: "Password for edxapp root mysql user (enter to skip)"
-      default: "None"
-      private: True
-    - name: "xqueue_db_root_pass"
-      prompt: "Password for xqueue root mysql user (enter to skip)"
-      default: "None"
-      private: True
-    - name: "ora_db_root_pass"
-      prompt: "Password for ora root mysql user (enter to skip)"
-      default: "None"
-      private: True
-    - name: "discern_db_root_pass"
-      prompt: "Password for discern root mysql user (enter to skip)"
-      default: "None"
+    - name: "db_root_pass"
+      prompt: "Password for root mysql user"
       private: True
 
 
@@ -72,17 +59,58 @@
           - db_name: "{{ EDXAPP_MYSQL_DB_NAME|default('None') }}"
             db_host: "{{ EDXAPP_MYSQL_HOST|default('None') }}"
             db_user: "{{ edxapp_db_root_user }}"
-            db_pass: "{{ edxapp_db_root_pass }}"
+            db_pass: "{{ db_root_pass }}"
           - db_name: "{{ XQUEUE_MYSQL_DB_NAME|default('None') }}"
             db_host: "{{ XQUEUE_MYSQL_HOST|default('None') }}"
             db_user: "{{ xqueue_db_root_user }}"
-            db_pass: "{{ xqueue_db_root_pass }}"
+            db_pass: "{{ db_root_pass }}"
           - db_name: "{{ ORA_MYSQL_DB_NAME|default('None') }}"
             db_host: "{{ ORA_MYSQL_HOST|default('None') }}"
             db_user: "{{ ora_db_root_user }}"
-            db_pass: "{{ ora_db_root_pass }}"
+            db_pass: "{{ db_root_pass }}"
+
+
+    - name: assign mysql user permissions for read_only user
+      mysql_user:
+        name: "{{ COMMON_MYSQL_READ_ONLY_USER }}"
+        priv: "*.*:SELECT"
+        password: "{{ COMMON_MYSQL_READ_ONLY_PASS }}"
+        login_host: "{{ item.db_host }}"
+        login_user: "{{ item.db_user }}"
+        login_password: "{{ item.db_pass }}"
+        host: '%'
+      with_items:
+        - db_host: "{{ EDXAPP_MYSQL_HOST|default('None') }}"
+          db_user: "{{ edxapp_db_root_user }}"
+          db_pass: "{{ db_root_pass }}"
+        - db_host: "{{ XQUEUE_MYSQL_HOST|default('None') }}"
+          db_user: "{{ xqueue_db_root_user }}"
+          db_pass: "{{ db_root_pass }}"
+        - db_host: "{{ ORA_MYSQL_HOST|default('None') }}"
+          db_user: "{{ ora_db_root_user }}"
+          db_pass: "{{ db_root_pass }}"
+
+    - name: assign mysql user permissions for admin user
+      mysql_user:
+        name: "{{ COMMON_MYSQL_ADMIN_USER }}"
+        priv: "*.*:CREATE USER"
+        password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+        login_host: "{{ item.db_host }}"
+        login_user: "{{ item.db_user }}"
+        login_password: "{{ item.db_pass }}"
+        host: '%'
+      with_items:
+        - db_host: "{{ EDXAPP_MYSQL_HOST|default('None') }}"
+          db_user: "{{ edxapp_db_root_user }}"
+          db_pass: "{{ db_root_pass }}"
+        - db_host: "{{ XQUEUE_MYSQL_HOST|default('None') }}"
+          db_user: "{{ xqueue_db_root_user }}"
+          db_pass: "{{ db_root_pass }}"
+        - db_host: "{{ ORA_MYSQL_HOST|default('None') }}"
+          db_user: "{{ ora_db_root_user }}"
+          db_pass: "{{ db_root_pass }}"
 
-    - name: assign mysql user permissions for db user
+    - name: assign mysql user permissions for db users
       mysql_user:
         name: "{{ item.db_user_to_modify }}"
         priv: "{{ item.db_name }}.*:SELECT,INSERT,UPDATE,DELETE"
@@ -99,19 +127,19 @@
         - db_name: "{{ EDXAPP_MYSQL_DB_NAME|default('None') }}"
           db_host: "{{ EDXAPP_MYSQL_HOST|default('None') }}"
           db_user: "{{ edxapp_db_root_user|default('None') }}"
-          db_pass: "{{ edxapp_db_root_pass|default('None') }}"
+          db_pass: "{{ db_root_pass|default('None') }}"
           db_user_to_modify: "{{ EDXAPP_MYSQL_USER }}"
           db_user_to_modify_pass: "{{ EDXAPP_MYSQL_PASSWORD }}"
         - db_name: "{{ XQUEUE_MYSQL_DB_NAME|default('None') }}"
           db_host: "{{ XQUEUE_MYSQL_HOST|default('None') }}"
           db_user: "{{ xqueue_db_root_user|default('None') }}"
-          db_pass: "{{ xqueue_db_root_pass|default('None') }}"
+          db_pass: "{{ db_root_pass|default('None') }}"
           db_user_to_modify: "{{ XQUEUE_MYSQL_USER }}"
           db_user_to_modify_pass: "{{ XQUEUE_MYSQL_PASSWORD }}"
         - db_name: "{{ ORA_MYSQL_DB_NAME|default('None') }}"
           db_host: "{{ ORA_MYSQL_HOST|default('None') }}"
           db_user: "{{ ora_db_root_user|default('None') }}"
-          db_pass: "{{ ora_db_root_pass|default('None') }}"
+          db_pass: "{{ db_root_pass|default('None') }}"
           db_user_to_modify: "{{ ORA_MYSQL_USER }}"
           db_user_to_modify_pass: "{{ ORA_MYSQL_PASSWORD }}"
 
@@ -139,18 +167,18 @@
         - db_name: "{{ EDXAPP_MYSQL_DB_NAME|default('None') }}"
           db_host: "{{ EDXAPP_MYSQL_HOST|default('None') }}"
           db_user: "{{ edxapp_db_root_user|default('None') }}"
-          db_pass: "{{ edxapp_db_root_pass|default('None') }}"
+          db_pass: "{{ db_root_pass|default('None') }}"
           db_user_to_modify: "{{ EDXAPP_MYSQL_USER }}"
           db_user_to_modify_pass: "{{ EDXAPP_MYSQL_PASSWORD }}"
         - db_name: "{{ XQUEUE_MYSQL_DB_NAME|default('None') }}"
           db_host: "{{ XQUEUE_MYSQL_HOST|default('None') }}"
           db_user: "{{ xqueue_db_root_user|default('None') }}"
-          db_pass: "{{ xqueue_db_root_pass|default('None') }}"
+          db_pass: "{{ db_root_pass|default('None') }}"
           db_user_to_modify: "{{ XQUEUE_MYSQL_USER }}"
           db_user_to_modify_pass: "{{ XQUEUE_MYSQL_PASSWORD }}"
         - db_name: "{{ ORA_MYSQL_DB_NAME|default('None') }}"
           db_host: "{{ ORA_MYSQL_HOST|default('None') }}"
           db_user: "{{ ora_db_root_user|default('None') }}"
-          db_pass: "{{ ora_db_root_pass|default('None') }}"
+          db_pass: "{{ db_root_pass|default('None') }}"
           db_user_to_modify: "{{ ORA_MYSQL_USER }}"
           db_user_to_modify_pass: "{{ ORA_MYSQL_PASSWORD }}"

playbooks/edx-east/create_user.yml

@@ -12,16 +12,17 @@
       when: user is not defined
     - name: give access with no sudo
       set_fact:
-        user_info:
+        CUSTOM_USER_INFO:
           - name: "{{ user }}"
             github: true
       when: give_sudo is not defined
     - name: give access with sudo
       set_fact:
-        user_info:
+        CUSTOM_USER_INFO:
           - name: "{{ user }}"
             type: admin
             github: true
       when: give_sudo is defined
   roles:
-    - user
+    - role: user
+      user_info: "{{ CUSTOM_USER_INFO }}"

playbooks/edx-west/prod-jumpbox.yml

@@ -9,7 +9,7 @@
     - common
     - supervisor
     - role: user
-      user_data:
+      USER_INFO:
         - name: sefk
           github: true
           type: admin

playbooks/roles/bastion/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+#
+# Defaults for role bastion
+#
+
+# These users are given access
+# to the databases from the bastion
+# box, it needs to be a subset of the
+# users created on the box which is
+# COMMON_USER_INFO + BASTION_USER_INFO
+
+BASTION_REPLICA_USERS: []
+
+# These users are created on the bastion
+# server.
+BASTION_USER_INFO: []
+#
+# vars are namespace with the module name.
+#
+bastion_role_name: bastion
+
+#
+# OS packages
+#
+
+bastion_debian_pkgs:
+  # for running ansible mysql module
+  - mysql-client-core-5.5
+  - libmysqlclient-dev
+  # for connecting to mongo
+  - mongodb-clients
+
+bastion_pip_pkgs:
+  # for running ansible mysql
+  - mysql-python

playbooks/roles/bastion/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role bastion
+#
+dependencies:
+  - role: user
+    user_info: "{{ BASTION_USER_INFO }}"
+  - aws

playbooks/roles/bastion/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role bastion
+#
+# Overview:
+#
+#
+# Dependencies:
+#   - common
+#
+- name: install system packages
+  apt: >
+    pkg={{','.join(bastion_debian_pkgs)}}
+    state=present
+
+- name: install bastion python packages
+  pip: >
+    name="{{ item }}" state=present
+    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+  with_items: bastion_pip_pkgs
+
+- template: >
+    src=mysql.sh.j2
+    dest=/home/{{ item[0] }}/{{ item[1].script_name }}
+    mode=0700 owner={{ item[0] }} group=root
+  with_nested:
+    - "{{ BASTION_REPLICA_USERS }}"
+    -
+      - db_host: "{{ EDXAPP_MYSQL_HOST }}"
+        db_name: "{{ EDXAPP_MYSQL_DB_NAME }}"
+        script_name: edxapp-rds.sh
+      - db_host: "{{ XQUEUE_MYSQL_HOST }}"
+        db_name: "{{ XQUEUE_MYSQL_DB_NAME }}"
+        script_name: xqueue-rds.sh
+      - db_host: "{{ ORA_MYSQL_HOST }}"
+        db_name: "{{ ORA_MYSQL_DB_NAME }}"
+        script_name: ora-rds.sh
+
+- template: >
+    src=mongo.sh.j2
+    dest=/home/{{ item[0] }}/{{ item[1].script_name }}
+    mode=0700 owner={{ item[0] }} group=root
+  with_nested:
+    - "{{ BASTION_REPLICA_USERS }}"
+    -
+      - db_host: "{{ EDXAPP_MONGO_HOSTS[1] }}"
+        db_name: "{{ EDXAPP_MONGO_DB_NAME }}"
+        db_port: "{{ EDXAPP_MONGO_PORT }}"
+        script_name: edxapp-mongo.sh
+      - db_host: "{{ FORUM_MONGO_HOSTS[1] }}"
+        db_name: "{{ FORUM_MONGO_DATABASE }}"
+        db_port: "{{ FORUM_MONGO_PORT }}"
+        script_name: forum-mongo.sh

playbooks/roles/bastion/templates/mongo.sh.j2

+#!/usr/bin/env bash
+mongo {{ item[1].db_host }}:{{ item[1].db_port }}/{{ item[1].db_name }} -u {{ COMMON_MONGO_READ_ONLY_USER }} -p"{{ COMMON_MONGO_READ_ONLY_PASS }}"

playbooks/roles/bastion/templates/mysql.sh.j2

+#!/usr/bin/env bash
+mysql -u {{ COMMON_MYSQL_READ_ONLY_USER }} -h {{ item[1].db_host }} -p"{{ COMMON_MYSQL_READ_ONLY_PASS }}" {{ item[1].db_name }}

playbooks/roles/common/defaults/main.yml

@@ -29,6 +29,18 @@ COMMON_CUSTOM_DHCLIENT_CONFIG: false
 
 COMMON_MOTD_TEMPLATE: "motd.tail.j2"
 
+# These are two special accounts across all databases
+# the read only user is is granted select privs on all dbs
+# the admin user is granted create user privs on all dbs
+
+COMMON_MYSQL_READ_ONLY_USER: 'read_only'
+COMMON_MYSQL_READ_ONLY_PASS: 'password'
+COMMON_MYSQL_ADMIN_USER: 'admin'
+COMMON_MYSQL_ADMIN_PASS: 'password'
+
+COMMON_MONGO_READ_ONLY_USER: 'read_only'
+COMMON_MONGO_READ_ONLY_PASS: 'password'
+
 common_debian_pkgs:
   - ntp
   - ack-grep

playbooks/roles/common/meta/main.yml

 ---
 dependencies:
-  - user
+  - role: user
+    user_info: "{{ COMMON_USER_INFO }}"
+
+

playbooks/roles/edxapp/defaults/main.yml

@@ -32,6 +32,9 @@ EDXAPP_MONGO_DB_NAME: 'edxapp'
 EDXAPP_MYSQL_DB_NAME: 'edxapp'
 EDXAPP_MYSQL_USER: 'edxapp001'
 EDXAPP_MYSQL_PASSWORD: 'password'
+EDXAPP_MYSQL_PASSWORD_READ_ONLY: 'password'
+EDXAPP_MYSQL_PASSWORD_ADMIN: 'password'
+
 EDXAPP_MYSQL_HOST: 'localhost'
 EDXAPP_MYSQL_PORT: '3306'
 

playbooks/roles/user/defaults/main.yml

@@ -11,15 +11,12 @@
 # Vars for role user
 #
 
+
 #
 # vars are namespace with the module name.
 #
 user_role_name: user
 
-# Role parameters
-# Override this list
-user_info: []
-
 # override this var to add a prefix to the prompt
 # also need to set commont_update_bashrc for to
 # update the system bashrc default
@@ -31,3 +28,11 @@ USER_CMD_PROMPT: ""
 # this var
 user_rbash_links:
   - /usr/bin/sudo
+
+# parameter for this role,
+# must be set when called and should NOT
+# be set in extra vars since it
+# will take precedence over the paramter
+user_info: []
+
+

playbooks/roles/user/tasks/main.yml

@@ -102,9 +102,13 @@
 
 # authorized_keys2 used here so that personal
 # keys can be copied to authorized_keys
+# force is set to yes here, otherwise the keys
+# won't update if they haven't changed on teh github
+# side
 - name: copy github key[s] to .ssh/authorized_keys2
   get_url:
     url=https://github.com/{{ item.name }}.keys
+    force=yes
     dest=/home/{{ item.name }}/.ssh/authorized_keys2 mode=0640
     owner={{ item.name }}
   when: item.github is defined

util/jenkins/ansible-provision.sh

@@ -172,7 +172,7 @@ instance_tags:
     owner: $BUILD_USER
 root_ebs_size: $root_ebs_size
 name_tag: $name_tag
-user_info:
+COMMON_USER_INFO:
   - name: ${github_username}
     github: true
     type: admin