Merge pull request #1660 from mitodl/cg/add_mariadb_galera

Add MariaDB Role (with galera clustering)

playbooks/roles/common/defaults/main.yml

@@ -29,6 +29,7 @@ COMMON_ENVIRONMENT: 'default_env'
 COMMON_DEPLOYMENT: 'default_deployment'
 COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
 COMMON_NPM_MIRROR_URL: 'http://registry.npmjs.org'
+COMMON_UBUNTU_APT_KEYSERVER: "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search="
 # do not include http/https
 COMMON_GIT_MIRROR: 'github.com'
 # override this var to set a different hostname
@@ -109,6 +110,10 @@ disable_edx_services: False
 # so different start scripts are generated in dev mode.
 devstack: False
 
+# Some cluster apps need special settings when in vagrant
+# due to eth0 always being the same IP address
+vagrant_cluster: False
+
 common_debian_variants:
   - Ubuntu
   - Debian

playbooks/roles/datadog/defaults/main.yml

 ---
 DATADOG_API_KEY: "SPECIFY_KEY_HERE"
 
-datadog_apt_key: "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x226AE980C7A7DA52"
+datadog_apt_key: "0x226AE980C7A7DA52"
 datadog_debian_pkgs:
   - apparmor-utils
   - build-essential

playbooks/roles/datadog/tasks/main.yml

@@ -22,7 +22,7 @@
     - datadog
 
 - name: add apt key
-  apt_key: id=C7A7DA52 url={{ datadog_apt_key }} state=present
+  apt_key: id=C7A7DA52 url={{ COMMON_UBUNTU_APT_KEYSERVER }}{{ datadog_apt_key }} state=present
   tags:
     - datadog
 

playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2

@@ -43,3 +43,8 @@ script.disable_dynamic: true
 discovery.zen.ping.unicast.hosts: ['{{hosts|join("\',\'") }}']
 
 {% endif -%}
+
+{% if vagrant_cluster|bool %}
+network:
+  host: {{ ansible_ssh_host }}
+{% endif %}

playbooks/roles/haproxy/defaults/main.yml

@@ -49,11 +49,30 @@ haproxy_default_config: |
 # desired applications
 haproxy_applications:
   - |
-    listen rabbitmq 127.0.0.1:5672
+    listen rabbitmq 127.0.0.1:35672
     mode    tcp
     balance roundrobin
     option  tcplog
     option  tcpka
-    server  rabbit01 172.23.128.10:5672 check inter 5000 rise 2 fall 3
-    server  rabbit02 172.23.129.10:5672 backup check inter 5000 rise 2 fall 3
-    server  rabbit03 172.23.130.10:5672 backup check inter 5000 rise 2 fall 3
+    server  rabbit01 192.168.33.100:5672 check inter 5000 rise 2 fall 3
+    server  rabbit02 192.168.33.110:5672 check inter 5000 rise 2 fall 3
+    server  rabbit03 192.168.33.120:5672 check inter 5000 rise 2 fall 3
+
+    listen mariadb 127.0.0.1:13306
+    mode    tcp
+    balance roundrobin
+    option  tcplog
+    option  tcpka
+    option mysql-check
+    server  galera1 192.168.33.100:3306 check weight 1
+    server  galera2 192.168.33.110:3306 check weight 1
+    server  galera3 192.168.33.120:3306 check weight 1
+
+    listen elasticsearch 127.0.0.1:19200
+    mode    tcp
+    balance roundrobin
+    option  tcplog
+    option  tcpka
+    server  galera1 192.168.33.100:9200 check weight 1
+    server  galera2 192.168.33.110:9200 check weight 1
+    server  galera3 192.168.33.120:9200 check weight 1

playbooks/roles/haproxy/meta/main.yml

@@ -18,3 +18,6 @@
 #   my_role_var0: "foo"
 #   my_role_var1: "bar"
 # }
+
+dependencies:
+  - common

playbooks/roles/haproxy/templates/haproxy.cfg.j2

 # this config needs haproxy-1.1.28 or haproxy-1.2.1
 
 global
-	log 127.0.0.1	local0
-	log 127.0.0.1	local1 notice
+	log /dev/log	local0 info
+	log /dev/log	local0 notice
 	#log loghost	local0 info
 	maxconn 4096
 	#chroot /usr/share/haproxy

playbooks/roles/mariadb/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role mariadb
+# 
+MARIADB_APT_KEY_ID: '0xcbcb082a1bb943db'
+# Note: version is determined by repo
+MARIADB_REPO: "deb http://mirrors.syringanetworks.net/mariadb/repo/10.0/ubuntu precise main"
+
+MARIADB_CREATE_DBS: yes
+MARIADB_CLUSTERED: no
+MARIADB_CLUSTER_USER_ADMIN: "mariadb_clu_root"
+MARIADB_CLUSTER_PASSWORD_ADMIN: "password"
+MARIADB_HOST_PRIV: '%'
+MARIADB_LISTEN_ALL: false
+
+MARIADB_DATABASES:
+  - "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}"
+  - "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}"
+  - "{{ ORA_MYSQL_DB_NAME|default('ora') }}"
+
+MARIADB_ANALYTICS_DATABASES:
+  - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('analytics-api') }}"
+  - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('reports') }}"
+
+MARIADB_USERS:
+  - name: "{{ EDXAPP_MYSQL_USER|default('edxapp001') }}"
+    pass: "{{ EDXAPP_MYSQL_PASSWORD|default('password') }}"
+    priv: "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ XQUEUE_MYSQL_USER|default('xqueue001') }}"
+    pass: "{{ XQUEUE_MYSQL_PASSWORD|default('password') }}"
+    priv: "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ ORA_MYSQL_USER|default('ora001') }}"
+    pass: "{{ ORA_MYSQL_PASSWORD|default('password') }}"
+    priv: "{{ ORA_MYSQL_DB_NAME|default('ora') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+    priv: "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+    priv: "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+    priv: "{{ ORA_MYSQL_DB_NAME|default('ora') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_READ_ONLY_USER|default('read_only') }}"
+    pass: "{{ COMMON_MYSQL_READ_ONLY_PASS|default('password') }}"
+    priv: "*.*:SELECT"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_ADMIN_USER|default('admin') }}"
+    pass: "{{ COMMON_MYSQL_ADMIN_PASS|default('password') }}"
+    priv: "*.*:CREATE USER"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+MARIADB_ANALYTICS_USERS:
+  - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['USER']|default('api001') }}"
+    pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['PASSWORD']|default('password') }}"
+    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['USER']|default('reports001') }}"
+    pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['PASSWORD']|default('password') }}"
+    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('analytics-api') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
+    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
+    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('reports') }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+#
+# OS packages
+#
+mariadb_debian_pkgs:
+  - python-software-properties
+  - python-mysqldb
+
+mariadb_redhat_pkgs: []
+
+mariadb_apt_repository:
+
+mariadb_solo_packages:
+  - mariadb-server
+
+mariadb_cluster_packages:
+  - mariadb-galera-server
+  - galera

playbooks/roles/mariadb/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role mariadb
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+dependencies:
+  - common

playbooks/roles/mariadb/tasks/cluster.yml

+- name: copy galera cluster config
+  template: >
+    src="etc/mysql/conf.d/galera.cnf.j2"
+    dest="/etc/mysql/conf.d/galera.cnf"
+    owner="root"
+    group="root"
+    mode=0600
+
+- name: check if we have already bootstrapped the cluster
+  stat: path=/etc/mysql/ansible_cluster_started
+  register: mariadb_bootstrap
+
+- name: stop mysql for cluster bootstrap
+  service: name=mysql state=stopped
+  when: not mariadb_bootstrap.stat.exists
+
+- name: setup bootstrap on primary
+  lineinfile: >
+    dest="/etc/mysql/conf.d/galera.cnf"
+    regexp="^wsrep_cluster_address=gcomm://{{ hostvars.keys()|sort|join(',') }}$"
+    line="wsrep_cluster_address=gcomm://"
+  when: ansible_ssh_host == hostvars[hostvars.keys()[0]].ansible_ssh_host and not mariadb_bootstrap.stat.exists
+
+
+- name: fetch debian.cnf file so start-stop will work properly
+  fetch: >
+    src=/etc/mysql/debian.cnf
+    dest=/tmp/debian.cnf
+    fail_on_missing=yes
+    flat=yes
+  when: ansible_ssh_host == hostvars[hostvars.keys()[0]].ansible_ssh_host and not mariadb_bootstrap.stat.exists
+  register: mariadb_new_debian_cnf
+
+- name: copy fetched file to other cluster members
+  copy: src=/tmp/debian.cnf dest=/etc/mysql/debian.cnf
+  when: mariadb_new_debian_cnf is defined  
+
+- name: start everything
+  service: name=mysql state=started
+  when: not mariadb_bootstrap.stat.exists
+
+- name: reset galera cluster config since we are bootstrapped
+  template: >
+    src="etc/mysql/conf.d/galera.cnf.j2"
+    dest="/etc/mysql/conf.d/galera.cnf"
+    owner="root"
+    group="root"
+    mode=0600
+  when: not mariadb_bootstrap.stat.exists
+
+- name: touch bootstrap file to confirm we are fully up
+  file: path="/etc/mysql/ansible_cluster_started" state=touch
\ No newline at end of file

playbooks/roles/mariadb/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role mariadb
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: Install pre-req debian packages
+  apt: name={{ item }} state=present
+  with_items: mariadb_debian_pkgs
+
+- name: Add mongo key
+  apt_key: url="{{ COMMON_UBUNTU_APT_KEYSERVER }}{{ MARIADB_APT_KEY_ID }}"
+
+- name: add the mariadb repo to the sources list
+  apt_repository: >
+    repo='{{ MARIADB_REPO }}'
+    state=present
+
+- name: install mariadb solo packages
+  apt: name={{ item }} update_cache=yes
+  with_items: mariadb_solo_packages
+  when: not MARIADB_CLUSTERED|bool
+
+- name: install mariadb cluster packages
+  apt: name={{ item }} update_cache=yes
+  with_items: mariadb_cluster_packages
+  when: MARIADB_CLUSTERED|bool
+
+- name: remove bind-address
+  lineinfile: >
+    dest=/etc/mysql/my.cnf
+    regexp="^bind-address\s+=\s+127\.0\.0\.1$"
+    state=absent
+  when: MARIADB_LISTEN_ALL|bool or MARIADB_CLUSTERED|bool
+
+- include: cluster.yml
+  when: MARIADB_CLUSTERED|bool
+
+- name: start everything
+  service: name=mysql state=started
+
+- name: create all databases
+  mysql_db: >
+    db={{ item }}
+    state=present
+    encoding=utf8
+  with_items: MARIADB_DATABASES
+  when: MARIADB_CREATE_DBS|bool
+
+- name: create all analytics dbs
+  mysql_db: >
+    db={{ item }}
+    state=present
+    encoding=utf8
+  with_items: MARIADB_ANALYTICS_DATABASES
+  when: MARIADB_CREATE_DBS|bool and ANALYTICS_API_CONFIG is defined
+
+- name: create all users/privs
+  mysql_user: >
+    name="{{ item.name }}"
+    password="{{ item.pass }}"
+    priv="{{ item.priv }}"
+    host="{{ item.host }}"
+    append_privs=yes
+  with_items: MARIADB_USERS
+  when: MARIADB_CREATE_DBS|bool
+
+- name: create all analytics users/privs
+  mysql_user: >
+    name="{{ item.name }}"
+    password="{{ item.pass }}"
+    priv="{{ item.priv }}"
+    host="{{ item.host }}"
+    append_privs=yes
+  with_items: MARIADB_ANALYTICS_USERS
+  when: MARIADB_CREATE_DBS|bool and ANALYTICS_API_CONFIG is defined

playbooks/roles/mariadb/templates/etc/mysql/conf.d/galera.cnf.j2

+{%- set hosts= [] -%}
+{%- for host in hostvars.keys()|sort -%}
+    {% do hosts.append(host) %}
+{%- endfor %}
+[mysqld]
+binlog_format=ROW
+innodb_autoinc_lock_mode=2
+innodb_doublewrite=1
+query_cache_size=0
+
+wsrep_provider=/usr/lib/galera/libgalera_smm.so
+wsrep_cluster_address=gcomm://{{ hosts|join(',') }}?pc.wait_prim=no
+wsrep_sst_auth={{ MARIADB_CLUSTER_USER_ADMIN }}:{{ MARIADB_CLUSTER_PASSWORD_ADMIN }}
+
+{% if vagrant_cluster|bool %}
+wsrep_node_address={{ ansible_ssh_host }}
+{% endif %}

playbooks/vagrant-cluster.yml

+- name: Configure group cluster 
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    vagrant_cluster: yes
+    mongo_cluster_members:
+      - "cluster1"
+      - "cluster2"
+      - "cluster3"
+    MONGO_CLUSTERED: yes
+    MONGO_CLUSTER_KEY: 'password'
+    mongo_create_users: no
+    ELASTICSEARCH_CLUSTERED: yes
+    MARIADB_CLUSTERED: yes
+    MARIADB_CREATE_DBS: no
+  vars_files:
+    - "group_vars/all"
+  roles:
+    - user
+    - mongo
+    - oraclejdk
+    - elasticsearch
+    - mariadb
+    - edx_ansible
+
+# Rabbit needs to be built serially
+- name: Configure group cluster serial roles 
+  hosts: all
+  sudo: True
+  serial: 1
+  gather_facts: True
+  vars:
+    rabbitmq_clustered_hosts: 
+      - "rabbit@cluster1"
+      - "rabbit@cluster2"
+      - "rabbit@cluster3"
+    rabbitmq_ip: ""
+  vars_files:
+    - "group_vars/all"
+  roles:
+    - rabbitmq
+
+# Mongo user doesn't handle slave's gracefully when
+# creating users and there are race conditions
+# in MariaDB occasionally so this play will work
+# but will also show as failed
+- name: Configure group with tasks that will always fail
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    mongo_cluster_members:
+      - "cluster1"
+      - "cluster2"
+      - "cluster3"
+    MONGO_CLUSTERED: yes
+    MONGO_CLUSTER_KEY: 'password'
+    mongo_create_users: yes
+    RABBITMQ_CLUSTERED: yes
+    MARIADB_CLUSTERED: yes
+    MARIADB_CREATE_DBS: yes
+  vars_files:
+    - "group_vars/all"
+    - "roles/analytics-api/defaults/main.yml"
+  roles:
+    - mongo
+    - mariadb

vagrant/base/cluster/Vagrantfile

+# -*- mode: ruby -*-
+
+# vi: set ft=ruby :
+
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.require_version ">= 1.5.0"
+
+$script = <<SCRIPT
+# Silly Ubuntu 12.04 doesn't have the
+# --stdin option in the passwd utility
+echo root:vagrant | chpasswd
+
+cat << EOF >> /etc/hosts
+192.168.33.100 cluster1
+192.168.33.110 cluster2
+192.168.33.120 cluster3
+EOF
+SCRIPT
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  config.vm.box     = "precise64"
+  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+
+  # Turn off shared folders
+  #config.vm.synced_folder ".", "/vagrant", id: "vagrant-root", disabled: true
+
+  # Begin cluster1
+  config.vm.define "cluster1" do |cluster1_config|
+    cluster1_config.vm.hostname = "cluster1"
+
+    cluster1_config.vm.provision "shell", inline: $script
+
+    cluster1_config.vm.network :private_network, ip: "192.168.33.100"
+
+    cluster1_config.vm.provider "virtualbox" do |v|
+        v.customize ["modifyvm", :id, "--memory", "2048"]
+        v.customize ["modifyvm", :id, "--cpus", "2"]
+    end
+  end
+  # End cluster1
+
+ # Begin cluster2
+  config.vm.define "cluster2" do |cluster2_config|
+    cluster2_config.vm.hostname = "cluster2"
+
+    cluster2_config.vm.provision "shell", inline: $script
+
+    cluster2_config.vm.network :private_network, ip: "192.168.33.110"
+
+    cluster2_config.vm.provider "virtualbox" do |v|
+        v.customize ["modifyvm", :id, "--memory", "2048"]
+        v.customize ["modifyvm", :id, "--cpus", "2"]
+    end
+  end
+  # End cluster2
+
+
+  # Begin cluster3
+  config.vm.define "cluster3" do |cluster3_config|
+    cluster3_config.vm.hostname = "cluster3"
+
+    cluster3_config.vm.provision "shell", inline: $script
+
+    cluster3_config.vm.network :private_network, ip: "192.168.33.120"
+
+    cluster3_config.vm.provider "virtualbox" do |v|
+        v.customize ["modifyvm", :id, "--memory", "2048"]
+        v.customize ["modifyvm", :id, "--cpus", "2"]
+    end
+    # Now that all machines are up, provision the group
+    # See https://github.com/mitchellh/vagrant/issues/1784 for why
+    # we do it here
+    cluster3_config.vm.provision :ansible do |ansible|
+      # point Vagrant at the location of your playbook you want to run
+      ansible.playbook = "../../../playbooks/vagrant-cluster.yml"
+      ansible.verbose = "vvv"
+      ansible.inventory_path = "inventory.ini"
+      ansible.limit = 'all'
+    end
+    
+  end
+  # End cluster3
+end

vagrant/base/cluster/ansible.cfg

+# config file for ansible -- http://ansible.github.com
+# nearly all parameters can be overridden in ansible-playbook or with command line flags
+# ansible will read ~/.ansible.cfg or /etc/ansible/ansible.cfg, whichever it finds first
+
+[defaults]
+
+jinja2_extensions=jinja2.ext.do
+host_key_checking = False
+roles_path=../../ansible-roles/roles:../../ansible-private/roles:../../ansible-roles/

vagrant/base/cluster/inventory.ini

+[cluster]
+cluster1 ansible_ssh_host=192.168.33.100 ansible_ssh_user=vagrant ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key
+cluster2 ansible_ssh_host=192.168.33.110 ansible_ssh_user=vagrant ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key
+cluster3 ansible_ssh_host=192.168.33.120 ansible_ssh_user=vagrant ansible_ssh_private_key_file=~/.vagrant.d/insecure_private_key