Merge pull request #3729 from panessa/mariadb

Update mariadb role for Xenial and add groups capability

playbooks/roles/mariadb/defaults/main.yml

@@ -10,9 +10,11 @@
 ##
 # Defaults for role mariadb
 # 
+MARIADB_APT_KEY_XENIAL_ID: '0xF1656F24C74CD1D8'
 MARIADB_APT_KEY_ID: '0xcbcb082a1bb943db'
+
 # Note: version is determined by repo
-MARIADB_REPO: "deb http://mirrors.syringanetworks.net/mariadb/repo/10.0/ubuntu precise main"
+MARIADB_REPO: "deb http://mirrors.syringanetworks.net/mariadb/repo/10.0/ubuntu {{ ansible_distribution_release }} main"
 
 MARIADB_CREATE_DBS: yes
 MARIADB_CLUSTERED: no
@@ -29,69 +31,107 @@ MARIADB_HAPROXY_HOSTS:
 MARIADB_LISTEN_ALL: false
 
 MARIADB_DATABASES:
-  - "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}"
-  - "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}"
-
-MARIADB_ANALYTICS_DATABASES:
-  - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('analytics-api') }}"
-  - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('reports') }}"
+  - {
+      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ ANALYTICS_API_DEFAULT_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}",
+      encoding: "utf8" 
+    }
+  - {
+      db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}",
+      encoding: "latin1" 
+    }
 
 MARIADB_USERS:
-  - name: "{{ EDXAPP_MYSQL_USER|default('edxapp001') }}"
-    pass: "{{ EDXAPP_MYSQL_PASSWORD|default('password') }}"
-    priv: "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ XQUEUE_MYSQL_USER|default('xqueue001') }}"
-    pass: "{{ XQUEUE_MYSQL_PASSWORD|default('password') }}"
-    priv: "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
-    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
-    priv: "{{ EDXAPP_MYSQL_DB_NAME|default('edxapp') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
-    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
-    priv: "{{ XQUEUE_MYSQL_DB_NAME|default('xqueue') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ COMMON_MYSQL_READ_ONLY_USER|default('read_only') }}"
-    pass: "{{ COMMON_MYSQL_READ_ONLY_PASS|default('password') }}"
-    priv: "*.*:SELECT"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ COMMON_MYSQL_ADMIN_USER|default('admin') }}"
-    pass: "{{ COMMON_MYSQL_ADMIN_PASS|default('password') }}"
-    priv: "*.*:CREATE USER"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ EDX_NOTES_API_MYSQL_DB_USER|default('notes001') }}"
-    pass: "{{ EDX_NOTES_API_MYSQL_DB_PASS|default('secret') }}"
-    priv: "{{ EDX_NOTES_API_MYSQL_DB_NAME|default('edx-notes-api') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-MARIADB_ANALYTICS_USERS:
-  - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['USER']|default('api001') }}"
-    pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['PASSWORD']|default('password') }}"
-    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['USER']|default('reports001') }}"
-    pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['PASSWORD']|default('password') }}"
-    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
-    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
-    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('analytics-api') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
-
-  - name: "{{ COMMON_MYSQL_MIGRATE_USER|default('migrate') }}"
-    pass: "{{ COMMON_MYSQL_MIGRATE_PASSWORD|default('password') }}"
-    priv: "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('reports') }}.*:ALL"
-    host: "{{ MARIADB_HOST_PRIV }}"
+  - {
+      db: "{{ ECOMMERCE_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ ECOMMERCE_DATABASE_USER | default(None) }}",
+      pass: "{{ ECOMMERCE_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}",
+      user: "{{ INSIGHTS_MYSQL_USER | default(None) }}",
+      pass: "{{ INSIGHTS_MYSQL_USER | default(None) }}"
+    }
+  - {
+      db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}",
+      user: "{{ XQUEUE_MYSQL_USER | default(None) }}",
+      pass: "{{ XQUEUE_MYSQL_PASSWORD | default(None) }}"
+
+    }
+  - {
+      db: "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}",
+      user: "{{ EDXAPP_MYSQL_USER | default(None) }}",
+      pass: "{{ EDXAPP_MYSQL_PASSWORD | default(None) }}"
+   }
+  - {
+      db: "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}",
+      user: "{{ EDXAPP_MYSQL_CSMH_USER | default(None) }}",
+      pass: "{{ EDXAPP_MYSQL_CSMH_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ PROGRAMS_DATABASE_USER | default(None) }}",
+      pass: "{{ PROGRAMS_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_NAME | default(None) }}",
+      user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_USER | default(None) }}",
+      pass: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}",
+      user: "{{ HIVE_METASTORE_DATABASE_USER | default(None) }}",
+      pass: "{{ HIVE_METASTORE_DATABASE_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ CREDENTIALS_MYSQL_USER | default(None) }}",
+      pass: "{{ CREDENTIALS_MYSQL_PASSWORD | default(None) }}"
+    }
+  - {
+      db: "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ DISCOVERY_MYSQL_USER | default(None) }}",
+      pass: "{{ DISCOVERY_MYSQL_PASSWORD | default(None) }}"
+    }
 
 #
 # OS packages

playbooks/roles/mariadb/tasks/cluster.yml

@@ -17,17 +17,17 @@
 - name: setup bootstrap on primary
   lineinfile:
     dest: "/etc/mysql/conf.d/galera.cnf"
-    regexp: "^wsrep_cluster_address=gcomm://{{ hostvars.keys()|sort|join(',') }}$"
+    regexp: "^wsrep_cluster_address=gcomm://{{ groups[group_names[0]]|sort|join(',') }}$"
     line: "wsrep_cluster_address=gcomm://"
-  when: ansible_hostname == hostvars[hostvars.keys()[0]].ansible_hostname and not mariadb_bootstrap.stat.exists
+  when: inventory_hostname == hostvars[groups[group_names[0]][0]].inventory_hostname and not mariadb_bootstrap.stat.exists
 
 - name: fetch debian.cnf file so start-stop will work properly
   fetch:
-    src: /etc/mysql/debian.cnf
-    dest: /tmp/debian.cnf
+    src: "/etc/mysql/debian.cnf"
+    dest: "/tmp/debian.cnf"
     fail_on_missing: yes
     flat: yes
-  when: ansible_hostname == hostvars[hostvars.keys()[0]].ansible_hostname and not mariadb_bootstrap.stat.exists
+  when: inventory_hostname == hostvars[groups[group_names[0]][0]].inventory_hostname and not mariadb_bootstrap.stat.exists
   register: mariadb_new_debian_cnf
 
 - name: copy fetched file to other cluster members
@@ -53,5 +53,10 @@
 # This is needed for mysql-check in haproxy or other mysql monitor
 # scripts to prevent haproxy checks exceeding `max_connect_errors`.
 - name: create haproxy monitor user
-  command: "mysql -e \"INSERT INTO mysql.user (Host,User) values ('{{ item }}','{{ MARIADB_HAPROXY_USER }}'); FLUSH PRIVILEGES;\""
-  with_items: "{{ MARIADB_HAPROXY_HOSTS }}"
+  mysql_user:
+    name: "{{ MARIADB_HAPROXY_USER }}"
+    host: "{{ item }}"
+    password: ""
+    priv: "*.*:USAGE,RELOAD"
+    state: present
+  with_items: MARIADB_HAPROXY_HOSTS

playbooks/roles/mariadb/tasks/main.yml

@@ -28,7 +28,13 @@
 - name: Add mariadb apt key
   apt_key:
     url: "{{ COMMON_UBUNTU_APT_KEYSERVER }}{{ MARIADB_APT_KEY_ID }}"
-
+  when: ansible_distribution_release != 'xenial'
+      
+- name: Add Xenial mariadb apt key
+  apt_key:
+    url: "{{ COMMON_UBUNTU_APT_KEYSERVER }}{{ MARIADB_APT_KEY_XENIAL_ID }}"
+  when: ansible_distribution_release == 'xenial'
+ 
 - name: add the mariadb repo to the sources list
   apt_repository:
     repo: "{{ MARIADB_REPO }}"
@@ -57,38 +63,69 @@
 - name: start everything
   service: name=mysql state=started
 
-- name: create all databases
+- name: create databases
   mysql_db:
-    db: "{{ item }}"
+    db: "{{ item.db }}"
     state: present
-    encoding: utf8
+    encoding: "{{ item.encoding }}"
+  when: item != None and item != '' and MARIADB_CREATE_DBS|bool
   with_items: "{{ MARIADB_DATABASES }}"
-  when: MARIADB_CREATE_DBS|bool
-
-- name: create all analytics dbs
-  mysql_db:
-    db: "{{ item }}"
-    state: present
-    encoding: utf8
-  with_items: "{{ MARIADB_ANALYTICS_DATABASES }}"
-  when: MARIADB_CREATE_DBS|bool and ANALYTICS_API_CONFIG is defined
 
-- name: create all users/privs
+- name: create database users
   mysql_user:
-    name: "{{ item.name }}"
+    name: "{{ item.user }}"
     password: "{{ item.pass }}"
-    priv: "{{ item.priv }}"
-    host: "{{ item.host }}"
+    priv: "{{ item.db }}.*:SELECT,INSERT,UPDATE,DELETE"
+    host: "{{ MARIADB_HOST_PRIV }}"
     append_privs: yes
+  when: item.db != None and item.db != ''
   with_items: "{{ MARIADB_USERS }}"
-  when: MARIADB_CREATE_DBS|bool
 
-- name: create all analytics users/privs
+- name: setup the migration db user
   mysql_user:
-    name: "{{ item.name }}"
-    password: "{{ item.pass }}"
-    priv: "{{ item.priv }}"
-    host: "{{ item.host }}"
+    name: "{{ COMMON_MYSQL_MIGRATE_USER }}"
+    password: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
+    priv: "{{ item.db }}.*:ALL"
+    host: "{{ MARIADB_HOST_PRIV }}"
     append_privs: yes
-  with_items: "{{ MARIADB_ANALYTICS_USERS }}"
-  when: MARIADB_CREATE_DBS|bool and ANALYTICS_API_CONFIG is defined
+  when: item != None and item != ''
+  with_items: "{{ MARIADB_DATABASES }}"
+
+- name: create api user for the analytics api
+  mysql_user:
+    name: "api001"
+    password: "{{ ANALYTICS_API_DATABASES.default.PASSWORD }}"
+    priv: '{{ ANALYTICS_API_DATABASES.default.NAME }}.*:SELECT,INSERT,UPDATE,DELETE/reports.*:SELECT'
+    host: "{{ MARIADB_HOST_PRIV }}"
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
+
+- name: create read-only reports user for the analytics-api
+  mysql_user:
+    name: reports001
+    password: "{{ ANALYTICS_API_DATABASES.reports.PASSWORD }}"
+    priv: '{{ ANALYTICS_API_DATABASES.reports.NAME }}.*:SELECT'
+    host: "{{ MARIADB_HOST_PRIV }}" 
+  when: ANALYTICS_API_SERVICE_CONFIG is defined
+
+- name: setup the edx-notes-api db user
+  mysql_user:
+    name: "{{ EDX_NOTES_API_MYSQL_DB_USER }}"
+    password: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}"
+    priv: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE"
+    host: "{{ MARIADB_HOST_PRIV }}"
+  when: EDX_NOTES_API_MYSQL_DB_USER is defined
+
+- name: setup the read-only db user
+  mysql_user:
+    name: "{{ COMMON_MYSQL_READ_ONLY_USER }}"
+    password: "{{ COMMON_MYSQL_READ_ONLY_PASS }}"
+    priv: "*.*:SELECT"
+    host: "{{ MARIADB_HOST_PRIV }}"
+
+- name: setup the admin db user
+  mysql_user:
+    name: "{{ COMMON_MYSQL_ADMIN_USER }}"
+    password: "{{ COMMON_MYSQL_ADMIN_PASS }}"
+    priv: "*.*:CREATE USER"
+    host: "{{ MARIADB_HOST_PRIV }}"
+