Merge pull request #2209 from edx/derf/mongo_3_0_new_role

mongo 3.0 role

playbooks/edx-east/mongo.yml

@@ -7,7 +7,6 @@
   serial: "{{ serial_count }}"
   roles:
     - mongo
-    - mongo_mms
     - role: datadog
       when: COMMON_ENABLE_DATADOG
     - role: splunkforwarder

playbooks/edx-east/mongo_3_0.yml

+- name: Deploy MongoDB
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 3
+  serial: "{{ serial_count }}"
+  roles:
+    - mongo_3_0
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/roles/mongo_3_0/defaults/main.yml

+mongo_logappend: true
+mongo_version: 3.0.4
+mongo_port: "27017"
+mongo_extra_conf: ''
+mongo_key_file: '/etc/mongodb_key'
+mongo_repl_set: rs0
+mongo_cluster_members: []
+pymongo_version: 2.8.1
+
+mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo"
+mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo"
+mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal"
+mongo_user: mongodb
+
+MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse"
+MONGODB_APT_KEY: "7F0CEB10"
+MONGODB_APT_KEYSERVER: "keyserver.ubuntu.com"
+
+mongodb_debian_pkgs:
+  - "mongodb-org={{ mongo_version }}"
+  - "mongodb-org-server={{ mongo_version }}" 
+  - "mongodb-org-shell={{ mongo_version }}"
+  - "mongodb-org-mongos={{ mongo_version }}" 
+  - "mongodb-org-tools={{ mongo_version }}"
+
+
+# Vars Meant to be overridden
+MONGO_ADMIN_USER: 'admin'
+MONGO_ADMIN_PASSWORD: 'password'
+MONGO_USERS:
+  - user: cs_comments_service
+    password: password
+    database: cs_comments_service
+    roles: readWrite
+  - user: edxapp
+    password: password
+    database: edxapp
+    roles: readWrite
+
+MONGO_CLUSTERED: !!null
+MONGO_BIND_IP: 127.0.0.1
+
+# Storage engine options in 3.0: "mmapv1" or "wiredTiger" 
+MONGO_STORAGE_ENGINE: "mmapv1"
+##
+
+mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
+mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
+
+# If the system is running out of an Amazon Web Services
+# cloudformation stack, this group name can used to pull out
+# the name of the stack the mongo server resides in.
+mongo_aws_stack_name: "tag_aws_cloudformation_stack-name_"
+
+# In environments that do not require durability (devstack / Jenkins)
+# you can disable the journal to reduce disk usage
+mongo_enable_journal: true
+
+# We can do regular backups of MongoDB to S3.
+MONGO_S3_BACKUP: false
+# backup cron time:
+MONGO_S3_BACKUP_HOUR: "*/12"
+MONGO_S3_BACKUP_DAY: "*"
+# override with a secondary node that will perform backups
+MONGO_S3_BACKUP_NODE: "undefined"
+# back up data into a specific S3 bucket
+MONGO_S3_BACKUP_BUCKET: "undefined"
+# temporary directory mongodump will use to store data
+MONGO_S3_BACKUP_TEMPDIR: "{{ mongo_data_dir }}"
+MONGO_S3_NOTIFY_EMAIL: "dummy@example.com"
+mongo_s3_logfile: "{{ COMMON_LOG_DIR }}/mongo/s3-mongo-backup.log"
+MONGO_S3_S3CMD_CONFIG: "{{ COMMON_DATA_DIR }}/mongo-s3-backup.s3cfg"
+MONGO_S3_BACKUP_AWS_ACCESS_KEY: !!null
+MONGO_S3_BACKUP_AWS_SECRET_KEY: !!null
+

playbooks/roles/mongo_3_0/handlers/main.yml

+---
+- name: restart mongo
+  service: name=mongod state=restarted
+

playbooks/roles/mongo_3_0/meta/main.yml

+---
+dependencies:
+  - common

playbooks/roles/mongo_3_0/tasks/main.yml

+---
+- name: check to see that MongoDB 2.4 is not installed
+  stat: path=/etc/init.d/mongodb
+  register: mongodb_needs_upgrade
+
+- name: verify 2.4 not installed
+  fail: msg="MongoDB 2.4 is currently installed and cannot be safely upgraded in a clustered configuration.  Please read http://docs.mongodb.org/manual/release-notes/2.6-upgrade/#upgrade-considerations and upgrade to 2.6."
+  when: mongodb_needs_upgrade.stat.exists and MONGO_CLUSTERED
+  
+- name: check to see if MongoDB is already installed
+  stat: path=/etc/init.d/mongod
+  register: mongodb_already_installed
+  when: MONGO_CLUSTERED
+
+- name: verify MongoDB not installed (clustered upgrades)
+  fail: msg="MongoDB is currently installed and cannot be safely upgraded in a clustered configuration.  FIXME"
+  when: mongodb_already_installed.stat.exists and MONGO_CLUSTERED
+
+- name: remove mongo 2.4 if present
+  apt: >
+    pkg=mongodb-10gen
+    state=absent purge=yes
+    force=yes
+  when: mongodb_needs_upgrade.stat.exists and not MONGO_CLUSTERED
+  
+- name: install python pymongo for mongo_user ansible module
+  pip: >
+    name=pymongo state=present
+    version={{ pymongo_version }} extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+
+- name: add the mongodb signing key
+  apt_key: >
+    id={{ MONGODB_APT_KEY }}
+    keyserver={{ MONGODB_APT_KEYSERVER }}
+    state=present
+
+- name: add the mongodb repo to the sources list
+  apt_repository: >
+    repo='{{ MONGODB_REPO }}'
+    state=present
+
+- name: install mongo server and recommends
+  apt: >
+  apt: 
+    pkg={{','.join(mongodb_debian_pkgs)}}
+    state=present install_recommends=yes
+    force=yes update_cache=yes
+
+- name: create mongo dirs
+  file: >
+    path="{{ item }}" state=directory
+    owner="{{ mongo_user  }}"
+    group="{{ mongo_user }}"
+  with_items:
+    - "{{ mongo_data_dir }}"
+    - "{{ mongo_dbpath }}"
+    - "{{ mongo_log_dir }}"
+    - "{{ mongo_journal_dir }}"
+
+- name: stop mongod service
+  service: name=mongod state=stopped
+
+- name: move mongodb to {{ mongo_data_dir }}
+  command: >
+    mv /var/lib/mongodb  {{ mongo_data_dir}}/.
+    creates={{ mongo_data_dir }}/mongodb
+
+- name: copy mongodb key file
+  copy: >
+    content="{{ MONGO_CLUSTER_KEY }}"
+    dest={{ mongo_key_file }}
+    mode=0600
+    owner=mongodb
+    group=mongodb
+  when: MONGO_CLUSTERED
+
+- name: copy configuration template
+  template: src=mongodb-standalone.conf.j2 dest=/etc/mongod.conf backup=yes
+  notify: restart mongo
+
+- name: start mongo service
+  service: name=mongod state=started
+
+- name: wait for mongo server to start
+  wait_for: port=27017 delay=2
+
+- name: drop super user script
+  template: src="create_root.js.j2" dest="/tmp/create_root.js"
+
+- name: create super user with js
+  shell: >
+    /usr/bin/mongo admin /tmp/create_root.js
+
+- name: delete super user script
+  file: path=/tmp/create_root.js state=absent
+
+- name: copy custered configuration template
+  template: src=mongodb-clustered.conf.j2 dest=/etc/mongod.conf backup=yes
+  when: MONGO_CLUSTERED
+
+- name: restart mongo service
+  service: name=mongod state=restarted
+  when: MONGO_CLUSTERED
+
+- name: wait for mongo server to start
+  wait_for: port=27017 delay=2
+  when: MONGO_CLUSTERED
+
+- name: Create the file to initialize the mongod replica set
+  template: src=repset_init.js.j2 dest=/tmp/repset_init.js
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+- name: Initialize the replication set
+  shell: >
+    /usr/bin/mongo /tmp/repset_init.js
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+- name: delete repset script
+  file: path=/tmp/repset_init.js state=absent
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+- name: Create the file to add hosts to the mongod replica set
+  template: src=repset_add_secondaries.js.j2 dest=/tmp/repset_add_secondaries.js
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+- name: Initialize the replication set
+  shell: >
+    /usr/bin/mongo /tmp/repset_add_secondaries.js
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+- name: delete repset script
+  file: path=/tmp/repset_add_secondaries.js state=absent
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+- name: create a mongodb user
+  mongodb_user: >
+    database={{ item.database }}
+    login_user={{ MONGO_ADMIN_USER }}
+    login_password={{ MONGO_ADMIN_PASSWORD }}
+    name={{ item.user }}
+    password={{ item.password }}
+    roles={{ item.roles }}
+    state=present
+  with_items: MONGO_USERS
+  when: not MONGO_CLUSTERED
+
+- name: create a mongodb user
+  mongodb_user: >
+    database={{ item.database }}
+    login_user={{ MONGO_ADMIN_USER }}
+    login_password={{ MONGO_ADMIN_PASSWORD }}
+    name={{ item.user }}
+    password={{ item.password }}
+    roles={{ item.roles }}
+    state=present
+    replica_set={{ mongo_repl_set }}
+  with_items: MONGO_USERS
+  when: MONGO_CLUSTERED and MONGO_PRIMARY == ansible_default_ipv4["address"] 
+
+

playbooks/roles/mongo_3_0/templates/backup-mongo-to-s3.j2

+{% set lb = '{' %}
+{% set rb = '}' %}
+#!/bin/bash
+#
+
+exec > >(tee "{{ mongo_s3_logfile }}")
+exec 2>&1
+
+shopt -s extglob
+
+usage() {
+
+  cat<<EO
+
+  A script that will run a mongodump of all databases, tar/gz them 
+  and upload to an s3 bucket, will send mail to 
+  {{ MONGO_S3_NOTIFY_EMAIL }} on failures.
+
+  Usage: $PROG
+            -v    add verbosity (set -x)
+            -n    echo what will be done
+            -h    this
+EO
+}
+
+while getopts "vhn" opt; do
+  case $opt in
+    v)
+      set -x
+      shift
+      ;;
+    h)
+      usage
+      exit 0
+      ;;
+    n)
+      noop="echo Would have run: "
+      shift
+      ;;
+  esac
+done
+
+if [[ "{{ MONGO_S3_BACKUP }}" != "true" ]]; then
+  # only run if explicitly enabled
+  exit
+fi
+
+MYNODENAME=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep \"me\" | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//');
+if [[ "$MYNODENAME" != "{{ MONGO_S3_BACKUP_NODE }}" ]]; then
+  # only run on specified node
+  exit
+fi
+  
+
+ISSECONDARY=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep secondary | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//')
+if [[ "$ISSECONDARY" != "true" ]]; then 
+  # backups should be run on secondary server
+  exit; 
+fi
+
+MONGOOUTDIR=$(mktemp -d -p {{ MONGO_S3_BACKUP_TEMPDIR }})
+DATESTAMP=$(date +'%Y-%m-%d-%H%M')
+$noop mongodump --host {{ EDXAPP_MONGO_HOSTS[0] }} -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" -o $MONGOOUTDIR
+cd $MONGOOUTDIR
+$noop tar zcf {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz .
+cd {{ MONGO_S3_BACKUP_TEMPDIR }}
+$noop s3cmd -c {{ MONGO_S3_S3CMD_CONFIG }} sync {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz "s3://{{ MONGO_S3_BACKUP_BUCKET }}/mongo/"
+rm -rf $MONGOOUTDIR {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz

playbooks/roles/mongo_3_0/templates/create_root.js.j2

+// Add super user
+conn = new Mongo();
+db = conn.getDB("admin");
+db.createUser(
+    {
+        "user": "{{ MONGO_ADMIN_USER }}",
+        "pwd": "{{ MONGO_ADMIN_PASSWORD }}",
+        "roles": ["root"]
+    }
+);
+
+

playbooks/roles/mongo_3_0/templates/mongo-s3-backup-s3cfg.j2

+[default]
+access_key = {{ MONGO_S3_BACKUP_AWS_ACCESS_KEY }}
+secret_key = {{ MONGO_S3_BACKUP_AWS_SECRET_KEY }}
+bucket_location = US

playbooks/roles/mongo_3_0/templates/mongodb-clustered.conf.j2

+# Do not edit this file directly, it was generated by ansible
+# mongodb.conf
+
+
+storage:
+  # Where to store the data.
+  dbPath: {{ mongo_dbpath }}
+  # Storage Engine
+  engine: {{ MONGO_STORAGE_ENGINE }}
+  # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
+  journal:
+{% if mongo_enable_journal %}
+    enabled: true
+{% else %}
+    enabled: false
+{% endif %}
+
+systemLog:
+  #where to log
+  destination: file
+  path: "{{ mongo_logpath }}"
+{% if mongo_logappend  %}
+  logAppend: true
+{% else %}
+  logAppend: false
+{% endif %}
+
+{% if MONGO_CLUSTERED %}
+replication:
+  replSetName: {{ mongo_repl_set }}
+
+security:
+  keyFile: {{ mongo_key_file }}
+
+{% endif %}
+net:
+{% if MONGO_CLUSTERED is not defined %}
+  {## Bind to all ips(default) if in clustered mode, 
+  # otherwise only to the specified local ip.
+:  #}   
+  bindIp: {{ MONGO_BIND_IP }}
+{% endif %}
+  port: {{ mongo_port }}
+
+
+{{ mongo_extra_conf }}

playbooks/roles/mongo_3_0/templates/mongodb-standalone.conf.j2

+# Do not edit this file directly, it was generated by ansible
+# mongodb.conf
+
+
+storage:
+  # Where to store the data.
+  dbPath: {{ mongo_dbpath }}
+  # Storage Engine
+  engine: {{ MONGO_STORAGE_ENGINE }}
+  # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
+  journal:
+{% if mongo_enable_journal %}
+    enabled: true
+{% else %}
+    enabled: false
+{% endif %}
+
+systemLog:
+  #where to log
+  destination: file
+  path: "{{ mongo_logpath }}"
+{% if mongo_logappend  %}
+  logAppend: true
+{% else %}
+  logAppend: false
+{% endif %}
+
+net:
+{% if MONGO_CLUSTERED is not defined %}
+  {## Bind to all ips(default) if in clustered mode, 
+  # otherwise only to the specified local ip.
+  #}   
+  bindIp: {{ MONGO_BIND_IP }}
+{% endif %}
+  port: {{ mongo_port }}
+
+
+{{ mongo_extra_conf }}

playbooks/roles/mongo_3_0/templates/repset_add_secondaries.js.j2

+conn = new Mongo();
+db = conn.getDB("admin");
+db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}');
+
+{# Generate a list of hosts if no cluster members are give. Otherwise use the
+   hosts provided in the variable.
+#}
+{%- if mongo_cluster_members|length == 0 -%}
+  {%- set hosts = [] -%}
+  {%- set all_mongo_hosts = [] -%}
+  {%- do all_mongo_hosts.extend(groups.tag_role_mongo) -%}
+  {%- do all_mongo_hosts.extend(groups.tag_group_mongo) -%}
+  {%- for name in  group_names -%}
+    {%- if name.startswith(mongo_aws_stack_name) -%}
+      {%- for host in all_mongo_hosts -%}
+        {%- if host in groups[name] -%}
+          {% do hosts.append("ip-" + host.replace('.','-') + ":" + mongo_port) %}
+        {%- endif -%}
+      {%- endfor -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- else -%}
+  {%- set hosts = mongo_cluster_members -%}
+{%- endif -%}
+
+// Check that the cluster is ok
+if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
+
+// Check that the cluster has the right number of members
+// and add them if we are the master
+if(rs.isMaster().ismaster) {
+    if(rs.status().members.length!={{ hosts|length }}) {
+        {% for host in mongo_cluster_members %}
+          {%- if host != ansible_default_ipv4["address"] -%}
+            rs.add({_id: {{ loop.index }}, host: '{{ host }}'});
+          {%- endif -%}
+        {% endfor %}
+        sleep(30000);
+        // Check status and member account, throw exception if not
+        if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
+        if(rs.status().members.length!={{ hosts|length }}) {
+            throw 'Could not add all members to cluster'
+        }
+    }
+}

playbooks/roles/mongo_3_0/templates/repset_init.js.j2

+conn = new Mongo();
+db = conn.getDB("admin");
+db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}');
+
+{%- if MONGO_PRIMARY == ansible_default_ipv4["address"] -%}
+
+{# Generate a list of hosts if no cluster members are give. Otherwise use the
+   hosts provided in the variable.
+#}
+{%- if mongo_cluster_members|length == 0 -%}
+  {%- set hosts = [] -%}
+  {%- set all_mongo_hosts = [] -%}
+  {%- do all_mongo_hosts.extend(groups.tag_role_mongo) -%}
+  {%- do all_mongo_hosts.extend(groups.tag_group_mongo) -%}
+  {%- for name in  group_names -%}
+    {%- if name.startswith(mongo_aws_stack_name) -%}
+      {%- for host in all_mongo_hosts -%}
+        {%- if host in groups[name] -%}
+          {% do hosts.append("ip-" + host.replace('.','-') + ":" + mongo_port) %}
+        {%- endif -%}
+      {%- endfor -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- else -%}
+  {%- set hosts = mongo_cluster_members -%}
+{%- endif -%}
+
+
+config = {_id: '{{ mongo_repl_set }}', members: [{% for host in mongo_cluster_members %}
+{%- if host == ansible_default_ipv4["address"] -%}
+{_id: {{ loop.index }}, host: '{{ host }}'}
+{%- endif -%}
+{% endfor %}
+]};
+rs.initiate(config)
+
+sleep(30000)
+rs.slaveOk()
+printjson(rs.status())
+
+// Check that the cluster is ok
+if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
+{%- endif -%}