OPS-967: Make mongo_3_0 role idempotent

Remove check for mongo 2.4 since this is the mongo_3_0 role
This was probably carried over from the mongo 2 role. I see no reason
why mongo 2.x and mongo 3 couldn't coexist on the same machine.

Remove old hugepages init script check
This was probably added while we were still iterating on our mongo 3
deployment, but it should no longer be necessary.

Clean up ansible syntax

Don't move the old mongo data dirs
This actually skips for edX because we provision machines that already
have {{mongo_data_dir}} mounted on an external disk.  However, for
non-edX use, this could fail if you turn on WiredTiger since it will
move mmapv1 files into the mongo_data_dir and then mongo will fail to
start because it has been told to use WiredTiger.

Don't make this a serial play
We usually run this on 3 machines, so serial: 3 was equivalent to
ansible's default of "run everything in parallel" but this causes
problems if you ever run it on 4 like we do for prod envs.
In addition, this prevents run_once from working properly, and there are
a number of things we only one to do on one machine (like creating a
superuser).

playbooks/edx-east/mongo_3_0.yml

@@ -2,9 +2,6 @@
   hosts: all
   sudo: True
   gather_facts: True
-  vars:
-    serial_count: 3
-  serial: "{{ serial_count }}"
   roles:
     - aws
     - mongo_3_0

playbooks/roles/mongo_3_0/defaults/main.yml

@@ -7,8 +7,6 @@ mongo_version: 3.0.8
 mongo_port: "27017"
 mongo_extra_conf: ''
 mongo_key_file: '/etc/mongodb_key'
-mongo_repl_set: "{{ MONGO_REPL_SET }}"
-mongo_cluster_members: []
 pymongo_version: 2.8.1
 
 mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo"
@@ -45,9 +43,13 @@ MONGO_CLUSTERED: !!null
 MONGO_BIND_IP: 127.0.0.1
 MONGO_REPL_SET: "rs0"
 
+# Cluster member configuration
+# Fed directly into mongodb_replica_set module
+MONGO_RS_CONFIG:
+  members: []
+
 # Storage engine options in 3.0: "mmapv1" or "wiredTiger" 
 MONGO_STORAGE_ENGINE: "mmapv1"
-##
 
 # WiredTiger takes a number of optional configuration settings
 # which can be defined as a yaml structure in your secure configuration.
@@ -56,31 +58,8 @@ MONGO_STORAGE_ENGINE_OPTIONS: !!null
 mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
 mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
 
-# If the system is running out of an Amazon Web Services
-# cloudformation stack, this group name can used to pull out
-# the name of the stack the mongo server resides in.
-mongo_aws_stack_name: "tag_aws_cloudformation_stack-name_"
-
 # In environments that do not require durability (devstack / Jenkins)
 # you can disable the journal to reduce disk usage
 mongo_enable_journal: true
 
-# We can do regular backups of MongoDB to S3.
-MONGO_S3_BACKUP: false
-# backup cron time:
-MONGO_S3_BACKUP_HOUR: "*/12"
-MONGO_S3_BACKUP_DAY: "*"
-# override with a secondary node that will perform backups
-MONGO_S3_BACKUP_NODE: "undefined"
-# back up data into a specific S3 bucket
-MONGO_S3_BACKUP_BUCKET: "undefined"
-# temporary directory mongodump will use to store data
-MONGO_S3_BACKUP_TEMPDIR: "{{ mongo_data_dir }}"
-MONGO_S3_NOTIFY_EMAIL: "dummy@example.com"
-mongo_s3_logfile: "{{ COMMON_LOG_DIR }}/mongo/s3-mongo-backup.log"
-MONGO_S3_S3CMD_CONFIG: "{{ COMMON_DATA_DIR }}/mongo-s3-backup.s3cfg"
-MONGO_S3_BACKUP_AWS_ACCESS_KEY: !!null
-MONGO_S3_BACKUP_AWS_SECRET_KEY: !!null
-
 MONGO_LOG_SERVERSTATUS: true
-MONGO_HEARTBEAT_TIMEOUT_SECS: 10

playbooks/roles/mongo_3_0/templates/backup-mongo-to-s3.j2

-{% set lb = '{' %}
-{% set rb = '}' %}
-#!/bin/bash
-#
-
-exec > >(tee "{{ mongo_s3_logfile }}")
-exec 2>&1
-
-shopt -s extglob
-
-usage() {
-
-  cat<<EO
-
-  A script that will run a mongodump of all databases, tar/gz them 
-  and upload to an s3 bucket, will send mail to 
-  {{ MONGO_S3_NOTIFY_EMAIL }} on failures.
-
-  Usage: $PROG
-            -v    add verbosity (set -x)
-            -n    echo what will be done
-            -h    this
-EO
-}
-
-while getopts "vhn" opt; do
-  case $opt in
-    v)
-      set -x
-      shift
-      ;;
-    h)
-      usage
-      exit 0
-      ;;
-    n)
-      noop="echo Would have run: "
-      shift
-      ;;
-  esac
-done
-
-if [[ "{{ MONGO_S3_BACKUP }}" != "true" ]]; then
-  # only run if explicitly enabled
-  exit
-fi
-
-MYNODENAME=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep \"me\" | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//');
-if [[ "$MYNODENAME" != "{{ MONGO_S3_BACKUP_NODE }}" ]]; then
-  # only run on specified node
-  exit
-fi
-  
-
-ISSECONDARY=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep secondary | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//')
-if [[ "$ISSECONDARY" != "true" ]]; then 
-  # backups should be run on secondary server
-  exit; 
-fi
-
-MONGOOUTDIR=$(mktemp -d -p {{ MONGO_S3_BACKUP_TEMPDIR }})
-DATESTAMP=$(date +'%Y-%m-%d-%H%M')
-$noop mongodump --host {{ EDXAPP_MONGO_HOSTS[0] }} -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" -o $MONGOOUTDIR
-cd $MONGOOUTDIR
-$noop tar zcf {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz .
-cd {{ MONGO_S3_BACKUP_TEMPDIR }}
-$noop s3cmd -c {{ MONGO_S3_S3CMD_CONFIG }} sync {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz "s3://{{ MONGO_S3_BACKUP_BUCKET }}/mongo/"
-rm -rf $MONGOOUTDIR {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz

playbooks/roles/mongo_3_0/templates/mongo-s3-backup-s3cfg.j2

-[default]
-access_key = {{ MONGO_S3_BACKUP_AWS_ACCESS_KEY }}
-secret_key = {{ MONGO_S3_BACKUP_AWS_SECRET_KEY }}
-bucket_location = US

playbooks/roles/mongo_3_0/templates/mongodb-clustered.conf.j2 → playbooks/roles/mongo_3_0/templates/mongod.conf.j2

@@ -31,7 +31,7 @@ systemLog:
 
 {% if MONGO_CLUSTERED %}
 replication:
-  replSetName: {{ mongo_repl_set }}
+  replSetName: {{ MONGO_REPL_SET }}
 
 security:
   keyFile: {{ mongo_key_file }}
@@ -39,9 +39,8 @@ security:
 {% endif %}
 net:
 {% if MONGO_CLUSTERED is not defined %}
-  {## Bind to all ips(default) if in clustered mode, 
-  # otherwise only to the specified local ip.
-:  #}   
+  {# Bind to all ips(default) if in clustered mode, 
+     otherwise only to the specified local ip. #}   
   bindIp: {{ MONGO_BIND_IP }}
 {% endif %}
   port: {{ mongo_port }}

playbooks/roles/mongo_3_0/templates/mongodb-standalone.conf.j2

-# Do not edit this file directly, it was generated by ansible
-# mongodb.conf
-
-
-storage:
-  # Where to store the data.
-  dbPath: {{ mongo_dbpath }}
-  # Storage Engine
-  engine: {{ MONGO_STORAGE_ENGINE }}
-  # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
-  journal:
-{% if mongo_enable_journal %}
-    enabled: true
-{% else %}
-    enabled: false
-{% endif %}
-{% if MONGO_STORAGE_ENGINE_OPTIONS %}
-  {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }}
-{% endif %}
-
-systemLog:
-  #where to log
-  destination: file
-  path: "{{ mongo_logpath }}"
-{% if mongo_logappend  %}
-  logAppend: true
-{% else %}
-  logAppend: false
-{% endif %}
-  logRotate: {{ mongo_logrotate }}
-
-net:
-{% if MONGO_CLUSTERED is not defined %}
-  {## Bind to all ips(default) if in clustered mode, 
-  # otherwise only to the specified local ip.
-  #}   
-  bindIp: {{ MONGO_BIND_IP }}
-{% endif %}
-  port: {{ mongo_port }}
-
-
-{{ mongo_extra_conf }}

playbooks/roles/mongo_3_0/templates/repset_add_secondaries.js.j2

-conn = new Mongo();
-db = conn.getDB("admin");
-db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}');
-
-{# Generate a list of hosts if no cluster members are give. Otherwise use the
-   hosts provided in the variable.
-#}
-{%- if mongo_cluster_members|length == 0 -%}
-  {%- set hosts = [] -%}
-  {%- set all_mongo_hosts = [] -%}
-  {%- do all_mongo_hosts.extend(groups.tag_role_mongo) -%}
-  {%- do all_mongo_hosts.extend(groups.tag_group_mongo) -%}
-  {%- for name in  group_names -%}
-    {%- if name.startswith(mongo_aws_stack_name) -%}
-      {%- for host in all_mongo_hosts -%}
-        {%- if host in groups[name] -%}
-          {% do hosts.append("ip-" + host.replace('.','-') + ":" + mongo_port) %}
-        {%- endif -%}
-      {%- endfor -%}
-    {%- endif -%}
-  {%- endfor -%}
-{%- else -%}
-  {%- set hosts = mongo_cluster_members|map(attribute="name") -%}
-{%- endif -%}
-
-// Check that the cluster is ok
-if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
-
-// Check that the cluster has the right number of members
-// and add them if we are the master
-if(rs.isMaster().ismaster) {
-    if(rs.status().members.length!={{ hosts|length }}) {
-        {% for host in hosts %}
-          {%- if host != ansible_default_ipv4["address"] -%}
-            rs.add({_id: {{ loop.index }}, host: '{{ host }}'});
-          {%- endif -%}
-        {% endfor %}
-        sleep(30000);
-        // Check status and member account, throw exception if not
-        if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
-        if(rs.status().members.length!={{ hosts|length }}) {
-            throw 'Could not add all members to cluster'
-        }
-    }
-}

playbooks/roles/mongo_3_0/templates/repset_init.js.j2

-conn = new Mongo();
-db = conn.getDB("admin");
-db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}');
-
-{%- if MONGO_PRIMARY == ansible_default_ipv4["address"] -%}
-
-{# Generate a list of hosts if no cluster members are give. Otherwise use the
-   hosts provided in the variable.
-#}
-{%- if mongo_cluster_members|length == 0 -%}
-  {%- set hosts = [] -%}
-  {%- set all_mongo_hosts = [] -%}
-  {%- do all_mongo_hosts.extend(groups.tag_role_mongo) -%}
-  {%- do all_mongo_hosts.extend(groups.tag_group_mongo) -%}
-  {%- for name in  group_names -%}
-    {%- if name.startswith(mongo_aws_stack_name) -%}
-      {%- for host in all_mongo_hosts -%}
-        {%- if host in groups[name] -%}
-          {% do hosts.append("ip-" + host.replace('.','-') + ":" + mongo_port) %}
-        {%- endif -%}
-      {%- endfor -%}
-    {%- endif -%}
-  {%- endfor -%}
-{%- else -%}
-  {%- set hosts = mongo_cluster_members|map(attribute="name") -%}
-{%- endif -%}
-
-
-config = {_id: '{{ mongo_repl_set }}', members: [{% for host in hosts %}
-{%- if host == ansible_default_ipv4["address"] -%}
-{_id: {{ loop.index }}, host: '{{ host }}'}
-{%- endif -%}
-{% endfor %}
-],
-settings: { heartbeatTimeoutSecs: {{ MONGO_HEARTBEAT_TIMEOUT_SECS }} }};
-rs.initiate(config)
-
-sleep(30000)
-rs.slaveOk()
-printjson(rs.status())
-
-// Check that the cluster is ok
-if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
-{%- endif -%}