Merge pull request #2587 from edx/max/idempotent-mongo_3_0

Max/idempotent mongo 3 0

playbooks/edx-east/mongo_3_0.yml

+# Manages a mongo cluster.
+# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG
+# as used by mongo_replica_set in the mongo_3_0 role.
+#
+# If you are initializing a cluster, your command might look like:
+# ansible-playbook mongo_3_0.yml -i 10.1.1.1,10.2.2.2,10.3.3.3 -e@/path/to/edx.yml -e@/path/to/ed.yml
+# If you just want to deploy an updated replica set config, you can run
+# ansible-playbook mongo_3_0.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set
+#
+# ADDING A NEW CLUSTER MEMBER
+# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory
+# ansible-playbook mongo_3_0.yml -i 10.1.1.1,10.2.2.2,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml
 - name: Deploy MongoDB
   hosts: all
   sudo: True
   gather_facts: True
-  vars:
-    serial_count: 3
-  serial: "{{ serial_count }}"
   roles:
     - aws
     - mongo_3_0

playbooks/library/mongodb_rs_status

@@ -6,8 +6,13 @@ module: mongodb_rs_status
 short_description: Get the status of a replica set of a mongo cluster.
 description:
   - Get the status of the replica set of a mongo cluster. Provide the same info as rs.status() or replSetGetStatus.
+    Returns a status dictionary key containing the replica set JSON document from Mongo, or no status key if there
+    was no status found.  This usually indicates that either Mongo was configured to run without replica sets or
+    that the replica set has not been initiated yet.
 version_added: "1.9"
-author: Feanil Patel
+author:
+  - Feanil Patel
+  - Kevin Falcone
 options:
   host:
     description:
@@ -35,17 +40,26 @@ options:
 
 EXAMPLES = '''
 - name: Get status for the stage cluster
-  mongo_status:
+  mongodb_rs_status:
     host: localhost:27017
     username: root
     password: password
   register: mongo_status
+
+Note that you're testing for the presence of the status member of the dictionary not the contents of it
+
+- debug: msg="I don't have a replica set available"
+  when: mongo_status.status is not defined
+
+- debug: var=mongo_status.status
+
 '''
 # Magic import
 from ansible.module_utils.basic import *
 
 try:
     from pymongo import MongoClient
+    from pymongo.errors import OperationFailure
     from bson import json_util
 except ImportError:
     pymongo_found = False
@@ -89,7 +103,22 @@ def main():
         mongo_uri += '/{}'.format(quote_plus(auth_database))
 
     client = MongoClient(mongo_uri)
-    status = client.admin.command("replSetGetStatus")
+
+    # This checks to see if you have a replSetName configured
+    # This generally means that /etc/mongod.conf has been changed
+    # from the default to use a replica set and mongo has been
+    # restarted to use it.
+
+    try:
+        repl_set = client.admin.command('getCmdLineOpts')['parsed']['replication']['replSetName']
+    except (OperationFailure, KeyError):
+        module.exit_json(changed=False)
+
+    # If mongo was started with a repl_set, it is safe to run replSetGetStatus
+    if repl_set:
+        status = client.admin.command("replSetGetStatus")
+    else:
+        module.exit_json(changed=False)
 
     # This converts the bson into a python dictionary that ansible's standard
     # jsonify function can process and output without throwing errors on bson

playbooks/roles/mongo_3_0/defaults/main.yml

@@ -7,8 +7,6 @@ mongo_version: 3.0.8
 mongo_port: "27017"
 mongo_extra_conf: ''
 mongo_key_file: '/etc/mongodb_key'
-mongo_repl_set: "{{ MONGO_REPL_SET }}"
-mongo_cluster_members: []
 pymongo_version: 2.8.1
 
 mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo"
@@ -45,9 +43,13 @@ MONGO_CLUSTERED: !!null
 MONGO_BIND_IP: 127.0.0.1
 MONGO_REPL_SET: "rs0"
 
+# Cluster member configuration
+# Fed directly into mongodb_replica_set module
+MONGO_RS_CONFIG:
+  members: []
+
 # Storage engine options in 3.0: "mmapv1" or "wiredTiger" 
 MONGO_STORAGE_ENGINE: "mmapv1"
-##
 
 # WiredTiger takes a number of optional configuration settings
 # which can be defined as a yaml structure in your secure configuration.
@@ -56,31 +58,8 @@ MONGO_STORAGE_ENGINE_OPTIONS: !!null
 mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
 mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
 
-# If the system is running out of an Amazon Web Services
-# cloudformation stack, this group name can used to pull out
-# the name of the stack the mongo server resides in.
-mongo_aws_stack_name: "tag_aws_cloudformation_stack-name_"
-
 # In environments that do not require durability (devstack / Jenkins)
 # you can disable the journal to reduce disk usage
 mongo_enable_journal: true
 
-# We can do regular backups of MongoDB to S3.
-MONGO_S3_BACKUP: false
-# backup cron time:
-MONGO_S3_BACKUP_HOUR: "*/12"
-MONGO_S3_BACKUP_DAY: "*"
-# override with a secondary node that will perform backups
-MONGO_S3_BACKUP_NODE: "undefined"
-# back up data into a specific S3 bucket
-MONGO_S3_BACKUP_BUCKET: "undefined"
-# temporary directory mongodump will use to store data
-MONGO_S3_BACKUP_TEMPDIR: "{{ mongo_data_dir }}"
-MONGO_S3_NOTIFY_EMAIL: "dummy@example.com"
-mongo_s3_logfile: "{{ COMMON_LOG_DIR }}/mongo/s3-mongo-backup.log"
-MONGO_S3_S3CMD_CONFIG: "{{ COMMON_DATA_DIR }}/mongo-s3-backup.s3cfg"
-MONGO_S3_BACKUP_AWS_ACCESS_KEY: !!null
-MONGO_S3_BACKUP_AWS_SECRET_KEY: !!null
-
 MONGO_LOG_SERVERSTATUS: true
-MONGO_HEARTBEAT_TIMEOUT_SECS: 10

playbooks/roles/mongo_3_0/templates/backup-mongo-to-s3.j2

-{% set lb = '{' %}
-{% set rb = '}' %}
-#!/bin/bash
-#
-
-exec > >(tee "{{ mongo_s3_logfile }}")
-exec 2>&1
-
-shopt -s extglob
-
-usage() {
-
-  cat<<EO
-
-  A script that will run a mongodump of all databases, tar/gz them 
-  and upload to an s3 bucket, will send mail to 
-  {{ MONGO_S3_NOTIFY_EMAIL }} on failures.
-
-  Usage: $PROG
-            -v    add verbosity (set -x)
-            -n    echo what will be done
-            -h    this
-EO
-}
-
-while getopts "vhn" opt; do
-  case $opt in
-    v)
-      set -x
-      shift
-      ;;
-    h)
-      usage
-      exit 0
-      ;;
-    n)
-      noop="echo Would have run: "
-      shift
-      ;;
-  esac
-done
-
-if [[ "{{ MONGO_S3_BACKUP }}" != "true" ]]; then
-  # only run if explicitly enabled
-  exit
-fi
-
-MYNODENAME=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep \"me\" | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//');
-if [[ "$MYNODENAME" != "{{ MONGO_S3_BACKUP_NODE }}" ]]; then
-  # only run on specified node
-  exit
-fi
-  
-
-ISSECONDARY=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep secondary | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//')
-if [[ "$ISSECONDARY" != "true" ]]; then 
-  # backups should be run on secondary server
-  exit; 
-fi
-
-MONGOOUTDIR=$(mktemp -d -p {{ MONGO_S3_BACKUP_TEMPDIR }})
-DATESTAMP=$(date +'%Y-%m-%d-%H%M')
-$noop mongodump --host {{ EDXAPP_MONGO_HOSTS[0] }} -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" -o $MONGOOUTDIR
-cd $MONGOOUTDIR
-$noop tar zcf {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz .
-cd {{ MONGO_S3_BACKUP_TEMPDIR }}
-$noop s3cmd -c {{ MONGO_S3_S3CMD_CONFIG }} sync {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz "s3://{{ MONGO_S3_BACKUP_BUCKET }}/mongo/"
-rm -rf $MONGOOUTDIR {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz

playbooks/roles/mongo_3_0/templates/mongo-s3-backup-s3cfg.j2

-[default]
-access_key = {{ MONGO_S3_BACKUP_AWS_ACCESS_KEY }}
-secret_key = {{ MONGO_S3_BACKUP_AWS_SECRET_KEY }}
-bucket_location = US

playbooks/roles/mongo_3_0/templates/mongodb-clustered.conf.j2 → playbooks/roles/mongo_3_0/templates/mongod.conf.j2

@@ -29,9 +29,9 @@ systemLog:
 {% endif %}
   logRotate: {{ mongo_logrotate }}
 
-{% if MONGO_CLUSTERED %}
+{% if MONGO_CLUSTERED and not skip_replica_set %}
 replication:
-  replSetName: {{ mongo_repl_set }}
+  replSetName: {{ MONGO_REPL_SET }}
 
 security:
   keyFile: {{ mongo_key_file }}
@@ -39,9 +39,8 @@ security:
 {% endif %}
 net:
 {% if MONGO_CLUSTERED is not defined %}
-  {## Bind to all ips(default) if in clustered mode, 
-  # otherwise only to the specified local ip.
-:  #}   
+  {# Bind to all ips(default) if in clustered mode, 
+     otherwise only to the specified local ip. #}   
   bindIp: {{ MONGO_BIND_IP }}
 {% endif %}
   port: {{ mongo_port }}

playbooks/roles/mongo_3_0/templates/mongodb-standalone.conf.j2

-# Do not edit this file directly, it was generated by ansible
-# mongodb.conf
-
-
-storage:
-  # Where to store the data.
-  dbPath: {{ mongo_dbpath }}
-  # Storage Engine
-  engine: {{ MONGO_STORAGE_ENGINE }}
-  # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
-  journal:
-{% if mongo_enable_journal %}
-    enabled: true
-{% else %}
-    enabled: false
-{% endif %}
-{% if MONGO_STORAGE_ENGINE_OPTIONS %}
-  {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }}
-{% endif %}
-
-systemLog:
-  #where to log
-  destination: file
-  path: "{{ mongo_logpath }}"
-{% if mongo_logappend  %}
-  logAppend: true
-{% else %}
-  logAppend: false
-{% endif %}
-  logRotate: {{ mongo_logrotate }}
-
-net:
-{% if MONGO_CLUSTERED is not defined %}
-  {## Bind to all ips(default) if in clustered mode, 
-  # otherwise only to the specified local ip.
-  #}   
-  bindIp: {{ MONGO_BIND_IP }}
-{% endif %}
-  port: {{ mongo_port }}
-
-
-{{ mongo_extra_conf }}

playbooks/roles/mongo_3_0/templates/repset_add_secondaries.js.j2

-conn = new Mongo();
-db = conn.getDB("admin");
-db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}');
-
-{# Generate a list of hosts if no cluster members are give. Otherwise use the
-   hosts provided in the variable.
-#}
-{%- if mongo_cluster_members|length == 0 -%}
-  {%- set hosts = [] -%}
-  {%- set all_mongo_hosts = [] -%}
-  {%- do all_mongo_hosts.extend(groups.tag_role_mongo) -%}
-  {%- do all_mongo_hosts.extend(groups.tag_group_mongo) -%}
-  {%- for name in  group_names -%}
-    {%- if name.startswith(mongo_aws_stack_name) -%}
-      {%- for host in all_mongo_hosts -%}
-        {%- if host in groups[name] -%}
-          {% do hosts.append("ip-" + host.replace('.','-') + ":" + mongo_port) %}
-        {%- endif -%}
-      {%- endfor -%}
-    {%- endif -%}
-  {%- endfor -%}
-{%- else -%}
-  {%- set hosts = mongo_cluster_members|map(attribute="name") -%}
-{%- endif -%}
-
-// Check that the cluster is ok
-if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
-
-// Check that the cluster has the right number of members
-// and add them if we are the master
-if(rs.isMaster().ismaster) {
-    if(rs.status().members.length!={{ hosts|length }}) {
-        {% for host in hosts %}
-          {%- if host != ansible_default_ipv4["address"] -%}
-            rs.add({_id: {{ loop.index }}, host: '{{ host }}'});
-          {%- endif -%}
-        {% endfor %}
-        sleep(30000);
-        // Check status and member account, throw exception if not
-        if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
-        if(rs.status().members.length!={{ hosts|length }}) {
-            throw 'Could not add all members to cluster'
-        }
-    }
-}

playbooks/roles/mongo_3_0/templates/repset_init.js.j2

-conn = new Mongo();
-db = conn.getDB("admin");
-db.auth( '{{ MONGO_ADMIN_USER }}', '{{ MONGO_ADMIN_PASSWORD }}');
-
-{%- if MONGO_PRIMARY == ansible_default_ipv4["address"] -%}
-
-{# Generate a list of hosts if no cluster members are give. Otherwise use the
-   hosts provided in the variable.
-#}
-{%- if mongo_cluster_members|length == 0 -%}
-  {%- set hosts = [] -%}
-  {%- set all_mongo_hosts = [] -%}
-  {%- do all_mongo_hosts.extend(groups.tag_role_mongo) -%}
-  {%- do all_mongo_hosts.extend(groups.tag_group_mongo) -%}
-  {%- for name in  group_names -%}
-    {%- if name.startswith(mongo_aws_stack_name) -%}
-      {%- for host in all_mongo_hosts -%}
-        {%- if host in groups[name] -%}
-          {% do hosts.append("ip-" + host.replace('.','-') + ":" + mongo_port) %}
-        {%- endif -%}
-      {%- endfor -%}
-    {%- endif -%}
-  {%- endfor -%}
-{%- else -%}
-  {%- set hosts = mongo_cluster_members|map(attribute="name") -%}
-{%- endif -%}
-
-
-config = {_id: '{{ mongo_repl_set }}', members: [{% for host in hosts %}
-{%- if host == ansible_default_ipv4["address"] -%}
-{_id: {{ loop.index }}, host: '{{ host }}'}
-{%- endif -%}
-{% endfor %}
-],
-settings: { heartbeatTimeoutSecs: {{ MONGO_HEARTBEAT_TIMEOUT_SECS }} }};
-rs.initiate(config)
-
-sleep(30000)
-rs.slaveOk()
-printjson(rs.status())
-
-// Check that the cluster is ok
-if(!rs.status().ok) { throw 'Mongo Cluster Not Ok';}
-{%- endif -%}

tests/test_mongodb_replica_set.py

+# Tests for mongodb_replica_set ansible module
+#
+# How to run these tests:
+# 1. move this file to playbooks/library
+# 2. rename mongodb_replica_set to mongodb_replica_set.py
+# 3. python test_mongodb_replica_set.py
+
+import mongodb_replica_set as mrs
+import unittest, mock
+from urllib import quote_plus
+from copy import deepcopy
+
+class TestNoPatchingMongodbReplicaSet(unittest.TestCase):
+  def test_host_port_transformation(self):
+    unfixed = {
+      'members': [
+        {'host': 'foo.bar'},
+        {'host': 'bar.baz', 'port': 1234},
+        {'host': 'baz.bing:54321'}
+    ]}
+    fixed = {
+      'members': [
+        {'host': 'foo.bar:27017'},
+        {'host': 'bar.baz:1234'},
+        {'host': 'baz.bing:54321'}
+    ]}
+
+    mrs.fix_host_port(unfixed)
+    self.assertEqual(fixed, unfixed)
+
+    fixed_2 = deepcopy(fixed)
+    mrs.fix_host_port(fixed_2)
+    self.assertEqual(fixed, fixed_2)
+
+  def test_member_id_managed(self):
+    new = [
+        {'host': 'foo.bar', '_id': 1},
+        {'host': 'bar.baz'},
+        {'host': 'baz.bing'}
+    ]
+    old = [
+      {'host': 'baz.bing', '_id': 0}
+    ]
+
+    fixed = deepcopy(new)
+    mrs.set_member_ids(fixed, old)
+    
+    #test that each id is unique
+    unique_ids = {m['_id'] for m in fixed}
+    self.assertEqual(len(unique_ids), len(new))
+    
+    #test that it "prefers" the "matching" one in old_members
+    self.assertEqual(fixed[0]['_id'], new[0]['_id'])
+    self.assertEqual(fixed[2]['_id'], old[0]['_id'])
+    self.assertIn('_id', fixed[1])
+
+  def test_mongo_uri_escaped(self):
+    host = username = password = auth_database = ':!@#$%/'
+    port = 1234
+    uri = mrs.get_mongo_uri(host=host, port=port, username=username, password=password, auth_database=auth_database)
+    self.assertEqual(uri, "mongodb://{un}:{pw}@{host}:{port}/{db}".format(
+      un=quote_plus(username), pw=quote_plus(password),
+      host=quote_plus(host), port=port, db=quote_plus(auth_database),
+    ))
+
+
+rs_id = 'a replset id'
+members = [
+  {'host': 'foo.bar:1234'},
+  {'host': 'bar.baz:4321'},
+]
+old_rs_config = {
+  'version': 1,
+  '_id': rs_id,
+  'members': [
+    {'_id': 0, 'host': 'foo.bar:1234',},
+    {'_id': 1, 'host': 'bar.baz:4321',},
+  ]
+}
+new_rs_config = {
+  'version': 2,
+  '_id': rs_id,
+  'members': [
+    {'_id': 0, 'host': 'foo.bar:1234',},
+    {'_id': 1, 'host': 'bar.baz:4321',},
+    {'_id': 2, 'host': 'baz.bing:27017',},
+  ]
+}
+rs_config = {
+  'members': [
+    {'host': 'foo.bar', 'port': 1234,},
+    {'host': 'bar.baz', 'port': 4321,},
+    {'host': 'baz.bing', 'port': 27017,},
+  ]
+}
+
+def init_replset_mock(f):
+  get_replset_initialize_mock = mock.patch.object(mrs, 'get_replset', 
+    side_effect=(None, deepcopy(new_rs_config)))
+  initialize_replset_mock = mock.patch.object(mrs, 'initialize_replset')
+  return get_replset_initialize_mock(initialize_replset_mock(f))
+  
+def update_replset_mock(f):
+  get_replset_update_mock = mock.patch.object(mrs, 'get_replset', 
+    side_effect=(deepcopy(old_rs_config), deepcopy(new_rs_config)))
+  reconfig_replset_mock = mock.patch.object(mrs, 'reconfig_replset')
+  return get_replset_update_mock(reconfig_replset_mock(f))
+
+@mock.patch.object(mrs, 'get_rs_config_id', return_value=rs_id)
+@mock.patch.object(mrs, 'client', create=True)
+@mock.patch.object(mrs, 'module', create=True)
+class TestPatchingMongodbReplicaSet(unittest.TestCase):
+  
+  @update_replset_mock
+  def test_version_managed(self, _1, _2, module, *args):
+    # Version set automatically on initialize
+    mrs.update_replset(deepcopy(rs_config))
+    new_version = module.exit_json.call_args[1]['config']['version']
+    self.assertEqual(old_rs_config['version'], new_version - 1)
+
+  @init_replset_mock
+  def test_doc_id_managed_on_initialize(self, _1, _2, module, *args):
+    #old_rs_config provided by init_replset_mock via mrs.get_replset().
+    #That returns None on the first call, so it falls through to get_rs_config_id(),
+    #which is also mocked.
+    mrs.update_replset(deepcopy(rs_config))
+    new_id = module.exit_json.call_args[1]['config']['_id']
+    self.assertEqual(rs_id, new_id)
+
+  @update_replset_mock
+  def test_doc_id_managed_on_update(self, _1, _2, module, *args):
+    #old_rs_config provided by update_replset_mock via mrs.get_replset()
+    mrs.update_replset(deepcopy(rs_config))
+    new_id = module.exit_json.call_args[1]['config']['_id']
+    self.assertEqual(rs_id, new_id)
+
+  @init_replset_mock
+  def test_initialize_if_necessary(self, initialize_replset, _2, module, *args):
+    mrs.update_replset(deepcopy(rs_config))
+    self.assertTrue(initialize_replset.called)
+    #self.assertFalse(reconfig_replset.called)
+
+  @update_replset_mock
+  def test_reconfig_if_necessary(self, reconfig_replset, _2, module, *args):
+    mrs.update_replset(deepcopy(rs_config))
+    self.assertTrue(reconfig_replset.called)
+    #self.assertFalse(initialize_replset.called)
+
+  @update_replset_mock
+  def test_not_changed_when_docs_match(self, _1, _2, module, *args):
+    rs_config = {'members': members}  #This way the docs "match", but aren't identical
+
+    mrs.update_replset(deepcopy(rs_config))
+    changed = module.exit_json.call_args[1]['changed']
+    self.assertFalse(changed)
+
+  @update_replset_mock
+  def test_ignores_magic_given_full_doc(self, _1, _2, module, _3, get_rs_config_id, *args):
+    mrs.update_replset(deepcopy(new_rs_config))
+    new_doc = module.exit_json.call_args[1]['config']
+    self.assertEqual(new_doc, new_rs_config)
+    self.assertFalse(get_rs_config_id.called)
+
+
+if __name__ == '__main__':
+  unittest.main()