A hidden secondaries upgrade script

This will connect to a cluster, find the hidden secondaries and upgrade
them to the current version specified in the mongo_3_0 play.

This required the creation of a new library module since our two current
ones return rs.status and call replSetReconfig, but neither return
rs.config, and rs.config is what tells you if a host is hidden and the
priority, which are needed for this.

playbooks/edx-east/mongo_upgrade_hidden_secondaries.yml

+# Upgrades the hidden secondary in a mongo cluster if one exists
+#
+# This is useful for using the hidden secondary to sniff out any problems with
+# your point upgrade before you upgrade your primary/secondary servers using the
+# mongo_rolling_upgrade.yml play
+#
+# This play expects to have access to a config file where MONGO_RS_CONFIG, as described
+# in the mongo_3_0 role, is defined, as well as MONGO_ADMIN_USER and MONGO_ADMIN_PASSWORD.
+#
+# ansible-playbook -i 127.0.0.1, mongo_upgrade_hidden_secondaries.yml -e@/path/to/config-file.yml
+#
+# This play uses MONGO_RS_CONFIG to find a host to connect to and fetch replset config and build an
+# inventory, so you can just target localhost.
+#
+# If there are no hidden secondaries, the 'Upgrade hidden members' task will just skip.
+
+- name: Find hidden secondaries
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: False
+  tasks:
+    - name: Get status of mongo cluster
+      mongodb_rs_config:
+        host: "{{ (MONGO_RS_CONFIG.members|map(attribute='host')|list)[0] }}"
+        username: "{{ MONGO_ADMIN_USER }}"
+        password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: rs_config
+    - name: Build inventory of hidden secondaries
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: hidden_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.hidden }}"
+
+- name: Upgrade hidden members
+  hosts: hidden_hosts
+  gather_facts: True
+  become: True
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs

playbooks/library/mongodb_rs_config

+#!/usr/bin/env python
+
+DOCUMENTATION = """
+---
+module: mongodb_rs_config
+short_description: Get the configuration of a replica set of a mongo cluster.
+description:
+  - Get the config of the replica set of a mongo cluster. Provides a filtered version of the info from rs.config().
+    returns primary, secondary, hidden.  Each contains a list of the members in that state.  Lists
+    may be empty.  Additionally returns the full config document in the config key.  Keep in mind that hosts may be
+    duplicated secondary and hidden since hidden hosts are secondaries.
+version_added: "1.9"
+author:
+  - Feanil Patel
+  - Kevin Falcone
+options:
+  host:
+    description:
+      - The hostname or ip of a server in the mongo cluster.
+    required: false
+    default: 'localhost'
+  port:
+    description:
+      - The port to connect to mongo on.
+    required: false
+    default: 27017
+  username:
+    description:
+      - The username of the mongo user to connect as.
+    required: false
+  password:
+    description:
+      - The password to use when authenticating.
+    required: false
+  auth_database:
+    description:
+      - The database to authenticate against.
+    required: false
+"""
+
+EXAMPLES = '''
+- name: Get status for the stage cluster
+  mongodb_rs_config:
+    host: localhost:27017
+    username: root
+    password: password
+  register: rs_config
+'''
+# Magic import
+from ansible.module_utils.basic import *
+
+try:
+    from pymongo import MongoClient
+    from pymongo.errors import OperationFailure
+    from bson import json_util
+except ImportError:
+    pymongo_found = False
+else:
+    pymongo_found = True
+
+import json
+from urllib import quote_plus
+
+def main():
+
+    arg_spec = dict(
+        host=dict(required=False, type='str', default="localhost"),
+        port=dict(required=False, type='int', default=27017),
+        username=dict(required=False, type='str'),
+        password=dict(required=False, type='str'),
+        auth_database=dict(required=False, type='str')
+    )
+
+    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=False)
+
+    if not pymongo_found:
+        module.fail_json(msg="The python pymongo module is not installed.")
+
+    mongo_uri = 'mongodb://'
+    host = module.params.get('host')
+    port = module.params.get('port')
+    username = module.params.get('username')
+    password = module.params.get('password')
+    auth_database = module.params.get('auth_database')
+
+    if (username and not password) or (password and not username):
+        module.fail_json(msg="Must provide both username and password or neither.")
+
+    if username:
+        mongo_uri += "{}:{}@".format(*map(quote_plus, [username,password]))
+
+    mongo_uri += "{}:{}".format(quote_plus(host),port)
+
+    if auth_database:
+        mongo_uri += '/{}'.format(quote_plus(auth_database))
+
+    client = MongoClient(mongo_uri)
+
+    # This checks to see if you have a replSetName configured
+    # This generally means that /etc/mongod.conf has been changed
+    # from the default to use a replica set and mongo has been
+    # restarted to use it.
+
+    try:
+        repl_set = client.admin.command('getCmdLineOpts')['parsed']['replication']['replSetName']
+    except (OperationFailure, KeyError):
+        module.exit_json(changed=False)
+
+    if repl_set:
+        status = client.admin.command("replSetGetStatus")
+    else:
+        module.exit_json(changed=False)
+
+    # Not using `replSetGetConfig` because it's not supported in MongoDB 2.x.
+    if repl_set:
+        rs_config = client.local.system.replset.find_one()
+    else:
+        module.exit_json(changed=False)
+
+    # This converts the bson into a python dictionary that ansible's standard
+    # jsonify function can process and output without throwing errors on bson
+    # types that don't exist in JSON
+    status    = json.loads(json_util.dumps(status))
+    rs_config = json.loads(json_util.dumps(rs_config))
+
+    primary   = [ c for m in status['members'] if m['stateStr'] == 'PRIMARY'   for c in rs_config['members'] if m['name'] == c['host'] ]
+    secondary = [ c for m in status['members'] if m['stateStr'] == 'SECONDARY' for c in rs_config['members'] if m['name'] == c['host'] ]
+    # we're parsing the config here, much simpler
+    hidden    = [ m for m in rs_config['members'] if m['hidden'] ]
+
+    module.exit_json(changed=False, primary=primary, secondary=secondary, hidden=hidden, config=rs_config)
+
+if __name__ == '__main__':
+    main()
+