Merge pull request #3106 from edx/jibsheet/mongo-full-rolling-upgrades

Full rolling updates of a cluster

playbooks/edx-east/mongo_rolling_upgrade.yml

+# Upgrades a full mongo cluster, starting with the hidden members, then the
+# secondary, and finally steps down the primary and upgrades it.  It checks along
+# the way for a healthy cluster, failing if that is not true.
+#
+# This play expects to have access to a config file where MONGO_RS_CONFIG, as described
+# in the mongo_3_0 role, is defined, as well as MONGO_ADMIN_USER and MONGO_ADMIN_PASSWORD.
+#
+# ansible-playbook -i 127.0.0.1, mongo_rolling_upgrade.yml -e@/path/to/config-file.yml
+#
+# This play uses MONGO_RS_CONFIG to find a host to connect to and fetch replset config and build an
+# inventory, so you can just target localhost.
+#
+# If there are no hidden secondaries, the 'Upgrade hidden members' task block will just skip.
+#
+# This will process a hidden secondary twice - first as a 'hidden' server, then as a 'secondary' but
+# this is effectively a no-op except for apt checking the versions and then checking that mongo is running.
+# It is valid to have other types of hidden machines, so this seemed better than skipping.
+#
+# If you wish to avoid updating the primary, you can add -e 'SKIP_PRIMARY=true' to your ansible
+# invocation.
+
+- name: Find hidden secondaries
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: False
+  vars:
+    - SKIP_PRIMARY: False
+  tasks:
+    - name: Get configuration of mongo cluster
+      mongodb_rs_config:
+        host: "{{ (MONGO_RS_CONFIG.members|map(attribute='host')|list)[0] }}"
+        username: "{{ MONGO_ADMIN_USER }}"
+        password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: rs_config
+    - name: Build inventory of hidden members
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: hidden_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.hidden }}"
+    - name: Build inventory of secondary members
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: secondary_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.secondary }}"
+    - name: Build inventory of primary members
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: primary_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.primary }}"
+      when: not SKIP_PRIMARY
+
+- name: Upgrade hidden members
+  hosts: hidden_hosts
+  gather_facts: True
+  become: True
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      # This ensures that no servers are in a state other than PRIMARY or SECONDARY.  https://docs.mongodb.com/manual/reference/replica-states/
+      until: status.status is defined and not (['PRIMARY','SECONDARY'] | symmetric_difference(status.status.members|map(attribute='stateStr')|list|unique))
+      retries: 5
+      delay: 2
+
+- name: Upgrade secondary members
+  hosts: secondary_hosts
+  gather_facts: True
+  become: True
+  serial: 1
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      # This ensures that no servers are in a state other than PRIMARY or SECONDARY.  https://docs.mongodb.com/manual/reference/replica-states/
+      until: status.status is defined and not (['PRIMARY','SECONDARY'] | symmetric_difference(status.status.members|map(attribute='stateStr')|list|unique))
+      retries: 5
+      delay: 2
+
+- name: Upgrade primary members
+  hosts: primary_hosts
+  gather_facts: True
+  become: True
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: Step down (this can take up to a minute to complete while the primary waits on a secondary)
+      mongodb_step_down:
+        host: "{{ ansible_default_ipv4['address'] }}"
+        username: "{{ MONGO_ADMIN_USER }}"
+        password: "{{ MONGO_ADMIN_PASSWORD }}"
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      # This ensures that no servers are in a state other than PRIMARY or SECONDARY.  https://docs.mongodb.com/manual/reference/replica-states/
+      until: status.status is defined and not (['PRIMARY','SECONDARY'] | symmetric_difference(status.status.members|map(attribute='stateStr')|list|unique))
+      retries: 5
+      delay: 2

playbooks/library/mongodb_step_down

+#!/usr/bin/env python
+
+DOCUMENTATION = """
+---
+module: mongodb_step_down
+short_description: Issues a stepdown on the primary.
+description:
+  - Issues replSetStepDown on the host provided in host:.
+    Afterwards, loops several times to ensure that a new primary is elected and that
+    it is a different host than the previous primary.  Errors if the stepdown fails
+    or if the cluster fails to elect a new primary.
+version_added: "1.9"
+author:
+  - Kevin Falcone
+options:
+  host:
+    description:
+      - The hostname or ip of a server in the mongo cluster.
+    required: false
+    default: 'localhost'
+  port:
+    description:
+      - The port to connect to mongo on.
+    required: false
+    default: 27017
+  username:
+    description:
+      - The username of the mongo user to connect as.
+    required: false
+  password:
+    description:
+      - The password to use when authenticating.
+    required: false
+  auth_database:
+    description:
+      - The database to authenticate against.
+    required: false
+"""
+
+EXAMPLES = '''
+- name: Get status for the stage cluster
+  mongodb_step_down
+    host: localhost:27017
+    username: root
+    password: password
+
+'''
+# Magic import
+from ansible.module_utils.basic import *
+
+try:
+    from pymongo import MongoClient
+    from pymongo.errors import AutoReconnect
+    from bson import json_util
+except ImportError:
+    pymongo_found = False
+else:
+    pymongo_found = True
+
+import json
+from urllib import quote_plus
+
+def main():
+
+    arg_spec = dict(
+        host=dict(required=False, type='str', default="localhost"),
+        port=dict(required=False, type='int', default=27017),
+        username=dict(required=False, type='str'),
+        password=dict(required=False, type='str'),
+        auth_database=dict(required=False, type='str')
+    )
+
+    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=False)
+
+    if not pymongo_found:
+        module.fail_json(msg="The python pymongo module is not installed.")
+
+    mongo_uri = 'mongodb://'
+    host = module.params.get('host')
+    port = module.params.get('port')
+    username = module.params.get('username')
+    password = module.params.get('password')
+    auth_database = module.params.get('auth_database')
+
+    if (username and not password) or (password and not username):
+        module.fail_json(msg="Must provide both username and password or neither.")
+
+    if username:
+        mongo_uri += "{}:{}@".format(*map(quote_plus, [username,password]))
+
+    mongo_uri += "{}:{}".format(quote_plus(host),port)
+
+    if auth_database:
+        mongo_uri += '/{}'.format(quote_plus(auth_database))
+
+    client = MongoClient(mongo_uri)
+
+    # This has no return since it forces a disconnect or throws an error
+    # about being unable to elect a secondary.  We only catch the AutoReconnect
+    # so we see any other errors bubble up.
+    try:
+        client.admin.command("replSetStepDown",60,secondaryCatchUpPeriodSecs=30)
+    except AutoReconnect:
+        pass
+
+    for i in range(5):
+        status = client.admin.command("replSetGetStatus")
+        primary = [m for m in status['members'] if m['stateStr'] == 'PRIMARY']
+        # This won't work as well if you mix hostnames and IPs in your cluster.
+        # We use only IPs.
+        if primary and primary[0]['name'] != "{}:{}".format(quote_plus(host),port):
+            module.exit_json(changed=True, stepdown=True)
+        time.sleep(2)
+
+    module.fail_json(msg="Unable to step down {}".format(host))
+
+if __name__ == '__main__':
+    main()