Merge pull request #3104 from edx/jibsheet/rolling-mongo-upgrade

WIP - rolling mongodb upgrades

playbooks/edx-east/mongo_rolling_upgrade.yml

+# Upgrades a full mongo cluster, starting with the hidden members, then the
+# secondary, and finally steps down the primary and upgrades it.  It checks along
+# the way for a healthy cluster, failing if that is not true.
+#
+# This play expects to have access to a config file where MONGO_RS_CONFIG, as described
+# in the mongo_3_0 role, is defined, as well as MONGO_ADMIN_USER and MONGO_ADMIN_PASSWORD.
+#
+# ansible-playbook -i 127.0.0.1, mongo_rolling_upgrade.yml -e@/path/to/config-file.yml
+#
+# This play uses MONGO_RS_CONFIG to find a host to connect to and fetch replset config and build an
+# inventory, so you can just target localhost.
+#
+# If there are no hidden secondaries, the 'Upgrade hidden members' task block will just skip.
+#
+# This will process a hidden secondary twice - first as a 'hidden' server, then as a 'secondary' but
+# this is effectively a no-op except for apt checking the versions and then checking that mongo is running.
+# It is valid to have other types of hidden machines, so this seemed better than skipping.
+#
+# If you wish to avoid updating the primary, you can add -e 'SKIP_PRIMARY=true' to your ansible
+# invocation.
+
+- name: Find hidden secondaries
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: False
+  vars:
+    - SKIP_PRIMARY: False
+  tasks:
+    - name: Get configuration of mongo cluster
+      mongodb_rs_config:
+        host: "{{ (MONGO_RS_CONFIG.members|map(attribute='host')|list)[0] }}"
+        username: "{{ MONGO_ADMIN_USER }}"
+        password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: rs_config
+    - name: Build inventory of hidden members
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: hidden_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.hidden }}"
+    - name: Build inventory of secondary members
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: secondary_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.secondary }}"
+    - name: Build inventory of primary members
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: primary_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.primary }}"
+      when: not SKIP_PRIMARY
+
+- name: Upgrade hidden members
+  hosts: hidden_hosts
+  gather_facts: True
+  become: True
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      # This ensures that no servers are in a state other than PRIMARY or SECONDARY.  https://docs.mongodb.com/manual/reference/replica-states/
+      until: status.status is defined and not (['PRIMARY','SECONDARY'] | symmetric_difference(status.status.members|map(attribute='stateStr')|list|unique))
+      retries: 5
+      delay: 2
+
+- name: Upgrade secondary members
+  hosts: secondary_hosts
+  gather_facts: True
+  become: True
+  serial: 1
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      # This ensures that no servers are in a state other than PRIMARY or SECONDARY.  https://docs.mongodb.com/manual/reference/replica-states/
+      until: status.status is defined and not (['PRIMARY','SECONDARY'] | symmetric_difference(status.status.members|map(attribute='stateStr')|list|unique))
+      retries: 5
+      delay: 2
+
+- name: Upgrade primary members
+  hosts: primary_hosts
+  gather_facts: True
+  become: True
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: Step down (this can take up to a minute to complete while the primary waits on a secondary)
+      mongodb_step_down:
+        host: "{{ ansible_default_ipv4['address'] }}"
+        username: "{{ MONGO_ADMIN_USER }}"
+        password: "{{ MONGO_ADMIN_PASSWORD }}"
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      # This ensures that no servers are in a state other than PRIMARY or SECONDARY.  https://docs.mongodb.com/manual/reference/replica-states/
+      until: status.status is defined and not (['PRIMARY','SECONDARY'] | symmetric_difference(status.status.members|map(attribute='stateStr')|list|unique))
+      retries: 5
+      delay: 2

playbooks/edx-east/mongo_upgrade_hidden_secondaries.yml

+# Upgrades the hidden secondary in a mongo cluster if one exists
+#
+# This is useful for using the hidden secondary to sniff out any problems with
+# your point upgrade before you upgrade your primary/secondary servers using the
+# mongo_rolling_upgrade.yml play
+#
+# This play expects to have access to a config file where MONGO_RS_CONFIG, as described
+# in the mongo_3_0 role, is defined, as well as MONGO_ADMIN_USER and MONGO_ADMIN_PASSWORD.
+#
+# ansible-playbook -i 127.0.0.1, mongo_upgrade_hidden_secondaries.yml -e@/path/to/config-file.yml
+#
+# This play uses MONGO_RS_CONFIG to find a host to connect to and fetch replset config and build an
+# inventory, so you can just target localhost.
+#
+# If there are no hidden secondaries, the 'Upgrade hidden members' task will just skip.
+
+- name: Find hidden secondaries
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: False
+  tasks:
+    - name: Get configuration of mongo cluster
+      mongodb_rs_config:
+        host: "{{ (MONGO_RS_CONFIG.members|map(attribute='host')|list)[0] }}"
+        username: "{{ MONGO_ADMIN_USER }}"
+        password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: rs_config
+    - name: Build inventory of hidden secondaries
+      add_host:
+        hostname: "{{ (item.host.split(':'))[0] }}"
+        instance_id: "{{ item._id }}"
+        groups: hidden_hosts
+        ansible_ssh_user: ubuntu
+      with_items:
+        - "{{ rs_config.hidden }}"
+
+- name: Upgrade hidden members
+  hosts: hidden_hosts
+  gather_facts: True
+  become: True
+  vars_files:
+    - ../roles/mongo_3_0/defaults/main.yml
+  tasks:
+    - name: install mongo server and recommends
+      apt:
+        pkg: "{{ item }}"
+        state: present
+        install_recommends: yes
+        force: yes
+        update_cache: yes
+      with_items: mongodb_debian_pkgs
+    - name: wait for mongo server to start
+      wait_for:
+        port: 27017
+        delay: 2
+    - name: Wait for the replica set to update and (if needed) elect a primary
+      mongodb_rs_status:
+          host: "{{ ansible_default_ipv4['address'] }}"
+          username: "{{ MONGO_ADMIN_USER }}"
+          password: "{{ MONGO_ADMIN_PASSWORD }}"
+      register: status
+      until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list
+      retries: 5
+      delay: 2
+      run_once: true

playbooks/library/mongodb_rs_config

+#!/usr/bin/env python
+
+DOCUMENTATION = """
+---
+module: mongodb_rs_config
+short_description: Get the configuration of a replica set of a mongo cluster.
+description:
+  - Get the config of the replica set of a mongo cluster. Provides a filtered version of the info from rs.config().
+    returns primary, secondary, hidden.  Each contains a list of the members in that state.  Lists
+    may be empty.  Additionally returns the full config document in the config key.  Keep in mind that hosts may be
+    duplicated secondary and hidden since hidden hosts are secondaries.
+version_added: "1.9"
+author:
+  - Feanil Patel
+  - Kevin Falcone
+options:
+  host:
+    description:
+      - The hostname or ip of a server in the mongo cluster.
+    required: false
+    default: 'localhost'
+  port:
+    description:
+      - The port to connect to mongo on.
+    required: false
+    default: 27017
+  username:
+    description:
+      - The username of the mongo user to connect as.
+    required: false
+  password:
+    description:
+      - The password to use when authenticating.
+    required: false
+  auth_database:
+    description:
+      - The database to authenticate against.
+    required: false
+"""
+
+EXAMPLES = '''
+- name: Get status for the stage cluster
+  mongodb_rs_config:
+    host: localhost:27017
+    username: root
+    password: password
+  register: rs_config
+'''
+# Magic import
+from ansible.module_utils.basic import *
+
+try:
+    from pymongo import MongoClient
+    from pymongo.errors import OperationFailure
+    from bson import json_util
+except ImportError:
+    pymongo_found = False
+else:
+    pymongo_found = True
+
+import json
+from urllib import quote_plus
+
+def main():
+
+    arg_spec = dict(
+        host=dict(required=False, type='str', default="localhost"),
+        port=dict(required=False, type='int', default=27017),
+        username=dict(required=False, type='str'),
+        password=dict(required=False, type='str'),
+        auth_database=dict(required=False, type='str')
+    )
+
+    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=False)
+
+    if not pymongo_found:
+        module.fail_json(msg="The python pymongo module is not installed.")
+
+    mongo_uri = 'mongodb://'
+    host = module.params.get('host')
+    port = module.params.get('port')
+    username = module.params.get('username')
+    password = module.params.get('password')
+    auth_database = module.params.get('auth_database')
+
+    if (username and not password) or (password and not username):
+        module.fail_json(msg="Must provide both username and password or neither.")
+
+    if username:
+        mongo_uri += "{}:{}@".format(*map(quote_plus, [username,password]))
+
+    mongo_uri += "{}:{}".format(quote_plus(host),port)
+
+    if auth_database:
+        mongo_uri += '/{}'.format(quote_plus(auth_database))
+
+    client = MongoClient(mongo_uri)
+
+    # This checks to see if you have a replSetName configured
+    # This generally means that /etc/mongod.conf has been changed
+    # from the default to use a replica set and mongo has been
+    # restarted to use it.
+
+    try:
+        repl_set = client.admin.command('getCmdLineOpts')['parsed']['replication']['replSetName']
+    except (OperationFailure, KeyError):
+        module.exit_json(changed=False)
+
+    if repl_set:
+        status = client.admin.command("replSetGetStatus")
+        # Not using `replSetGetConfig` because it's not supported in MongoDB 2.x.
+        rs_config = client.local.system.replset.find_one()
+    else:
+        module.exit_json(changed=False)
+
+    # This converts the bson into a python dictionary that ansible's standard
+    # jsonify function can process and output without throwing errors on bson
+    # types that don't exist in JSON
+    status    = json.loads(json_util.dumps(status))
+    rs_config = json.loads(json_util.dumps(rs_config))
+
+    # Status contains information about Primary/Secondary, so we iterate that list
+    # But we want to return config for that host, not status (since this is the config module),
+    # this is the inner loop of the comprehension, where we match on the hostname (could also
+    # match on _id).
+    primary   = [ c for m in status['members'] if m['stateStr'] == 'PRIMARY'   for c in rs_config['members'] if m['name'] == c['host'] ]
+    secondary = [ c for m in status['members'] if m['stateStr'] == 'SECONDARY' for c in rs_config['members'] if m['name'] == c['host'] ]
+    # we're parsing the config directly here, much simpler
+    hidden    = [ m for m in rs_config['members'] if m['hidden'] ]
+
+    module.exit_json(changed=False, primary=primary, secondary=secondary, hidden=hidden, config=rs_config)
+
+if __name__ == '__main__':
+    main()
+

playbooks/library/mongodb_step_down

+#!/usr/bin/env python
+
+DOCUMENTATION = """
+---
+module: mongodb_step_down
+short_description: Issues a stepdown on the primary.
+description:
+  - Issues replSetStepDown on the host provided in host:.
+    Afterwards, loops several times to ensure that a new primary is elected and that
+    it is a different host than the previous primary.  Errors if the stepdown fails
+    or if the cluster fails to elect a new primary.
+version_added: "1.9"
+author:
+  - Kevin Falcone
+options:
+  host:
+    description:
+      - The hostname or ip of a server in the mongo cluster.
+    required: false
+    default: 'localhost'
+  port:
+    description:
+      - The port to connect to mongo on.
+    required: false
+    default: 27017
+  username:
+    description:
+      - The username of the mongo user to connect as.
+    required: false
+  password:
+    description:
+      - The password to use when authenticating.
+    required: false
+  auth_database:
+    description:
+      - The database to authenticate against.
+    required: false
+"""
+
+EXAMPLES = '''
+- name: Get status for the stage cluster
+  mongodb_step_down
+    host: localhost:27017
+    username: root
+    password: password
+
+'''
+# Magic import
+from ansible.module_utils.basic import *
+
+try:
+    from pymongo import MongoClient
+    from pymongo.errors import AutoReconnect
+    from bson import json_util
+except ImportError:
+    pymongo_found = False
+else:
+    pymongo_found = True
+
+import json
+from urllib import quote_plus
+
+def main():
+
+    arg_spec = dict(
+        host=dict(required=False, type='str', default="localhost"),
+        port=dict(required=False, type='int', default=27017),
+        username=dict(required=False, type='str'),
+        password=dict(required=False, type='str'),
+        auth_database=dict(required=False, type='str')
+    )
+
+    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=False)
+
+    if not pymongo_found:
+        module.fail_json(msg="The python pymongo module is not installed.")
+
+    mongo_uri = 'mongodb://'
+    host = module.params.get('host')
+    port = module.params.get('port')
+    username = module.params.get('username')
+    password = module.params.get('password')
+    auth_database = module.params.get('auth_database')
+
+    if (username and not password) or (password and not username):
+        module.fail_json(msg="Must provide both username and password or neither.")
+
+    if username:
+        mongo_uri += "{}:{}@".format(*map(quote_plus, [username,password]))
+
+    mongo_uri += "{}:{}".format(quote_plus(host),port)
+
+    if auth_database:
+        mongo_uri += '/{}'.format(quote_plus(auth_database))
+
+    client = MongoClient(mongo_uri)
+
+    # This has no return since it forces a disconnect or throws an error
+    # about being unable to elect a secondary.  We only catch the AutoReconnect
+    # so we see any other errors bubble up.
+    try:
+        client.admin.command("replSetStepDown",60,secondaryCatchUpPeriodSecs=30)
+    except AutoReconnect:
+        pass
+
+    for i in range(5):
+        status = client.admin.command("replSetGetStatus")
+        primary = [m for m in status['members'] if m['stateStr'] == 'PRIMARY']
+        # This won't work as well if you mix hostnames and IPs in your cluster.
+        # We use only IPs.
+        if primary and primary[0]['name'] != "{}:{}".format(quote_plus(host),port):
+            module.exit_json(changed=True, stepdown=True)
+        time.sleep(2)
+
+    module.fail_json(msg="Unable to step down {}".format(host))
+
+if __name__ == '__main__':
+    main()

playbooks/roles/ad_hoc_reporting/defaults/main.yml

@@ -43,9 +43,8 @@ ad_hoc_reporting_pip_pkgs:
 
 MONGODB_APT_KEY: "7F0CEB10"
 MONGODB_APT_KEYSERVER: "keyserver.ubuntu.com"
-MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse"
-mongo_version: 3.0.8
-
+MONGODB_REPO: "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen"
+mongo_version: 3.0.12
 
 # AD_HOC_REPORTING_REPLICA_DB_HOSTS:
 #   - db_host: "{{ EDXAPP_MYSQL_REPLICA_HOST }}"
@@ -55,6 +54,3 @@ mongo_version: 3.0.8
 #     depends_on: True
     
 AD_HOC_REPORTING_REPLICA_DB_HOSTS: []
-
-
-

playbooks/roles/mongo_3_0/defaults/main.yml

@@ -3,7 +3,7 @@ mongo_logappend: true
 #This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle
 mongo_logrotate: reopen
 
-mongo_version: 3.0.8
+mongo_version: 3.0.12
 mongo_port: "27017"
 mongo_extra_conf: ''
 mongo_key_file: '/etc/mongodb_key'