Refactor of role to handle initialization and reruns

Previously on this branch, it would attempt to create a superuser and
initialize the replica set after the local host exemption had passed
(once a mongod is started with a keyfile and the replicationSetName
 config file option or --replSet command line argument, you can't create
 users using the localhost exemption anymore ).

This necesarily complicates the logic and requires us to track if we're
initializing a replica set or not (with facts) and to template out
/etc/mongod.conf twice in the initialization case.

Further modifications to the replica set can be run against all the
members and will install mongo on new machines while updating the
replica set on the old machines.  With a further commit on this branch,
we can avoid needlessly restarting mongo on all machines in the cluster.

When we run this on an existing cluster, this stops and starts mongo for
all machines it is run against.  We can't rely on the notify: since that
happens at the end of the play and during initial cluster setup, we need
to restart and pick up the changed config.

Try to stick sensible tags on things

Ensure mongo restarts on the cluster key file change

Add a few playbook notes about common operations

playbooks/edx-east/mongo_3_0.yml

+# Manages a mongo cluster.
+# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG
+# as used by mongo_replica_set in the mongo_3_0 role.
+#
+# If you are initializing a cluster, your command might look like:
+# ansible-playbook mongo_3_0.yml -i 10.1.1.1,10.2.2.2,10.3.3.3 -e@/path/to/edx.yml -e@/path/to/ed.yml
+# If you just want to deploy an updated replica set config, you can run
+# ansible-playbook mongo_3_0.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set
+#
+# ADDING A NEW CLUSTER MEMBER
+# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory
+# ansible-playbook mongo_3_0.yml -i 10.1.1.1,10.2.2.2,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml
 - name: Deploy MongoDB
   hosts: all
   sudo: True

playbooks/roles/mongo_3_0/tasks/main.yml

 ---
-# - name: check to see if MongoDB is already installed
-#   stat:
-#     path: /etc/init.d/mongod
-#   register: mongodb_already_installed
-#   when: MONGO_CLUSTERED
-
-# - name: verify MongoDB not installed (clustered upgrades)
-#   fail: msg="MongoDB is currently installed and cannot be safely upgraded in a clustered configuration.  FIXME"
-#   when: mongodb_already_installed.stat.exists and MONGO_CLUSTERED
-
 - name: disable transparent huge pages on startup (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/)
   copy:
     src: disable-transparent-hugepages.conf
@@ -37,19 +27,27 @@
     state: present
     version: "{{ pymongo_version }}"
     extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+  tags:
+    - "install"
+    - "install:system-requirements"
 
 - name: add the mongodb signing key
   apt_key:
     id: "{{ MONGODB_APT_KEY }}"
     keyserver: "{{ MONGODB_APT_KEYSERVER }}"
     state: present
+  tags:
+    - "install"
+    - "install:system-requirements"
 
 - name: add the mongodb repo to the sources list
   apt_repository:
     repo: "{{ MONGODB_REPO }}"
     state: present
+  tags:
+    - "install"
+    - "install:system-requirements"
 
-#Will this break if it actually upgrades?
 - name: install mongo server and recommends
   apt:
     pkg: "{{ item }}"
@@ -60,7 +58,7 @@
   with_items: mongodb_debian_pkgs
   tags:
     - install
-    - install:system-requirements
+    - install:app-requirements
     - mongo_packages
 
 - name: create mongo dirs
@@ -74,6 +72,9 @@
     - "{{ mongo_dbpath }}"
     - "{{ mongo_log_dir }}"
     - "{{ mongo_journal_dir }}"
+  tags:
+    - "install"
+    - "install:configuration"
 
 - name: add serverStatus logging script
   template:
@@ -83,6 +84,9 @@
     group: "{{ mongo_user }}"
     mode: 0700
   when: MONGO_LOG_SERVERSTATUS
+  tags:
+    - "install"
+    - "install:configuration"
 
 - name: add serverStatus logging script to cron
   cron:
@@ -90,12 +94,67 @@
     job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1
   become: yes
   when: MONGO_LOG_SERVERSTATUS
+  tags:
+    - "install"
+    - "install:configuration"
 
-#### DANGER WILL ROBINSON ####
-- name: stop mongod service
-  service:
-    name: mongod
-    state: stopped
+# This will error when run on a new replica set, so we ignore_errors
+# and connect anonymously next.
+- name: determine if there is a replica set already
+  mongodb_rs_status:
+    host: "{{ ansible_default_ipv4['address'] }}"
+    username: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+  run_once: true
+  register: authed_replica_set_already_configured
+  when: MONGO_CLUSTERED
+  ignore_errors: true
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: Try checking the replica set with no user/pass in case this is a new box
+  mongodb_rs_status:
+    host: "{{ ansible_default_ipv4['address'] }}"
+  run_once: true
+  register: unauthed_replica_set_already_configured
+  when: MONGO_CLUSTERED and authed_replica_set_already_configured.failed is defined
+  ignore_errors: true
+  tags:
+    - "install"
+    - "install:configuration"
+
+# We use these in the templates but also to control a whole bunch of logic
+- name: set facts that default to not initializing a replica set
+  set_fact:
+    initialize_replica_set: false
+    skip_replica_set: false
+  tags:
+    - "install"
+    - "install:configuration"
+    - "update_mongod_conf"
+
+# If either auth or unauthed access comes back with a replica set, we
+# do not want to initialize one.  Since initialization requires a bunch
+# of extra templating and restarting, it's not something we want to do on
+# existing boxes.
+- name: track if you have a replica set
+  set_fact:
+    initialize_replica_set: true
+    skip_replica_set: true
+  when: MONGO_CLUSTERED
+        and authed_replica_set_already_configured.status is not defined
+        and unauthed_replica_set_already_configured.status is not defined
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: warn about unconfigured replica sets
+  debug: msg="You do not appear to have a Replica Set configured, deploying one for you"
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
 
 - name: copy mongodb key file
   copy:
@@ -105,13 +164,25 @@
     owner: mongodb
     group: mongodb
   when: MONGO_CLUSTERED
+  notify: restart mongo
+  tags:
+    - "install"
+    - "install:configuration"
+    - "mongodb_key"
 
+# If skip_replica_set is true, this template will not contain a replica set stanza
+# because of the fact above.
 - name: copy configuration template
   template:
     src: mongod.conf.j2
     dest: /etc/mongod.conf
     backup: yes
   notify: restart mongo
+  register: update_mongod_conf
+  tags:
+    - "install"
+    - "install:configuration"
+    - "update_mongod_conf"
 
 - name: install logrotate configuration
   template:
@@ -122,16 +193,95 @@
     - "install:configuration"
     - "logrotate"
 
-- name: start mongo service
+- name: restart mongo service if we changed our configuration
   service:
     name: mongod
-    state: started
+    state: restarted
+  when: update_mongod_conf.changed
+  tags:
+    - "install"
+    - "install:configuration"
 
 - name: wait for mongo server to start
   wait_for:
     port: 27017
     delay: 2
-######### END DANGER #########
+  tags:
+    - "install"
+    - "install:configuration"
+
+# We only try passwordless superuser creation when
+# we're initializing the replica set and need to use
+# the localhost exemption to create a user who will be
+# able to initialize the replica set.
+# We can only create the users on one machine, the one
+# where we will initialize the replica set.  If we
+# create users on multiple hosts, then they will fail
+# to come into the replica set.
+- name: create super user
+  mongodb_user:
+    name: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+    database: admin
+    roles: root
+  when: initialize_replica_set
+  run_once: true
+  tags:
+    - "manage"
+    - "manage:db"
+
+- name: create super user
+  mongodb_user:
+    name: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+    login_user: "{{ MONGO_ADMIN_USER }}"
+    login_password: "{{ MONGO_ADMIN_PASSWORD }}"
+    database: admin
+    roles: root
+  run_once: true
+  when: not initialize_replica_set
+  tags:
+    - "manage"
+    - "manage:db"
+
+# Now that the localhost exemption has been used to create the superuser, we need
+# to add replica set to our configuration.  This will never happen if we detected
+# a replica set in the 'determine if there is a replica set already' task.
+- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set
+  set_fact:
+    skip_replica_set: false
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: re-copy configuration template with replica set enabled
+  template:
+    src: mongod.conf.j2
+    dest: /etc/mongod.conf
+    backup: yes
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: restart mongo service
+  service:
+    name: mongod
+    state: restarted
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: wait for mongo server to start
+  wait_for:
+    port: 27017
+    delay: 2
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
 
 - name: configure replica set
   mongodb_replica_set:
@@ -141,33 +291,53 @@
   run_once: true
   register: replset_status
   when: MONGO_CLUSTERED
+  tags:
+    - configure_replica_set
+  tags:
+    - "manage"
+    - "manage:db"
+    - "configure_replica_set"
 
-- name: Choose primary
-  set_fact:
-    mongo_rs_primary: >
-      {{ 
-      (replset_status.config.members
-      | selectattr('stateStr', 'equalto', 'PRIMARY')
-      | attr('name') | replace(':'+string(mongo_port), '')
-      )
-      }}
+# During initial replica set configuration, it can take a few seconds to vote
+# a primary and for all members to reflect that status.  During that window,
+# use creation or other writes can fail.  The best wait/check seems to be repeatedly
+# checking the replica set status until we see a PRIMARY in the results.
+- name: Wait for the replica set to update and (if needed) elect a primary
+  mongodb_rs_status:
+      host: "{{ ansible_default_ipv4['address'] }}"
+      username: "{{ MONGO_ADMIN_USER }}"
+      password: "{{ MONGO_ADMIN_PASSWORD }}"
+  register: status
+  until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list
+  retries: 5
+  delay: 2
+  run_once: true
   when: MONGO_CLUSTERED
+  tags:
+    - configure_replica_set
+  tags:
+    - "manage"
+    - "manage:db"
 
-- name: Set fake primary host
-  set_fact:
-    mongo_rs_primary: localhost
-  when: not MONGO_CLUSTERED
-
-- name: create super user
+- name: create mongodb users in a replica set
   mongodb_user:
-    name: "{{ MONGO_ADMIN_USER }}"
-    password: "{{ MONGO_ADMIN_PASSWORD }}"
-    database: admin
-    roles: root
+    database: "{{ item.database }}"
+    login_database: 'admin'
+    login_user: "{{ MONGO_ADMIN_USER }}"
+    login_password: "{{ MONGO_ADMIN_PASSWORD }}"
+    name: "{{ item.user }}"
+    password: "{{ item.password }}"
+    roles: "{{ item.roles }}"
+    state: present
+    replica_set: "{{ MONGO_REPL_SET }}"
+  with_items: MONGO_USERS
   run_once: true
-  delegate_to: "{{ mongo_rs_primary }}"
+  when: MONGO_CLUSTERED
+  tags:
+    - "manage"
+    - "manage:db"
 
-- name: create mongodb users
+- name: create mongodb users in a standalone configuration
   mongodb_user:
     database: "{{ item.database }}"
     login_user: "{{ MONGO_ADMIN_USER }}"
@@ -177,5 +347,7 @@
     roles: "{{ item.roles }}"
     state: present
   with_items: MONGO_USERS
-  run_once: true
-  delegate_to: "{{ mongo_rs_primary }}"
+  when: not MONGO_CLUSTERED
+  tags:
+    - "manage"
+    - "manage:db"

playbooks/roles/mongo_3_0/templates/mongod.conf.j2

@@ -29,7 +29,7 @@ systemLog:
 {% endif %}
   logRotate: {{ mongo_logrotate }}
 
-{% if MONGO_CLUSTERED %}
+{% if MONGO_CLUSTERED and not skip_replica_set %}
 replication:
   replSetName: {{ MONGO_REPL_SET }}