Copy mongo_3_0 role to create mongo_3_2 OPS-2205

playbooks/edx-east/mongo_3_2.yml

+# Manages a mongo cluster.
+# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG
+# as used by mongo_replica_set in the mongo_3_0 role.
+#
+# If you are initializing a cluster, your command might look like:
+# ansible-playbook mongo_3_0.yml -i 10.1.1.1,10.2.2.2,10.3.3.3 -e@/path/to/edx.yml -e@/path/to/ed.yml
+# If you just want to deploy an updated replica set config, you can run
+# ansible-playbook mongo_3_0.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set
+#
+# ADDING A NEW CLUSTER MEMBER
+# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory
+# ansible-playbook mongo_3_0.yml -i 10.1.1.1,10.2.2.2,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml
+- name: Deploy MongoDB
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - aws
+    - enhanced_networking
+    - mongo_3_0
+    - munin_node
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE

playbooks/roles/mongo_3_2/defaults/main.yml

+mongo_logappend: true
+
+#This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle
+mongo_logrotate: reopen
+
+mongo_version: 3.0.14
+mongo_port: "27017"
+mongo_extra_conf: ''
+mongo_key_file: '/etc/mongodb_key'
+pymongo_version: 2.8.1
+
+mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo"
+mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo"
+mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal"
+mongo_user: mongodb
+
+# The MONGODB_REPO variable should use {{ ansible_distribution_release }}
+# instead of hard coding a release name. Since we are already accidentally
+# running precise binaries on trusty, we are going to leave this alone for
+# mongo 3.0 and remedy it when we upgrade to mongo 3.2
+MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.0 multiverse"
+MONGODB_APT_KEY: "7F0CEB10"
+MONGODB_APT_KEYSERVER: "keyserver.ubuntu.com"
+
+mongodb_debian_pkgs:
+  - "mongodb-org={{ mongo_version }}"
+  - "mongodb-org-server={{ mongo_version }}" 
+  - "mongodb-org-shell={{ mongo_version }}"
+  - "mongodb-org-mongos={{ mongo_version }}" 
+  - "mongodb-org-tools={{ mongo_version }}"
+
+
+# Vars Meant to be overridden
+MONGO_ADMIN_USER: 'admin'
+MONGO_ADMIN_PASSWORD: 'password'
+MONGO_USERS:
+  - user: cs_comments_service
+    password: password
+    database: cs_comments_service
+    roles: readWrite
+  - user: edxapp
+    password: password
+    database: edxapp
+    roles: readWrite
+
+MONGO_CLUSTERED: false
+MONGO_BIND_IP: 127.0.0.1
+MONGO_REPL_SET: "rs0"
+MONGO_AUTH: true
+
+# Cluster member configuration
+# Fed directly into mongodb_replica_set module
+MONGO_RS_CONFIG:
+  members: []
+
+# Storage engine options in 3.0: "mmapv1" or "wiredTiger" 
+# As 3.2 and 3.4 default to wiredTiger, our 3.0 play will help future-proof
+MONGO_STORAGE_ENGINE: "wiredTiger"
+
+# List of dictionaries as described in the mount_ebs role's default
+# for the volumes.
+# Useful if you want to store your mongo data and/or journal on separate
+# disks from the root volume.  By default, they will end up mongo_data_dir
+# on the root disk.
+MONGO_VOLUMES: []
+
+# WiredTiger takes a number of optional configuration settings
+# which can be defined as a yaml structure in your secure configuration.
+MONGO_STORAGE_ENGINE_OPTIONS: !!null
+
+mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
+mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
+
+# In environments that do not require durability (devstack / Jenkins)
+# you can disable the journal to reduce disk usage
+mongo_enable_journal: true
+
+MONGO_LOG_SERVERSTATUS: true

playbooks/roles/mongo_3_2/files/disable-transparent-hugepages.conf

+description     "set transparent hugepage to never"
+
+start on starting mongod
+task
+
+script
+  echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled
+  echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag
+end script

playbooks/roles/mongo_3_2/handlers/main.yml

+---
+- name: restart mongo
+  service: name=mongod state=restarted
+

playbooks/roles/mongo_3_2/meta/main.yml

+---
+dependencies:
+  - common
+  - role: mount_ebs
+    volumes: "{{ MONGO_VOLUMES }}"

playbooks/roles/mongo_3_2/tasks/main.yml

+---
+- name: disable transparent huge pages on startup (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/)
+  copy:
+    src: disable-transparent-hugepages.conf
+    dest: /etc/init/disable-transparent-hugepages.conf
+    owner: root
+    group: root
+    mode: 0755
+  tags:
+    - "hugepages"
+    - "install"
+    - "install:system-requirements"
+
+- name: disable transparent huge pages
+  service:
+    name: disable-transparent-hugepages
+    enabled: yes
+    state: started
+  tags:
+    - "hugepages"
+    - "install"
+    - "install:system-requirements"
+
+- name: install python pymongo for mongo_user ansible module
+  pip:
+    name: pymongo
+    state: present
+    version: "{{ pymongo_version }}"
+    extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+  tags:
+    - "install"
+    - "install:system-requirements"
+
+- name: add the mongodb signing key
+  apt_key:
+    id: "{{ MONGODB_APT_KEY }}"
+    keyserver: "{{ MONGODB_APT_KEYSERVER }}"
+    state: present
+  tags:
+    - "install"
+    - "install:system-requirements"
+
+- name: add the mongodb repo to the sources list
+  apt_repository:
+    repo: "{{ MONGODB_REPO }}"
+    state: present
+  tags:
+    - "install"
+    - "install:system-requirements"
+
+- name: install mongo server and recommends
+  apt:
+    pkg: "{{ item }}"
+    state: present
+    install_recommends: yes
+    force: yes
+    update_cache: yes
+  with_items: "{{ mongodb_debian_pkgs }}"
+  tags:
+    - install
+    - install:app-requirements
+    - mongo_packages
+
+- name: create mongo dirs
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ mongo_user  }}"
+    group: "{{ mongo_user }}"
+  with_items:
+    - "{{ mongo_data_dir }}"
+    - "{{ mongo_dbpath }}"
+    - "{{ mongo_log_dir }}"
+    - "{{ mongo_journal_dir }}"
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: add serverStatus logging script
+  template:
+    src: "log-mongo-serverStatus.sh.j2"
+    dest: "{{ COMMON_BIN_DIR }}/log-mongo-serverStatus.sh"
+    owner: "{{ mongo_user }}"
+    group: "{{ mongo_user }}"
+    mode: 0700
+  when: MONGO_LOG_SERVERSTATUS
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: add serverStatus logging script to cron
+  cron:
+    name: mongostat logging job
+    minute: "*/3"
+    job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1
+  become: yes
+  when: MONGO_LOG_SERVERSTATUS
+  tags:
+    - "install"
+    - "install:configuration"
+
+# This will error when run on a new replica set, so we ignore_errors
+# and connect anonymously next.
+- name: determine if there is a replica set already
+  mongodb_rs_status:
+    host: "{{ ansible_default_ipv4['address'] }}"
+    username: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+  run_once: true
+  register: authed_replica_set_already_configured
+  when: MONGO_CLUSTERED
+  ignore_errors: true
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: Try checking the replica set with no user/pass in case this is a new box
+  mongodb_rs_status:
+    host: "{{ ansible_default_ipv4['address'] }}"
+  run_once: true
+  register: unauthed_replica_set_already_configured
+  when: MONGO_CLUSTERED and authed_replica_set_already_configured.failed is defined
+  ignore_errors: true
+  tags:
+    - "install"
+    - "install:configuration"
+
+# We use these in the templates but also to control a whole bunch of logic
+- name: set facts that default to not initializing a replica set
+  set_fact:
+    initialize_replica_set: false
+    skip_replica_set: false
+  tags:
+    - "install"
+    - "install:configuration"
+    - "update_mongod_conf"
+
+# If either auth or unauthed access comes back with a replica set, we
+# do not want to initialize one.  Since initialization requires a bunch
+# of extra templating and restarting, it's not something we want to do on
+# existing boxes.
+- name: track if you have a replica set
+  set_fact:
+    initialize_replica_set: true
+    skip_replica_set: true
+  when: MONGO_CLUSTERED
+        and authed_replica_set_already_configured.status is not defined
+        and unauthed_replica_set_already_configured.status is not defined
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: warn about unconfigured replica sets
+  debug: msg="You do not appear to have a Replica Set configured, deploying one for you"
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: copy mongodb key file
+  copy:
+    content: "{{ MONGO_CLUSTER_KEY }}"
+    dest: "{{ mongo_key_file }}"
+    mode: 0600
+    owner: mongodb
+    group: mongodb
+  when: MONGO_CLUSTERED
+  notify: restart mongo
+  tags:
+    - "install"
+    - "install:configuration"
+    - "mongodb_key"
+
+# If skip_replica_set is true, this template will not contain a replica set stanza
+# because of the fact above.
+- name: copy configuration template
+  template:
+    src: mongod.conf.j2
+    dest: /etc/mongod.conf
+    backup: yes
+  notify: restart mongo
+  register: update_mongod_conf
+  tags:
+    - "install"
+    - "install:configuration"
+    - "update_mongod_conf"
+
+- name: install logrotate configuration
+  template:
+    src: mongo_logrotate.j2
+    dest: /etc/logrotate.d/hourly/mongo
+  tags:
+    - "install"
+    - "install:configuration"
+    - "logrotate"
+
+- name: restart mongo service if we changed our configuration
+  service:
+    name: mongod
+    state: restarted
+  when: update_mongod_conf.changed
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: wait for mongo server to start
+  wait_for:
+    port: 27017
+    delay: 2
+  tags:
+    - "install"
+    - "install:configuration"
+
+# We only try passwordless superuser creation when
+# we're initializing the replica set and need to use
+# the localhost exemption to create a user who will be
+# able to initialize the replica set.
+# We can only create the users on one machine, the one
+# where we will initialize the replica set.  If we
+# create users on multiple hosts, then they will fail
+# to come into the replica set.
+- name: create super user
+  mongodb_user:
+    name: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+    database: admin
+    roles: root
+  when: initialize_replica_set
+  run_once: true
+  tags:
+    - "manage"
+    - "manage:db"
+
+- name: create super user
+  mongodb_user:
+    name: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+    login_user: "{{ MONGO_ADMIN_USER }}"
+    login_password: "{{ MONGO_ADMIN_PASSWORD }}"
+    database: admin
+    roles: root
+  run_once: true
+  when: not initialize_replica_set
+  tags:
+    - "manage"
+    - "manage:db"
+
+# Now that the localhost exemption has been used to create the superuser, we need
+# to add replica set to our configuration.  This will never happen if we detected
+# a replica set in the 'determine if there is a replica set already' task.
+- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set
+  set_fact:
+    skip_replica_set: false
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: re-copy configuration template with replica set enabled
+  template:
+    src: mongod.conf.j2
+    dest: /etc/mongod.conf
+    backup: yes
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: restart mongo service
+  service:
+    name: mongod
+    state: restarted
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: wait for mongo server to start
+  wait_for:
+    port: 27017
+    delay: 2
+  when: MONGO_CLUSTERED and initialize_replica_set
+  tags:
+    - "install"
+    - "install:configuration"
+
+- name: configure replica set
+  mongodb_replica_set:
+    username: "{{ MONGO_ADMIN_USER }}"
+    password: "{{ MONGO_ADMIN_PASSWORD }}"
+    rs_config: "{{ MONGO_RS_CONFIG }}"
+  run_once: true
+  register: replset_status
+  when: MONGO_CLUSTERED
+  tags:
+    - "manage"
+    - "manage:db"
+    - "configure_replica_set"
+
+# During initial replica set configuration, it can take a few seconds to vote
+# a primary and for all members to reflect that status.  During that window,
+# use creation or other writes can fail.  The best wait/check seems to be repeatedly
+# checking the replica set status until we see a PRIMARY in the results.
+- name: Wait for the replica set to update and (if needed) elect a primary
+  mongodb_rs_status:
+      host: "{{ ansible_default_ipv4['address'] }}"
+      username: "{{ MONGO_ADMIN_USER }}"
+      password: "{{ MONGO_ADMIN_PASSWORD }}"
+  register: status
+  until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list
+  retries: 5
+  delay: 2
+  run_once: true
+  when: MONGO_CLUSTERED
+  tags:
+    - "manage"
+    - "manage:db"
+
+- name: create mongodb users in a replica set
+  mongodb_user:
+    database: "{{ item.database }}"
+    login_database: 'admin'
+    login_user: "{{ MONGO_ADMIN_USER }}"
+    login_password: "{{ MONGO_ADMIN_PASSWORD }}"
+    name: "{{ item.user }}"
+    password: "{{ item.password }}"
+    roles: "{{ item.roles }}"
+    state: present
+    replica_set: "{{ MONGO_REPL_SET }}"
+  with_items: "{{ MONGO_USERS }}"
+  run_once: true
+  when: MONGO_CLUSTERED
+  tags:
+    - "manage"
+    - "manage:db"
+
+- name: create mongodb users in a standalone configuration
+  mongodb_user:
+    database: "{{ item.database }}"
+    login_user: "{{ MONGO_ADMIN_USER }}"
+    login_password: "{{ MONGO_ADMIN_PASSWORD }}"
+    name: "{{ item.user }}"
+    password: "{{ item.password }}"
+    roles: "{{ item.roles }}"
+    state: present
+  with_items: "{{ MONGO_USERS }}"
+  when: not MONGO_CLUSTERED
+  tags:
+    - "manage"
+    - "manage:db"

playbooks/roles/mongo_3_2/templates/create_root.js.j2

+// Add super user
+conn = new Mongo();
+db = conn.getDB("admin");
+db.createUser(
+    {
+        "user": "{{ MONGO_ADMIN_USER }}",
+        "pwd": "{{ MONGO_ADMIN_PASSWORD }}",
+        "roles": ["root"]
+    }
+);
+
+

playbooks/roles/mongo_3_2/templates/log-mongo-serverStatus.sh.j2

+#!/usr/bin/env bash
+# Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON
+/usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())'

playbooks/roles/mongo_3_2/templates/mongo_logrotate.j2

+{{ mongo_log_dir }}/serverStatus.log {
+  create
+  compress
+  copytruncate
+  delaycompress
+  dateext
+  dateformat -%Y%m%d-%s
+  missingok
+  notifempty
+  daily
+  rotate 90
+  size 1M
+}
+
+{{ mongo_log_dir }}/mongodb.log {
+  create
+  compress
+  copytruncate
+  delaycompress
+  dateext
+  dateformat -%Y%m%d-%s
+  missingok
+  notifempty
+  daily
+  rotate 90
+  size 1M
+  postrotate
+    /usr/bin/killall -USR1 mongod
+  endscript
+}

playbooks/roles/mongo_3_2/templates/mongod.conf.j2

+# Do not edit this file directly, it was generated by ansible
+# mongodb.conf
+
+
+storage:
+  # Where to store the data.
+  dbPath: {{ mongo_dbpath }}
+  # Storage Engine
+  engine: {{ MONGO_STORAGE_ENGINE }}
+  # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
+  journal:
+{% if mongo_enable_journal %}
+    enabled: true
+{% else %}
+    enabled: false
+{% endif %}
+{% if MONGO_STORAGE_ENGINE_OPTIONS %}
+  {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }}
+{% endif %}
+
+systemLog:
+  #where to log
+  destination: file
+  path: "{{ mongo_logpath }}"
+{% if mongo_logappend  %}
+  logAppend: true
+{% else %}
+  logAppend: false
+{% endif %}
+  logRotate: {{ mongo_logrotate }}
+
+{% if MONGO_CLUSTERED and not skip_replica_set %}
+replication:
+  replSetName: {{ MONGO_REPL_SET }}
+
+security:
+  authorization: {{ MONGO_AUTH | ternary("enabled", "disabled") }}
+  keyFile: {{ mongo_key_file }}
+
+{% endif %}
+net:
+{% if not MONGO_CLUSTERED %}
+  {# Bind to all ips(default) if in clustered mode, 
+     otherwise only to the specified local ip. #}   
+  bindIp: {{ MONGO_BIND_IP }}
+{% endif %}
+  port: {{ mongo_port }}
+
+
+{{ mongo_extra_conf }}