Merge pull request #1425 from edx/derf/mongo_backup_to_s3_role

add mongodb backup script

playbooks/roles/mongo/defaults/main.yml

@@ -31,7 +31,7 @@ mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
 mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
 
 # Have to use this conditional instead of ignore errors
-# because the mongo_user module fails and doesn't ginore errors.
+# because the mongo_user module fails and doesn't ignore errors.
 mongo_create_users: !!null
 
 # If the system is running out of an Amazon Web Services
@@ -42,3 +42,21 @@ mongo_aws_stack_name: "tag_aws_cloudformation_stack-name_"
 # In environments that do not require durability (devstack / Jenkins)
 # you can disable the journal to reduce disk usage
 mongo_enable_journal: True
+
+# We can do regular backups of MongoDB to S3.
+MONGO_S3_BACKUP: false
+# backup cron time:
+MONGO_S3_BACKUP_HOUR: "*/12"
+MONGO_S3_BACKUP_DAY: "*"
+# override with a secondary node that will perform backups
+MONGO_S3_BACKUP_NODE: "undefined"
+# back up data into a specific S3 bucket
+MONGO_S3_BACKUP_BUCKET: "undefined"
+# temporary directory mongodump will use to store data
+MONGO_S3_BACKUP_TEMPDIR: "{{ mongo_data_dir }}"
+MONGO_S3_NOTIFY_EMAIL: "dummy@example.com"
+mongo_s3_logfile: "{{ COMMON_LOG_DIR }}/mongo/s3-mongo-backup.log"
+MONGO_S3_S3CMD_CONFIG: "{{ COMMON_DATA_DIR }}/mongo-s3-backup.s3cfg"
+MONGO_S3_BACKUP_AWS_ACCESS_KEY: !!null
+MONGO_S3_BACKUP_AWS_SECRET_KEY: !!null
+

playbooks/roles/mongo/tasks/main.yml

@@ -76,3 +76,39 @@
     state=present
   with_items: MONGO_USERS
   when: mongo_create_users
+
+- name: install s3cmd
+  pip: >
+    name="s3cmd"
+    state=present
+    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+  when: MONGO_S3_BACKUP
+ 
+- name: configure s3cmd
+  template: >
+    dest="{{ MONGO_S3_S3CMD_CONFIG }}"
+    src=mongo-s3-backup-s3cfg.j2
+    owner=root
+    group=root
+    mode=0600
+  when: MONGO_S3_BACKUP
+
+- name: install backup-mongo-to-s3 script
+  template: >
+    src=backup-mongo-to-s3.j2
+    dest=/edx/bin/backup-mongo-to-s3.sh
+    owner=root
+    group=root
+    mode=0700
+  when: MONGO_S3_BACKUP
+
+- name: schedule backup-mongo-to-3s crontab
+  cron: 
+    name="backup-mongo-to-s3"
+    job="/edx/bin/backup-mongo-to-s3.sh"
+    backup=yes
+    cron_file=backup-mongo-to-s3
+    user=root
+    hour="{{ MONGO_S3_BACKUP_HOUR }}"
+    minute="0"
+    day="{{ MONGO_S3_BACKUP_DAY }}"

playbooks/roles/mongo/templates/backup-mongo-to-s3.j2

+{% set lb = '{' %}
+{% set rb = '}' %}
+#!/bin/bash
+#
+
+exec > >(tee "{{ mongo_s3_logfile }}")
+exec 2>&1
+
+shopt -s extglob
+
+usage() {
+
+  cat<<EO
+
+  A script that will run a mongodump of all databases, tar/gz them 
+  and upload to an s3 bucket, will send mail to 
+  {{ MONGO_S3_NOTIFY_EMAIL }} on failures.
+
+  Usage: $PROG
+            -v    add verbosity (set -x)
+            -n    echo what will be done
+            -h    this
+EO
+}
+
+while getopts "vhn" opt; do
+  case $opt in
+    v)
+      set -x
+      shift
+      ;;
+    h)
+      usage
+      exit 0
+      ;;
+    n)
+      noop="echo Would have run: "
+      shift
+      ;;
+  esac
+done
+
+if [[ "{{ MONGO_S3_BACKUP }}" != "true" ]]; then
+  # only run if explicitly enabled
+  exit
+fi
+
+MYNODENAME=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep \"me\" | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//');
+if [[ "$MYNODENAME" != "{{ MONGO_S3_BACKUP_NODE }}" ]]; then
+  # only run on specified node
+  exit
+fi
+  
+
+ISSECONDARY=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep secondary | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//')
+if [[ "$ISSECONDARY" != "true" ]]; then 
+  # backups should be run on secondary server
+  exit; 
+fi
+
+MONGOOUTDIR=$(mktemp -d -p {{ MONGO_S3_BACKUP_TEMPDIR }})
+DATESTAMP=$(date +'%Y-%m-%d-%H%M')
+$noop mongodump --host {{ EDXAPP_MONGO_HOSTS[0] }} -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" -o $MONGOOUTDIR
+cd $MONGOOUTDIR
+$noop tar zcf {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz .
+cd {{ MONGO_S3_BACKUP_TEMPDIR }}
+$noop s3cmd -c {{ MONGO_S3_S3CMD_CONFIG }} sync {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz "s3://{{ MONGO_S3_BACKUP_BUCKET }}/mongo/"
+rm -rf $MONGOOUTDIR {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz

playbooks/roles/mongo/templates/mongo-s3-backup-s3cfg.j2

+[default]
+access_key = {{ MONGO_S3_BACKUP_AWS_ACCESS_KEY }}
+secret_key = {{ MONGO_S3_BACKUP_AWS_SECRET_KEY }}
+bucket_location = US