Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
C
configuration
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
OpenEdx
configuration
Commits
eee0a720
Commit
eee0a720
authored
Sep 12, 2014
by
Fred Smith
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #1425 from edx/derf/mongo_backup_to_s3_role
add mongodb backup script
parents
f47c5acf
329a6a8b
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
127 additions
and
1 deletions
+127
-1
playbooks/roles/mongo/defaults/main.yml
+19
-1
playbooks/roles/mongo/tasks/main.yml
+36
-0
playbooks/roles/mongo/templates/backup-mongo-to-s3.j2
+68
-0
playbooks/roles/mongo/templates/mongo-s3-backup-s3cfg.j2
+4
-0
No files found.
playbooks/roles/mongo/defaults/main.yml
View file @
eee0a720
...
...
@@ -31,7 +31,7 @@ mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
mongo_dbpath
:
"
{{
mongo_data_dir
}}/mongodb"
# Have to use this conditional instead of ignore errors
# because the mongo_user module fails and doesn't
gi
nore errors.
# because the mongo_user module fails and doesn't
ig
nore errors.
mongo_create_users
:
!!null
# If the system is running out of an Amazon Web Services
...
...
@@ -42,3 +42,21 @@ mongo_aws_stack_name: "tag_aws_cloudformation_stack-name_"
# In environments that do not require durability (devstack / Jenkins)
# you can disable the journal to reduce disk usage
mongo_enable_journal
:
True
# We can do regular backups of MongoDB to S3.
MONGO_S3_BACKUP
:
false
# backup cron time:
MONGO_S3_BACKUP_HOUR
:
"
*/12"
MONGO_S3_BACKUP_DAY
:
"
*"
# override with a secondary node that will perform backups
MONGO_S3_BACKUP_NODE
:
"
undefined"
# back up data into a specific S3 bucket
MONGO_S3_BACKUP_BUCKET
:
"
undefined"
# temporary directory mongodump will use to store data
MONGO_S3_BACKUP_TEMPDIR
:
"
{{
mongo_data_dir
}}"
MONGO_S3_NOTIFY_EMAIL
:
"
dummy@example.com"
mongo_s3_logfile
:
"
{{
COMMON_LOG_DIR
}}/mongo/s3-mongo-backup.log"
MONGO_S3_S3CMD_CONFIG
:
"
{{
COMMON_DATA_DIR
}}/mongo-s3-backup.s3cfg"
MONGO_S3_BACKUP_AWS_ACCESS_KEY
:
!!null
MONGO_S3_BACKUP_AWS_SECRET_KEY
:
!!null
playbooks/roles/mongo/tasks/main.yml
View file @
eee0a720
...
...
@@ -76,3 +76,39 @@
state=present
with_items
:
MONGO_USERS
when
:
mongo_create_users
-
name
:
install s3cmd
pip
:
>
name="s3cmd"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
when
:
MONGO_S3_BACKUP
-
name
:
configure s3cmd
template
:
>
dest="{{ MONGO_S3_S3CMD_CONFIG }}"
src=mongo-s3-backup-s3cfg.j2
owner=root
group=root
mode=0600
when
:
MONGO_S3_BACKUP
-
name
:
install backup-mongo-to-s3 script
template
:
>
src=backup-mongo-to-s3.j2
dest=/edx/bin/backup-mongo-to-s3.sh
owner=root
group=root
mode=0700
when
:
MONGO_S3_BACKUP
-
name
:
schedule backup-mongo-to-3s crontab
cron
:
name="backup-mongo-to-s3"
job="/edx/bin/backup-mongo-to-s3.sh"
backup=yes
cron_file=backup-mongo-to-s3
user=root
hour="{{ MONGO_S3_BACKUP_HOUR }}"
minute="0"
day="{{ MONGO_S3_BACKUP_DAY }}"
playbooks/roles/mongo/templates/backup-mongo-to-s3.j2
0 → 100644
View file @
eee0a720
{% set lb = '{' %}
{% set rb = '}' %}
#!/bin/bash
#
exec > >(tee "{{ mongo_s3_logfile }}")
exec 2>&1
shopt -s extglob
usage() {
cat<<EO
A script that will run a mongodump of all databases, tar/gz them
and upload to an s3 bucket, will send mail to
{{ MONGO_S3_NOTIFY_EMAIL }} on failures.
Usage: $PROG
-v add verbosity (set -x)
-n echo what will be done
-h this
EO
}
while getopts "vhn" opt; do
case $opt in
v)
set -x
shift
;;
h)
usage
exit 0
;;
n)
noop="echo Would have run: "
shift
;;
esac
done
if [[ "{{ MONGO_S3_BACKUP }}" != "true" ]]; then
# only run if explicitly enabled
exit
fi
MYNODENAME=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep \"me\" | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//');
if [[ "$MYNODENAME" != "{{ MONGO_S3_BACKUP_NODE }}" ]]; then
# only run on specified node
exit
fi
ISSECONDARY=$(echo "db.isMaster()" | mongo -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" "{{ EDXAPP_MONGO_DB_NAME }}" | grep secondary | cut -f 2 -d ':' | sed -e 's/ //' -e 's/,//' -e 's/"//')
if [[ "$ISSECONDARY" != "true" ]]; then
# backups should be run on secondary server
exit;
fi
MONGOOUTDIR=$(mktemp -d -p {{ MONGO_S3_BACKUP_TEMPDIR }})
DATESTAMP=$(date +'%Y-%m-%d-%H%M')
$noop mongodump --host {{ EDXAPP_MONGO_HOSTS[0] }} -u "{{ COMMON_MONGO_READ_ONLY_USER }}" -p"{{ COMMON_MONGO_READ_ONLY_PASS }}" -o $MONGOOUTDIR
cd $MONGOOUTDIR
$noop tar zcf {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz .
cd {{ MONGO_S3_BACKUP_TEMPDIR }}
$noop s3cmd -c {{ MONGO_S3_S3CMD_CONFIG }} sync {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz "s3://{{ MONGO_S3_BACKUP_BUCKET }}/mongo/"
rm -rf $MONGOOUTDIR {{ MONGO_S3_BACKUP_TEMPDIR }}/{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-$DATESTAMP.tar.gz
playbooks/roles/mongo/templates/mongo-s3-backup-s3cfg.j2
0 → 100644
View file @
eee0a720
[default]
access_key = {{ MONGO_S3_BACKUP_AWS_ACCESS_KEY }}
secret_key = {{ MONGO_S3_BACKUP_AWS_SECRET_KEY }}
bucket_location = US
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment