Commit 8a80b541 by Kevin Falcone

Merge pull request #2801 from edx/jibsheet/migrate-multiple-databases

Jibsheet/migrate multiple databases
parents 452308b1 3fbdd6e1
...@@ -4,14 +4,32 @@ ...@@ -4,14 +4,32 @@
gather_facts: False gather_facts: False
vars: vars:
db_dry_run: "--list" db_dry_run: "--list"
roles:
- edxapp
tasks: tasks:
- name: migrate - name: migrate lms
shell: > shell: >
chdir={{ edxapp_code_dir }} chdir={{ edxapp_code_dir }}
python manage.py {{ item }} migrate --noinput {{ db_dry_run }} --settings=aws_migrate python manage.py lms migrate --database {{ item }} --noinput {{ db_dry_run }} --settings=aws_migrate
environment: environment:
DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}" DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}" DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
# Migrate any database in the config, but not the read_replica
when: item != 'read_replica'
with_items: with_items:
- lms - "{{ lms_auth_config.DATABASES.keys() }}"
- cms tags:
- always
- name: migrate cms
shell: >
chdir={{ edxapp_code_dir }}
python manage.py cms migrate --database {{ item }} --noinput {{ db_dry_run }} --settings=aws_migrate
environment:
DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
# Migrate any database in the config, but not the read_replica
when: item != 'read_replica'
with_items:
- "{{ cms_auth_config.DATABASES.keys() }}"
tags:
- always
...@@ -662,6 +662,9 @@ edxapp_generic_auth_config: &edxapp_generic_auth ...@@ -662,6 +662,9 @@ edxapp_generic_auth_config: &edxapp_generic_auth
ADDITIONAL_OPTIONS: "{{ EDXAPP_CONTENTSTORE_ADDITIONAL_OPTS }}" ADDITIONAL_OPTIONS: "{{ EDXAPP_CONTENTSTORE_ADDITIONAL_OPTS }}"
DOC_STORE_CONFIG: *edxapp_generic_default_docstore DOC_STORE_CONFIG: *edxapp_generic_default_docstore
DATABASES: DATABASES:
# edxapp's edxapp-migrate scripts and the edxapp_migrate play
# will ensure that any DB not named read_replica will be migrated
# for both the lms and cms.
read_replica: read_replica:
ENGINE: 'django.db.backends.mysql' ENGINE: 'django.db.backends.mysql'
NAME: "{{ EDXAPP_MYSQL_REPLICA_DB_NAME }}" NAME: "{{ EDXAPP_MYSQL_REPLICA_DB_NAME }}"
......
{% include "edxapp_common.j2" %} {% include "edxapp_common.j2" %}
sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin}}/python manage.py cms migrate --noinput --settings $EDX_PLATFORM_SETTINGS if [[ -z "$NO_EDXAPP_SUDO" ]]; then
SUDO='sudo -E -u {{ edxapp_user }} env "PATH=$PATH"'
fi
{% for db in cms_auth_config.DATABASES.keys() %}
{%- if db != 'read_replica' %}
$SUDO {{ edxapp_venv_bin}}/python manage.py cms migrate --database {{ db }} --noinput --settings $EDX_PLATFORM_SETTINGS $@
{% endif %}
{% endfor %}
{% include "edxapp_common.j2" %} {% include "edxapp_common.j2" %}
sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin}}/python manage.py lms migrate --noinput --settings $EDX_PLATFORM_SETTINGS if [[ -z "$NO_EDXAPP_SUDO" ]]; then
SUDO='sudo -E -u {{ edxapp_user }} env "PATH=$PATH"'
fi
{% for db in lms_auth_config.DATABASES.keys() %}
{%- if db != 'read_replica' %}
$SUDO {{ edxapp_venv_bin}}/python manage.py lms migrate --database {{ db }} --noinput --settings $EDX_PLATFORM_SETTINGS $@
{% endif %}
{% endfor %}
...@@ -11,8 +11,8 @@ import time ...@@ -11,8 +11,8 @@ import time
# Services that should be checked for migrations. # Services that should be checked for migrations.
MIGRATION_COMMANDS = { MIGRATION_COMMANDS = {
'lms': ". {env_file}; {python} {code_dir}/manage.py lms migrate --noinput --list --settings=aws", 'lms': "NO_EDXAPP_SUDO=1 /edx/bin/edxapp-migrate-lms --noinput --list",
'cms': ". {env_file}; {python} {code_dir}/manage.py cms migrate --noinput --list --settings=aws", 'cms': "NO_EDXAPP_SUDO=1 /edx/bin/edxapp-migrate-cms --noinput --list",
'xqueue': "{python} {code_dir}/manage.py xqueue migrate --noinput --settings=aws --db-dry-run --merge", 'xqueue': "{python} {code_dir}/manage.py xqueue migrate --noinput --settings=aws --db-dry-run --merge",
'ecommerce': ". {env_file}; {python} {code_dir}/manage.py migrate --noinput --list", 'ecommerce': ". {env_file}; {python} {code_dir}/manage.py migrate --noinput --list",
'programs': ". {env_file}; {python} {code_dir}/manage.py migrate --noinput --list", 'programs': ". {env_file}; {python} {code_dir}/manage.py migrate --noinput --list",
...@@ -217,7 +217,7 @@ if __name__ == '__main__': ...@@ -217,7 +217,7 @@ if __name__ == '__main__':
if 'Migrating' in output: if 'Migrating' in output:
raise Exception("Migrations have not been run for {}".format(service)) raise Exception("Migrations have not been run for {}".format(service))
else: else:
new_services = { services = {
"lms": {'python': args.edxapp_python, 'env_file': args.edxapp_env, 'code_dir': args.edxapp_code_dir}, "lms": {'python': args.edxapp_python, 'env_file': args.edxapp_env, 'code_dir': args.edxapp_code_dir},
"cms": {'python': args.edxapp_python, 'env_file': args.edxapp_env, 'code_dir': args.edxapp_code_dir}, "cms": {'python': args.edxapp_python, 'env_file': args.edxapp_env, 'code_dir': args.edxapp_code_dir},
"ecommerce": {'python': args.ecommerce_python, 'env_file': args.ecommerce_env, 'code_dir': args.ecommerce_code_dir}, "ecommerce": {'python': args.ecommerce_python, 'env_file': args.ecommerce_env, 'code_dir': args.ecommerce_code_dir},
...@@ -226,8 +226,8 @@ if __name__ == '__main__': ...@@ -226,8 +226,8 @@ if __name__ == '__main__':
"analytics_api": {'python': args.analytics_api_python, 'env_file': args.analytics_api_env, 'code_dir': args.analytics_api_code_dir} "analytics_api": {'python': args.analytics_api_python, 'env_file': args.analytics_api_env, 'code_dir': args.analytics_api_code_dir}
} }
if service in new_services and all(arg!=None for arg in new_services[service].values()) and service in MIGRATION_COMMANDS: if service in services and all(arg!=None for arg in services[service].values()) and service in MIGRATION_COMMANDS:
serv_vars = new_services[service] serv_vars = services[service]
cmd = MIGRATION_COMMANDS[service].format(**serv_vars) cmd = MIGRATION_COMMANDS[service].format(**serv_vars)
if os.path.exists(serv_vars['code_dir']): if os.path.exists(serv_vars['code_dir']):
......
...@@ -58,13 +58,12 @@ extra_var_args+=" -e edxapp_code_dir=${WORKSPACE}/edx-platform" ...@@ -58,13 +58,12 @@ extra_var_args+=" -e edxapp_code_dir=${WORKSPACE}/edx-platform"
extra_var_args+=" -e edxapp_user=jenkins" extra_var_args+=" -e edxapp_user=jenkins"
extra_var_args+=" -e EDXAPP_CFG_DIR=${WORKSPACE}" extra_var_args+=" -e EDXAPP_CFG_DIR=${WORKSPACE}"
# Generate the json configuration files
ansible-playbook -c local $extra_var_args --tags edxapp_cfg -i localhost, -s -U jenkins edxapp.yml
# Run migrations and replace literal '\n' with actual newlines to make the output # Run migrations and replace literal '\n' with actual newlines to make the output
# easier to read # easier to read
# We use the edxapp_cfg tag so that the edxapp role will generate config files but
# nothing else. The actual migrate commands are then run from the playbook.
ansible-playbook -v -c local $extra_var_args -i localhost, -s -U jenkins edxapp_migrate.yml | sed 's/\\n/\n/g' ansible-playbook -v -c local $extra_var_args --tags edxapp_cfg -i localhost, -s -U jenkins edxapp_migrate.yml | sed 's/\\n/\n/g'
#We don't care about the exit status of the `sed` #We don't care about the exit status of the `sed`
exit ${PIPESTATUS[0]} exit ${PIPESTATUS[0]}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment