Commit adae6cf3 by Ned Batchelder

Merge pull request #2772 from edx/ned/merge-dogwood-to-master

Merge dogwood to master
parents 43e2f6f8 5a80d2e4
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
#
# Handlers for role certs
#
# Overview:
#
- name: restart certs
supervisorctl: >
name=certs
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
state=restarted
when: certs_installed is defined and not disable_edx_services
...@@ -5,14 +5,12 @@ ...@@ -5,14 +5,12 @@
src=certs.env.json.j2 src=certs.env.json.j2
dest={{ certs_app_dir }}/env.json dest={{ certs_app_dir }}/env.json
sudo_user: "{{ certs_user }}" sudo_user: "{{ certs_user }}"
notify: restart certs
- name: create certificate auth file - name: create certificate auth file
template: > template: >
src=certs.auth.json.j2 src=certs.auth.json.j2
dest={{ certs_app_dir }}/auth.json dest={{ certs_app_dir }}/auth.json
sudo_user: "{{ certs_user }}" sudo_user: "{{ certs_user }}"
notify: restart certs
- name: writing supervisor script for certificates - name: writing supervisor script for certificates
template: > template: >
...@@ -27,14 +25,12 @@ ...@@ -27,14 +25,12 @@
state=link state=link
force=yes force=yes
mode=0644 mode=0644
notify: restart certs
when: not disable_edx_services when: not disable_edx_services
- name: create ssh script for git - name: create ssh script for git
template: > template: >
src={{ certs_git_ssh|basename }}.j2 dest={{ certs_git_ssh }} src={{ certs_git_ssh|basename }}.j2 dest={{ certs_git_ssh }}
owner={{ certs_user }} mode=750 owner={{ certs_user }} mode=750
notify: restart certs
# This key is only needed if you are pulling down a private # This key is only needed if you are pulling down a private
# certificates repo # certificates repo
...@@ -43,7 +39,6 @@ ...@@ -43,7 +39,6 @@
content="{{ CERTS_GIT_IDENTITY }}" dest={{ certs_git_identity }} content="{{ CERTS_GIT_IDENTITY }}" dest={{ certs_git_identity }}
force=yes owner={{ certs_user }} mode=0600 force=yes owner={{ certs_user }} mode=0600
when: CERTS_GIT_IDENTITY != "none" when: CERTS_GIT_IDENTITY != "none"
notify: restart certs
- name: checkout certificates repo into {{ certs_code_dir }} - name: checkout certificates repo into {{ certs_code_dir }}
git: > git: >
...@@ -54,7 +49,6 @@ ...@@ -54,7 +49,6 @@
GIT_SSH: "{{ certs_git_ssh }}" GIT_SSH: "{{ certs_git_ssh }}"
register: certs_checkout register: certs_checkout
when: CERTS_GIT_IDENTITY != "none" when: CERTS_GIT_IDENTITY != "none"
notify: restart certs
- name: checkout certificates repo into {{ certs_code_dir }} - name: checkout certificates repo into {{ certs_code_dir }}
git: > git: >
...@@ -63,19 +57,16 @@ ...@@ -63,19 +57,16 @@
sudo_user: "{{ certs_user }}" sudo_user: "{{ certs_user }}"
register: certs_checkout register: certs_checkout
when: CERTS_GIT_IDENTITY == "none" when: CERTS_GIT_IDENTITY == "none"
notify: restart certs
- name: remove read-only ssh key for the certs repo - name: remove read-only ssh key for the certs repo
file: path={{ certs_git_identity }} state=absent file: path={{ certs_git_identity }} state=absent
when: CERTS_GIT_IDENTITY != "none" when: CERTS_GIT_IDENTITY != "none"
notify: restart certs
- name : install python requirements - name : install python requirements
pip: > pip: >
requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}" extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
sudo_user: "{{ certs_user }}" sudo_user: "{{ certs_user }}"
notify: restart certs
# call supervisorctl update. this reloads # call supervisorctl update. this reloads
# the supervisorctl config and restarts # the supervisorctl config and restarts
...@@ -110,4 +101,10 @@ ...@@ -110,4 +101,10 @@
- include: tag_ec2.yml tags=deploy - include: tag_ec2.yml tags=deploy
when: COMMON_TAG_EC2_INSTANCE when: COMMON_TAG_EC2_INSTANCE
- set_fact: certs_installed=true - name: restart certs
supervisorctl: >
name=certs
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
state=restarted
when: not disable_edx_services
...@@ -37,7 +37,6 @@ ...@@ -37,7 +37,6 @@
home="{{ certs_app_dir }}" home="{{ certs_app_dir }}"
createhome=no createhome=no
shell=/bin/false shell=/bin/false
notify: restart certs
- name: create certs app dirs - name: create certs app dirs
file: > file: >
...@@ -45,7 +44,6 @@ ...@@ -45,7 +44,6 @@
state=directory state=directory
owner="{{ certs_user }}" owner="{{ certs_user }}"
group="{{ common_web_group }}" group="{{ common_web_group }}"
notify: restart certs
with_items: with_items:
- "{{ certs_app_dir }}" - "{{ certs_app_dir }}"
# needed for the ansible 1.5 git module # needed for the ansible 1.5 git module
...@@ -68,14 +66,12 @@ ...@@ -68,14 +66,12 @@
path="{{ certs_gpg_dir }}" state=directory path="{{ certs_gpg_dir }}" state=directory
owner="{{ common_web_user }}" owner="{{ common_web_user }}"
mode=0700 mode=0700
notify: restart certs
- name: copy the private gpg signing key - name: copy the private gpg signing key
copy: > copy: >
src={{ CERTS_LOCAL_PRIVATE_KEY }} src={{ CERTS_LOCAL_PRIVATE_KEY }}
dest={{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }} dest={{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
owner={{ common_web_user }} mode=0600 owner={{ common_web_user }} mode=0600
notify: restart certs
register: certs_gpg_key register: certs_gpg_key
- name: copy the pgp trust export - name: copy the pgp trust export
...@@ -83,20 +79,17 @@ ...@@ -83,20 +79,17 @@
content="{{ CERTS_OWNER_TRUST }}" content="{{ CERTS_OWNER_TRUST }}"
dest={{ certs_app_dir }}/trust.export dest={{ certs_app_dir }}/trust.export
owner={{ common_web_user }} mode=0600 owner={{ common_web_user }} mode=0600
notify: restart certs
- name: load the gpg key - name: load the gpg key
shell: > shell: >
/usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }} /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
sudo_user: "{{ common_web_user }}" sudo_user: "{{ common_web_user }}"
when: certs_gpg_key.changed when: certs_gpg_key.changed
notify: restart certs
- name: import the trust export - name: import the trust export
shell: > shell: >
/usr/bin/gpg --homedir {{ certs_gpg_dir }} --import-ownertrust {{ certs_app_dir }}/trust.export /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import-ownertrust {{ certs_app_dir }}/trust.export
sudo_user: "{{ common_web_user }}" sudo_user: "{{ common_web_user }}"
when: certs_gpg_key.changed when: certs_gpg_key.changed
notify: restart certs
- include: deploy.yml tags=deploy - include: deploy.yml tags=deploy
- name: restart xqueue
supervisorctl: >
name={{ item }}
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
state=restarted
when: xqueue_installed is defined and not disable_edx_services
with_items:
- xqueue
- xqueue_consumer
...@@ -23,8 +23,6 @@ ...@@ -23,8 +23,6 @@
template: > template: >
src=xqueue_gunicorn.py.j2 dest={{ xqueue_app_dir }}/xqueue_gunicorn.py src=xqueue_gunicorn.py.j2 dest={{ xqueue_app_dir }}/xqueue_gunicorn.py
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
notify:
- restart xqueue
tags: tags:
- install - install
- install:configuration - install:configuration
...@@ -32,8 +30,6 @@ ...@@ -32,8 +30,6 @@
- name: create xqueue application config - name: create xqueue application config
template: src=xqueue.env.json.j2 dest={{ xqueue_app_dir }}/xqueue.env.json mode=0644 template: src=xqueue.env.json.j2 dest={{ xqueue_app_dir }}/xqueue.env.json mode=0644
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
notify:
- restart xqueue
tags: tags:
- install - install
- install:configuration - install:configuration
...@@ -41,8 +37,6 @@ ...@@ -41,8 +37,6 @@
- name: create xqueue auth file - name: create xqueue auth file
template: src=xqueue.auth.json.j2 dest={{ xqueue_app_dir }}/xqueue.auth.json mode=0644 template: src=xqueue.auth.json.j2 dest={{ xqueue_app_dir }}/xqueue.auth.json mode=0644
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
notify:
- restart xqueue
tags: tags:
- install - install
- install:configuration - install:configuration
...@@ -54,8 +48,6 @@ ...@@ -54,8 +48,6 @@
accept_hostkey=yes accept_hostkey=yes
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
register: xqueue_checkout register: xqueue_checkout
notify:
- restart xqueue
tags: tags:
- install - install
- install:code - install:code
...@@ -67,8 +59,6 @@ ...@@ -67,8 +59,6 @@
requirements="{{ xqueue_pre_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present requirements="{{ xqueue_pre_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w" extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
notify:
- restart xqueue
tags: tags:
- install - install
- install:app-requirements - install:app-requirements
...@@ -79,8 +69,6 @@ ...@@ -79,8 +69,6 @@
requirements="{{ xqueue_post_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present requirements="{{ xqueue_post_requirements_file }}" virtualenv="{{ xqueue_venv_dir }}" state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w" extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
notify:
- restart xqueue
tags: tags:
- install - install
- install:app-requirements - install:app-requirements
...@@ -95,8 +83,6 @@ ...@@ -95,8 +83,6 @@
DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}" DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}" DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
when: migrate_db is defined and migrate_db|lower == "yes" and COMMON_MYSQL_MIGRATE_PASS when: migrate_db is defined and migrate_db|lower == "yes" and COMMON_MYSQL_MIGRATE_PASS
notify:
- restart xqueue
tags: tags:
- migrate - migrate
- migrate:db - migrate:db
...@@ -105,8 +91,6 @@ ...@@ -105,8 +91,6 @@
shell: > shell: >
SERVICE_VARIANT=xqueue {{ xqueue_venv_bin }}/django-admin.py update_users --settings=xqueue.aws_settings --pythonpath={{ xqueue_code_dir }} SERVICE_VARIANT=xqueue {{ xqueue_venv_bin }}/django-admin.py update_users --settings=xqueue.aws_settings --pythonpath={{ xqueue_code_dir }}
sudo_user: "{{ xqueue_user }}" sudo_user: "{{ xqueue_user }}"
notify:
- restart xqueue
tags: tags:
- manage - manage
- manage:app-users - manage:app-users
...@@ -148,4 +132,13 @@ ...@@ -148,4 +132,13 @@
- install - install
- install:app-requirements - install:app-requirements
- set_fact: xqueue_installed=true - name: restart xqueue
supervisorctl: >
name={{ item }}
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
state=restarted
when: not disable_edx_services
with_items:
- xqueue
- xqueue_consumer
...@@ -12,8 +12,6 @@ ...@@ -12,8 +12,6 @@
home="{{ xqueue_app_dir }}" home="{{ xqueue_app_dir }}"
createhome=no createhome=no
shell=/bin/false shell=/bin/false
notify:
- restart xqueue
tags: tags:
- install - install
- install:base - install:base
...@@ -24,8 +22,6 @@ ...@@ -24,8 +22,6 @@
state=directory state=directory
owner="{{ xqueue_user }}" owner="{{ xqueue_user }}"
group="{{ common_web_group }}" group="{{ common_web_group }}"
notify:
- restart xqueue
with_items: with_items:
- "{{ xqueue_app_dir }}" - "{{ xqueue_app_dir }}"
- "{{ xqueue_venvs_dir }}" - "{{ xqueue_venvs_dir }}"
...@@ -35,8 +31,6 @@ ...@@ -35,8 +31,6 @@
- name: install a bunch of system packages on which xqueue relies - name: install a bunch of system packages on which xqueue relies
apt: pkg={{','.join(xqueue_debian_pkgs)}} state=present apt: pkg={{','.join(xqueue_debian_pkgs)}} state=present
notify:
- restart xqueue
tags: tags:
- install - install
- install:system-requirements - install:system-requirements
......
...@@ -64,14 +64,8 @@ confirm_proceed () { ...@@ -64,14 +64,8 @@ confirm_proceed () {
} }
# Check we are in the right place, and have the info we need. # Check we are in the right place, and have the info we need.
if [[ "`whoami`" != "vagrant" ]]; then
echo "Run this from the vagrant account in your Open edX machine."
exit 1
fi
if [[ ! -d /edx/app/edxapp ]]; then if [[ ! -d /edx/app/edxapp ]]; then
echo "Run this from the vagrant account in your Open edX machine." echo "Run this on your Open edX machine."
exit 1 exit 1
fi fi
...@@ -90,6 +84,11 @@ if [[ $CONFIGURATION == none ]]; then ...@@ -90,6 +84,11 @@ if [[ $CONFIGURATION == none ]]; then
exit 1 exit 1
fi fi
APPUSER=edxapp
if [[ $CONFIGURATION == fullstack ]] ; then
APPUSER=www-data
fi
# Birch details # Birch details
if [[ $TARGET == *birch* && $INTERACTIVE == true ]] ; then if [[ $TARGET == *birch* && $INTERACTIVE == true ]] ; then
...@@ -173,6 +172,22 @@ make_config_venv ...@@ -173,6 +172,22 @@ make_config_venv
# Dogwood details # Dogwood details
if [[ $TARGET == *dogwood* ]] ; then if [[ $TARGET == *dogwood* ]] ; then
# Run the forum migrations.
cat > migrate-008-context.js <<"EOF"
// from: https://github.com/edx/cs_comments_service/blob/master/scripts/db/migrate-008-context.js
print ("Add the new indexes for the context field");
db.contents.ensureIndex({ _type: 1, course_id: 1, context: 1, pinned: -1, created_at: -1 }, {background: true})
db.contents.ensureIndex({ _type: 1, commentable_id: 1, context: 1, pinned: -1, created_at: -1 }, {background: true})
print ("Adding context to all comment threads where it does not yet exist\n");
var bulk = db.contents.initializeUnorderedBulkOp();
bulk.find( {_type: "CommentThread", context: {$exists: false}} ).update( {$set: {context: "course"}} );
bulk.execute();
printjson (db.runCommand({ getLastError: 1, w: "majority", wtimeout: 5000 } ));
EOF
mongo cs_comments_service migrate-008-context.js
# We are upgrading Python from 2.7.3 to 2.7.10, so remake the venvs. # We are upgrading Python from 2.7.3 to 2.7.10, so remake the venvs.
sudo rm -rf /edx/app/*/v*envs/* sudo rm -rf /edx/app/*/v*envs/*
...@@ -194,7 +209,7 @@ if [[ $TARGET == *dogwood* ]] ; then ...@@ -194,7 +209,7 @@ if [[ $TARGET == *dogwood* ]] ; then
make_config_venv make_config_venv
# Need to get rid of South from edx-platform, or things won't work. # Need to get rid of South from edx-platform, or things won't work.
sudo -u edxapp /edx/app/edxapp/venvs/edxapp/bin/pip uninstall -y South sudo -u edxapp /edx/bin/pip.edxapp uninstall -y South
echo "Upgrading to the beginning of Django 1.8" echo "Upgrading to the beginning of Django 1.8"
cd configuration/playbooks/vagrant cd configuration/playbooks/vagrant
...@@ -211,10 +226,8 @@ if [[ $TARGET == *dogwood* ]] ; then ...@@ -211,10 +226,8 @@ if [[ $TARGET == *dogwood* ]] ; then
echo "Running the Django 1.8 faked migrations" echo "Running the Django 1.8 faked migrations"
for item in lms cms; do for item in lms cms; do
sudo -u edxapp \ sudo -u $APPUSER -E /edx/bin/python.edxapp \
/edx/app/edxapp/venvs/edxapp/bin/python \ /edx/bin/manage.edxapp $item migrate --settings=aws --noinput --fake-initial
/edx/app/edxapp/edx-platform/manage.py $item migrate \
--settings=aws --noinput --fake-initial
done done
if [[ $CONFIGURATION == fullstack ]] ; then if [[ $CONFIGURATION == fullstack ]] ; then
...@@ -226,6 +239,7 @@ if [[ $TARGET == *dogwood* ]] ; then ...@@ -226,6 +239,7 @@ if [[ $TARGET == *dogwood* ]] ; then
fi fi
fi fi
echo "Updating to final version of code"
cd configuration/playbooks cd configuration/playbooks
echo "edx_platform_version: $TARGET" > vars.yml echo "edx_platform_version: $TARGET" > vars.yml
echo "ora2_version: $TARGET" >> vars.yml echo "ora2_version: $TARGET" >> vars.yml
...@@ -238,6 +252,20 @@ sudo ansible-playbook \ ...@@ -238,6 +252,20 @@ sudo ansible-playbook \
--extra-vars="@vars.yml" \ --extra-vars="@vars.yml" \
$SERVER_VARS \ $SERVER_VARS \
vagrant-$CONFIGURATION.yml vagrant-$CONFIGURATION.yml
cd ../..
if [[ $TARGET == *dogwood* ]] ; then
echo "Running data fixup management commands"
sudo -u $APPUSER -E /edx/bin/python.edxapp \
/edx/bin/manage.edxapp lms --settings=aws generate_course_overview --all
sudo -u $APPUSER -E /edx/bin/python.edxapp \
/edx/bin/manage.edxapp lms --settings=aws post_cohort_membership_fix --commit
# Run the forums migrations again to catch things made while this script
# was running.
mongo cs_comments_service migrate-008-context.js
fi
cd / cd /
sudo rm -rf $TEMPDIR sudo rm -rf $TEMPDIR
......
...@@ -63,15 +63,13 @@ openedx_releases = { ...@@ -63,15 +63,13 @@ openedx_releases = {
"named-release/dogwood.rc" => { "named-release/dogwood.rc" => {
:name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
}, },
"named-release/dogwood.rc2" => { "named-release/dogwood" => {
:name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
}, },
"named-release/dogwood.rc3" => { # Cypress is deprecated and unsupported
:name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box", # "named-release/cypress" => {
}, # :name => "cypress-devstack", :file => "cypress-devstack.box",
"named-release/cypress" => { # },
:name => "cypress-devstack", :file => "cypress-devstack.box",
},
# Birch is deprecated and unsupported # Birch is deprecated and unsupported
# "named-release/birch.2" => { # "named-release/birch.2" => {
# :name => "birch-devstack-2", :file => "birch-2-devstack.box", # :name => "birch-devstack-2", :file => "birch-2-devstack.box",
......
...@@ -9,18 +9,16 @@ CPU_COUNT = 2 ...@@ -9,18 +9,16 @@ CPU_COUNT = 2
# to a name and a file path, which are used for retrieving # to a name and a file path, which are used for retrieving
# a Vagrant box from the internet. # a Vagrant box from the internet.
openedx_releases = { openedx_releases = {
"named-release/dogwood.rc" => { "named-release/dogwood" => {
:name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
},
"named-release/dogwood.rc2" => {
:name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box", :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
}, },
"named-release/dogwood.rc3" => { "named-release/dogwood.rc" => {
:name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box", :name => "dogwood-fullstack-rc2", :file => "20151221-dogwood-fullstack-rc2.box",
}, },
"named-release/cypress" => { # Cypress is deprecated and unsupported
:name => "cypress-fullstack", :file => "cypress-fullstack.box", # "named-release/cypress" => {
}, # :name => "cypress-fullstack", :file => "cypress-fullstack.box",
# },
# Birch is deprecated and unsupported # Birch is deprecated and unsupported
# "named-release/birch.2" => { # "named-release/birch.2" => {
# :name => "birch-fullstack-2", :file => "birch-2-fullstack.box", # :name => "birch-fullstack-2", :file => "birch-2-fullstack.box",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment