Skip to content

C
configuration
Commit 1cc75f50 by Jason Bau
Options

Merge pull request #91 from edx/jarv/fab-updates 

Jarv/fab updates
parents 3a22b4fa c8204751
Showing with 126 additions and 123 deletions
fabfile/git.py
......@@ -4,7 +4,6 @@ import re
import socket
from functools import partial
from dogapi import dog_stats_api
from fabric.api import task, sudo, runs_once, execute
from fabric.api import cd, env, abort, parallel, prefix
from fabric.colors import white, green, red
......@@ -222,11 +221,9 @@ def deploy(auto_migrate=False):
metric_name = 'fabric.deployment'
# pre checkout commands
with dog_stats_api.timer(metric_name, tags=package_tags +
['step:pre_commands']):
with prefix("export GIT_SSH=/tmp/git.sh"):
for cmd in env.pre_post['pre']:
noopable(sudo)(cmd)
with prefix("export GIT_SSH=/tmp/git.sh"):
for cmd in env.pre_post['pre']:
noopable(sudo)(cmd)
put(os.path.join(os.path.dirname(__file__), 'git.sh'),
'/tmp/git.sh', mode=0755, use_sudo=True)
......@@ -238,52 +235,44 @@ def deploy(auto_migrate=False):
'existance:' + 'existing' if existing_repo else 'absent',
]
with dog_stats_api.timer(metric_name, tags=repo_tags + ['step:clone']):
if existing_repo:
if not files.exists(os.path.join(pkg.repo_root, '.git'),
use_sudo=True):
raise Exception("Repo root not a git repo - {0}".format(
os.path.join(pkg.repo_root, '.git')))
with cd(pkg.repo_root):
if pkg.revision == 'absent':
noopable(sudo)('rm -rf {0}'.format(pkg.repo_root))
else:
checkout(pkg.revision)
else:
with cd(os.path.dirname(pkg.repo_root)):
if pkg.revision != 'absent':
clone(pkg.repo_org, pkg.repo_name, pkg.name, pkg.revision)
if '~' in pkg.name:
_update_course_xml(pkg, pkg.name.split('~')[1])
with dog_stats_api.timer(metric_name, tags=repo_tags +
['step:requirements']):
_install_requirements(pkg)
_install_gemfile(pkg)
_install_npm_package(pkg)
# with dog_stats_api.timer(metric_name, tags=repo_tags + ['step:fact']):
# # drop a file for puppet so it knows that
# # code is installed for the service
# with cd('/etc/facter/facts.d'):
# pkg_config = PackageInfo()
# if pkg.repo_name in pkg_config.service_repos:
# # facts can't have dashes so they are converted
# # to underscores
# noopable(sudo)(
# 'echo "{0}_installed=true" > {0}_installed.txt'.format(
# pkg.repo_name.replace("-", "_")))
# with dog_stats_api.timer(metric_name, tags=package_tags +
# ['step:pkg_version']):
# pkg_version()
with dog_stats_api.timer(metric_name, tags=package_tags +
['step:post_commands']):
# post checkout commands
with prefix("export GIT_SSH=/tmp/git.sh"):
for cmd in env.pre_post['post']:
noopable(sudo)(cmd)
if existing_repo:
if not files.exists(os.path.join(pkg.repo_root, '.git'),
use_sudo=True):
raise Exception("Repo root not a git repo - {0}".format(
os.path.join(pkg.repo_root, '.git')))
with cd(pkg.repo_root):
if pkg.revision == 'absent':
noopable(sudo)('rm -rf {0}'.format(pkg.repo_root))
else:
checkout(pkg.revision)
else:
with cd(os.path.dirname(pkg.repo_root)):
if pkg.revision != 'absent':
clone(pkg.repo_org, pkg.repo_name, pkg.name, pkg.revision)
if '~' in pkg.name:
_update_course_xml(pkg, pkg.name.split('~')[1])
_install_requirements(pkg)
_install_gemfile(pkg)
_install_npm_package(pkg)
# drop a file for puppet so it knows that
# code is installed for the service
# with cd('/etc/facter/facts.d'):
# pkg_config = PackageInfo()
# if pkg.repo_name in pkg_config.service_repos:
# # facts can't have dashes so they are converted
# # to underscores
# noopable(sudo)(
# 'echo "{0}_installed=true" > {0}_installed.txt'.format(
# pkg.repo_name.replace("-", "_")))
pkg_version()
# post checkout commands
with prefix("export GIT_SSH=/tmp/git.sh"):
for cmd in env.pre_post['post']:
noopable(sudo)(cmd)
if 'mitx' in [pkg.name for pkg in packages]:
# do not slow down content deploys by checking
......@@ -384,15 +373,6 @@ def _install_requirements(pkg):
# suspended
sudo('{0} {1}'.format(AA_COMPLAIN, AA_SANDBOX_POLICY))
# Run old-style requirements TODO: remove
_run_if_changed(pkg, 'pre-requirements.txt', partial(
pip_install, file='pre-requirements.txt', venv='/opt/edx'),
'cat *requirements.txt')
_run_if_changed(pkg, 'requirements.txt', partial(
pip_install, file='requirements.txt', venv='/opt/edx'),
'cat *requirements.txt')
# end old-style requirements
# Run new-style requirements
for venv in VIRTUAL_ENVS:
if not files.exists(venv):
......@@ -409,7 +389,6 @@ def _install_requirements(pkg):
if files.exists(AA_ENFORCE) and files.exists(AA_SANDBOX_POLICY):
sudo('{0} {1}'.format(AA_ENFORCE, AA_SANDBOX_POLICY))
@task
@runs_once
def deploy_with_puppet():
......
package_data.yaml
# datadog integration configuration
datadog_api: ''
# repo_dirs : Exhaustive list of repo-dirs and their corresponding repo-names
......@@ -21,12 +20,16 @@ repo_dirs:
/opt/wwc/grading-controller: edx/grading-controller
/opt/wwc/ease: edx/ease
/opt/wwc/edx-platform: edx/edx-platform
/opt/wwc/data/cs50: MITx/cs50
/opt/wwc/data/edx4edx: MITx/edx4edx
/opt/wwc/data/700x: MITx/700x
/opt/wwc/xqueue: edx/xqueue
/opt/wwc/xserver: edx/xserver
/opt/wwc/certificates: edx/certificates
/opt/wwc/drupal: MITx/drupal
/opt/wwc/latex2edx: edx/latex2edx_xserver
/opt/wwc/worker: edx/edx-platform
/opt/sysadmin: MITx/sysadmin
# {pre,post}_checkout_regex:
# What actions to take before and after a repo checkout.
# Commands will be grouped together if multiple matches are made
......@@ -134,76 +137,97 @@ post_checkout_regex: !!omap
- ^edx-platform$|^content-.*$:
- |
# Requirements files have been moved, so check the location first
if [[ -e /opt/wwc/edx-platform/local-requirements.txt ]]; then
#install local requirements for the platform
if [[ -e /opt/wwc/edx-platform/requirements/edx/local.txt ]]; then
cd /opt/wwc/edx-platform
sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps -r \
/opt/wwc/edx-platform/local-requirements.txt
fi
# Then check the new location
if [[ -e /opt/wwc/edx-platform/requirements/local.txt ]]; then
cd /opt/wwc/edx-platform
sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps -r \
/opt/wwc/edx-platform/requirements/local.txt
sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps --ignore-installed \
--exists-action w -r /opt/wwc/edx-platform/requirements/edx/local.txt
fi
#install local requirements for the sandbox
if [[ -e /etc/apparmor.d/code.sandbox ]]; then
sudo /usr/sbin/aa-complain /etc/apparmor.d/code.sandbox
if [[ -e /opt/wwc/edx-platform/requirements/edx-sandbox/local.txt ]]; then
cd /opt/wwc/edx-platform
sudo -E /opt/edx-sandbox/bin/pip install -q --upgrade --no-deps --ignore-installed \
--exists-action w -r /opt/wwc/edx-platform/requirements/edx-sandbox/local.txt
fi
sudo /usr/sbin/aa-enforce /etc/apparmor.d/code.sandbox
fi
- ^edx-platform$:
- |
# On servers running the workers, which run from the same cloned
# repo, we don't want to run collect static. This could cause
# problems because the cache's are shared with the edx-platform instances
# it's also simply not necessary for the workers as they have no
# view component.
edx_workers_status=$(service edx-workers status 2>/dev/null || true)
# git.py already handle prerequisites, so don't let rake try to install them as well
export NO_PREREQ_INSTALL=1
# This is not a developer workspace, so we don't want to do the workspace migrations
# Instead, we rely on having a clean checkout every time
export SKIP_WS_MIGRATIONS=1
if [[ -z $edx_workers_status ]]; then
export NO_PREREQ_INSTALL=1
# This is not a developer workspace, so we don't want to do the workspace migrations
# Instead, we rely on having a clean checkout every time
export SKIP_WS_MIGRATIONS=1
export RBENV_ROOT=/opt/www/.rbenv
# Use rbenv binaries
export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
# Use binaries installed in rbenv
export PATH=/opt/wwc/edx-platform/bin:$PATH
# Use binaries installed in virtualenv
export PATH=/opt/edx/bin:$PATH
eval "$(rbenv init -)"
# {{}} is used here because of python .format upstream
LMS_VARIANT="$(ls -1 /opt/wwc/lms*.env.json | tail -1 | xargs -i basename {{}} .env.json)"
export RUN="sudo -E -u www-data env SERVICE_VARIANT=$LMS_VARIANT PATH=$PATH"
cd /opt/wwc/edx-platform
export RBENV_ROOT=/opt/www/.rbenv
# Use rbenv binaries
export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
# Use binaries installed in rbenv
export PATH=/opt/wwc/edx-platform/bin:$PATH
# Use binaries installed in virtualenv
export PATH=/opt/edx/bin:$PATH
eval "$(rbenv init -)"
# {{}} is used here because of python .format upstream
LMS_VARIANT="$(ls -1 /opt/wwc/lms*.env.json | tail -1 | xargs -i basename {{}} .env.json)"
export RUN="sudo -E -u www-data env SERVICE_VARIANT=$LMS_VARIANT PATH=$PATH"
cd /opt/wwc/edx-platform
# If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic)
if $(rake -T | grep --quiet gather_assets); then
$RUN rake lms:gather_assets:aws
# Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod]
elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws &>/dev/null); then
$RUN django-admin.py collectstatic \
--pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws \
--noinput --verbosity=0
# If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic)
if $(rake -T | grep --quiet gather_assets); then
$RUN rake lms:gather_assets:aws
# Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod]
elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws &>/dev/null); then
$RUN django-admin.py collectstatic \
--pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws \
--noinput --verbosity=0
fi
fi
- |
export NO_PREREQ_INSTALL=1
export SKIP_WS_MIGRATIONS=1
export RBENV_ROOT=/opt/www/.rbenv
# Use rbenv binaries
export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
# Use binaries installed in rbenv
export PATH=/opt/wwc/edx-platform/bin:$PATH
# Use binaries installed in virtualenv
export PATH=/opt/edx/bin:$PATH
eval "$(rbenv init -)"
export RUN="sudo -E -u www-data env SERVICE_VARIANT=cms PATH=$PATH"
cd /opt/wwc/edx-platform
# If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic)
if $(rake -T | grep --quiet gather_assets); then
$RUN rake cms:gather_assets:aws
# Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod]
elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then
$RUN django-admin.py collectstatic \
--pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws \
--noinput --verbosity=0
fi
if $($RUN django-admin.py help update_templates --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then
$RUN django-admin.py update_templates \
--pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws
# On servers running the workers, which run from the same cloned
# repo, we don't want to run collect static. This could cause
# problems because the cache's are shared with the edx-platform instances
# it's also simply not necessary for the workers as they have no
# view component.
edx_workers_status=$(service edx-workers status 2>/dev/null || true)
# git.py already handle prerequisites, so don't let rake try to install them as well
if [[ -z $edx_workers_status ]]; then
export NO_PREREQ_INSTALL=1
export SKIP_WS_MIGRATIONS=1
export RBENV_ROOT=/opt/www/.rbenv
# Use rbenv binaries
export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
# Use binaries installed in rbenv
export PATH=/opt/wwc/edx-platform/bin:$PATH
# Use binaries installed in virtualenv
export PATH=/opt/edx/bin:$PATH
eval "$(rbenv init -)"
export RUN="sudo -E -u www-data env SERVICE_VARIANT=cms PATH=$PATH"
cd /opt/wwc/edx-platform
# If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic)
if $(rake -T | grep --quiet gather_assets); then
$RUN rake cms:gather_assets:aws
# Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod]
elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then
$RUN django-admin.py collectstatic \
--pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws \
--noinput --verbosity=0
fi
if $($RUN django-admin.py help update_templates --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then
$RUN django-admin.py update_templates \
--pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws
fi
fi
- ^edx-platform$|^content-.*$:
- |
edxapp_status=$(service edxapp status 2>/dev/null || true)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment