# datadog integration configuration datadog_api: '' # repo_dirs : Exhaustive list of repo-dirs and their corresponding repo-names # the repos are specified by basename(repo_dir), therefore basename(repo_dir) # under repo_dirs must be unique. # The following repos have upstart services with the same name service_repos: - edx-platform - grading-controller - xqueue - xserver - certificates - latex2edx - analytics-experiments repo_dirs: /opt/wwc/analytics-experiments: MITx/analytics-experiments /opt/wwc/grading-controller: edx/grading-controller /opt/wwc/ease: edx/ease /opt/wwc/edx-platform: edx/edx-platform /opt/wwc/data/cs50: MITx/cs50 /opt/wwc/data/edx4edx: MITx/edx4edx /opt/wwc/data/700x: MITx/700x /opt/wwc/xqueue: edx/xqueue /opt/wwc/xserver: edx/xserver /opt/wwc/certificates: edx/certificates /opt/wwc/drupal: MITx/drupal /opt/wwc/latex2edx: edx/latex2edx_xserver /opt/wwc/worker: edx/edx-platform /opt/sysadmin: MITx/sysadmin # {pre,post}_checkout_regex: # What actions to take before and after a repo checkout. # Commands will be grouped together if multiple matches are made # with the duplicates removed. # # For example a deploy of 'edx-platform' and 'grading-controller' will execute # all pre_checkout_regex commands before the checkouts and then # execute post_checkout_regex commands after. # # Commmands will be displayed to the user during the confirm step. # # pre_checkout_regex: !!omap - ^edx-platform$|^content-.*$: - | edxapp_status=$(service edxapp status 2>/dev/null || true) edx_workers_status=$(service edx-workers status 2>/dev/null || true) if [[ -n $edxapp_status ]]; then if [[ $edxapp_status == *running* ]]; then service edxapp stop; fi fi if [[ -n $edx_workers_status ]]; then if [[ $edx_workers_status == *running* ]]; then service edx-workers stop; fi fi - ^grading-controller$|^ease$: - if [[ $(service grader status) == *running* ]]; then service grader stop; fi - if [[ $(service pull_from_xqueue status) == *running* ]]; then service pull_from_xqueue stop; fi - if [[ $(service ml_grader status) == *running* ]]; then service ml_grader stop; fi - if [[ $(service expire_old status) == *running* ]]; then service expire_old stop; fi - if [[ $(service ml_creator status) == *running* ]]; then service ml_creator stop; fi - ^xqueue$: - if [[ $(service xqueue status) == *running* ]]; then service xqueue stop; fi - if [[ $(service xqueue_consumer status) == *running* ]]; then service xqueue_consumer stop; fi - ^certificates$: - if [[ $(service certificates status) == *running* ]]; then service certificates stop; fi - ^xserver|^content-mit-600x.*$: - if [[ $(service xserver status) == *running* ]]; then service xserver stop; fi - ^latex2edx$: - if [[ $(service latex2edx status) == *running* ]]; then service latex2edx stop; fi - ^analytics-experiments$: - if [[ $(service analytics status) == *running* ]]; then service analytics stop; fi post_checkout_regex: !!omap - ^sandbox$|^staging$|^production$|^hiera$: - chown -R puppet:puppet /etc/puppet - ^(?P<repo>content-.*)$: - | COLLECT_STATIC_PKG_DIR=/opt/wwc/staticfiles/{repo} COURSE_STATIC_PKG_DIR=/opt/wwc/course_static/{repo} SOURCE_PKG_DIR=/opt/wwc/data/{repo} # Move from course static data in the collectstatic root # to being symlinked in a separate directory if [[ -e $COLLECT_STATIC_PKG_DIR ]]; then rm -rf "$COLLECT_STATIC_PKG_DIR" fi # If a course adds or removes a static subdirectory, # we want to adjust the symlink if [[ -e $COURSE_STATIC_PKG_DIR ]]; then rm -rf "$COURSE_STATIC_PKG_DIR" fi if [[ -h $COURSE_STATIC_PKG_DIR ]]; then rm "$COURSE_STATIC_PKG_DIR" fi if [[ -d $SOURCE_PKG_DIR/static ]]; then ln -s $SOURCE_PKG_DIR/static $COURSE_STATIC_PKG_DIR else ln -s $SOURCE_PKG_DIR $COURSE_STATIC_PKG_DIR fi - ^edx-platform$|^content-.*$: - | #install local requirements for the platform if [[ -e /opt/wwc/edx-platform/requirements/edx/local.txt ]]; then cd /opt/wwc/edx-platform sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps --ignore-installed \ --exists-action w -r /opt/wwc/edx-platform/requirements/edx/local.txt fi #install local requirements for the sandbox if [[ -e /etc/apparmor.d/code.sandbox ]]; then sudo /usr/sbin/aa-complain /etc/apparmor.d/code.sandbox if [[ -e /opt/wwc/edx-platform/requirements/edx-sandbox/local.txt ]]; then cd /opt/wwc/edx-platform sudo -E /opt/edx-sandbox/bin/pip install -q --upgrade --no-deps --ignore-installed \ --exists-action w -r /opt/wwc/edx-platform/requirements/edx-sandbox/local.txt fi sudo /usr/sbin/aa-enforce /etc/apparmor.d/code.sandbox fi - ^edx-platform$: - | # On servers running the workers, which run from the same cloned # repo, we don't want to run collect static. This could cause # problems because the cache's are shared with the edx-platform instances # it's also simply not necessary for the workers as they have no # view component. edx_workers_status=$(service edx-workers status 2>/dev/null || true) # git.py already handle prerequisites, so don't let rake try to install them as well if [[ -z $edx_workers_status ]]; then export NO_PREREQ_INSTALL=1 # This is not a developer workspace, so we don't want to do the workspace migrations # Instead, we rely on having a clean checkout every time export SKIP_WS_MIGRATIONS=1 export RBENV_ROOT=/opt/www/.rbenv # Use rbenv binaries export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH # Use binaries installed in rbenv export PATH=/opt/wwc/edx-platform/bin:$PATH # Use binaries installed in virtualenv export PATH=/opt/edx/bin:$PATH eval "$(rbenv init -)" # {{}} is used here because of python .format upstream LMS_VARIANT="$(ls -1 /opt/wwc/lms*.env.json | tail -1 | xargs -i basename {{}} .env.json)" export RUN="sudo -E -u www-data env SERVICE_VARIANT=$LMS_VARIANT PATH=$PATH" cd /opt/wwc/edx-platform # If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic) if $(rake -T | grep --quiet gather_assets); then $RUN rake lms:gather_assets:aws # Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod] elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws &>/dev/null); then $RUN django-admin.py collectstatic \ --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws \ --noinput --verbosity=0 fi fi - | # On servers running the workers, which run from the same cloned # repo, we don't want to run collect static. This could cause # problems because the cache's are shared with the edx-platform instances # it's also simply not necessary for the workers as they have no # view component. edx_workers_status=$(service edx-workers status 2>/dev/null || true) # git.py already handle prerequisites, so don't let rake try to install them as well if [[ -z $edx_workers_status ]]; then export NO_PREREQ_INSTALL=1 export SKIP_WS_MIGRATIONS=1 export RBENV_ROOT=/opt/www/.rbenv # Use rbenv binaries export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH # Use binaries installed in rbenv export PATH=/opt/wwc/edx-platform/bin:$PATH # Use binaries installed in virtualenv export PATH=/opt/edx/bin:$PATH eval "$(rbenv init -)" export RUN="sudo -E -u www-data env SERVICE_VARIANT=cms PATH=$PATH" cd /opt/wwc/edx-platform # If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic) if $(rake -T | grep --quiet gather_assets); then $RUN rake cms:gather_assets:aws # Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod] elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then $RUN django-admin.py collectstatic \ --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws \ --noinput --verbosity=0 fi if $($RUN django-admin.py help update_templates --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then $RUN django-admin.py update_templates \ --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws fi fi - ^edx-platform$|^content-.*$: - | edxapp_status=$(service edxapp status 2>/dev/null || true) edx_workers_status=$(service edx-workers status 2>/dev/null || true) if [[ -n $edxapp_status ]]; then if [[ $edxapp_status == *stop* ]]; then service edxapp start; fi fi if [[ -n $edx_workers_status ]]; then if [[ $edx_workers_status == *stop* ]]; then service edx-workers start; fi fi - ^grading-controller$|^ease$: - | if [[ ! -d /usr/share/nltk_data ]];then source /opt/edx/bin/activate && python -m nltk.downloader -d /usr/share/nltk_data all; fi - | if [[ -d /opt/wwc/.rbenv ]]; then export RBENV_ROOT=/opt/wwc/.rbenv export PATH=/opt/wwc/edx-platform/bin:$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH eval "$(rbenv init -)" fi if $(sudo -E -u www-data /opt/edx/bin/django-admin.py help collectstatic --pythonpath=/opt/wwc/grading-controller --settings=grading_controller.aws &>/dev/null); then cd /opt/wwc/staticfiles sudo -E -u www-data /opt/edx/bin/django-admin.py collectstatic \ --pythonpath=/opt/wwc/grading-controller --settings=grading_controller.aws \ --noinput --verbosity=0 fi - | if [[ $(service grader status) == *stop* ]]; then service grader start; fi - | if [[ $(service expire_old status) == *stop* ]]; then service expire_old start; fi - | if [[ $(service ml_grader status) == *stop* ]]; then service ml_grader start; fi - | if [[ $(service ml_creator status) == *stop* ]]; then service ml_creator start; fi - | if [[ $(service pull_from_xqueue status) == *stop* ]]; then service pull_from_xqueue start; fi - ^xqueue$: - | if [[ $(service xqueue status) == *stop* ]];then service xqueue start; fi - | if [[ $(service xqueue_consumer status) == *stop* ]];then service xqueue_consumer start; fi - ^xserver|^content-mit-600x.*$: - | if [[ $(service xserver status) == *stop* ]]; then service xserver start; fi - ^certificates$: - | if [[ $(service certificates status) == *stop* ]];then service certificates start; fi - ^latex2edx$: - | if [[ $(service latex2edx status) == *stop* ]];then service latex2edx start; fi - ^analytics-experiments$: - | if [[ $(service analytics status) == *stop* ]];then service analytics start; fi - ^drupal$: - | if [[ -r /opt/wwc/settings.php ]]; then cp /opt/wwc/settings.php /opt/wwc/drupal/sites/default/settings.php; fi; chown -R root:www-data /opt/wwc/drupal && find /opt/wwc/drupal -type d -exec chmod u=rwx,g=rx,o= '{}' \; && find /opt/wwc/drupal -type f -exec chmod u=rw,g=r,o= '{}' \; && find /opt/wwc/drupal/sites -type d -name files -exec chmod ug=rwx,o= '{}' \; && for x in /opt/wwc/drupal/sites/*/files; do find ${x} -type d -exec chmod ug=rwx,o= '{}' \; && find ${x} -type f -exec chmod ug=rw,o= '{}' \;; done; chmod g+w /opt/wwc/drupal/sites/default/tmp && if [[ -r /opt/wwc/drupal/db/edx.sql ]]; then cd /opt/wwc/drupal && `drush sql-connect` < db/edx.sql; fi