Merge pull request #91 from edx/jarv/fab-updates

Jarv/fab updates

fabfile/git.py

@@ -4,7 +4,6 @@ import re
 import socket
 
 from functools import partial
-from dogapi import dog_stats_api
 from fabric.api import task, sudo, runs_once, execute
 from fabric.api import cd, env, abort, parallel, prefix
 from fabric.colors import white, green, red
@@ -222,8 +221,6 @@ def deploy(auto_migrate=False):
     metric_name = 'fabric.deployment'
 
     # pre checkout commands
-    with dog_stats_api.timer(metric_name, tags=package_tags +
-                             ['step:pre_commands']):
     with prefix("export GIT_SSH=/tmp/git.sh"):
         for cmd in env.pre_post['pre']:
             noopable(sudo)(cmd)
@@ -238,7 +235,6 @@ def deploy(auto_migrate=False):
             'existance:' + 'existing' if existing_repo else 'absent',
         ]
 
-        with dog_stats_api.timer(metric_name, tags=repo_tags + ['step:clone']):
         if existing_repo:
             if not files.exists(os.path.join(pkg.repo_root, '.git'),
                                 use_sudo=True):
@@ -256,15 +252,12 @@ def deploy(auto_migrate=False):
         if '~' in pkg.name:
             _update_course_xml(pkg, pkg.name.split('~')[1])
 
-        with dog_stats_api.timer(metric_name, tags=repo_tags +
-                                 ['step:requirements']):
         _install_requirements(pkg)
         _install_gemfile(pkg)
         _install_npm_package(pkg)
 
-#        with dog_stats_api.timer(metric_name, tags=repo_tags + ['step:fact']):
-#            # drop a file for puppet so it knows that
-#            # code is installed for the service
+        # drop a file for puppet so it knows that
+        # code is installed for the service
 #        with cd('/etc/facter/facts.d'):
 #            pkg_config = PackageInfo()
 #            if pkg.repo_name in pkg_config.service_repos:
@@ -274,12 +267,8 @@ def deploy(auto_migrate=False):
 #                    'echo "{0}_installed=true" > {0}_installed.txt'.format(
 #                    pkg.repo_name.replace("-", "_")))
 
-#    with dog_stats_api.timer(metric_name, tags=package_tags +
-#                             ['step:pkg_version']):
-#        pkg_version()
+    pkg_version()
 
-    with dog_stats_api.timer(metric_name, tags=package_tags +
-                             ['step:post_commands']):
     # post checkout commands
     with prefix("export GIT_SSH=/tmp/git.sh"):
         for cmd in env.pre_post['post']:
@@ -384,15 +373,6 @@ def _install_requirements(pkg):
         # suspended
         sudo('{0} {1}'.format(AA_COMPLAIN, AA_SANDBOX_POLICY))
 
-    # Run old-style requirements TODO: remove
-    _run_if_changed(pkg, 'pre-requirements.txt', partial(
-                    pip_install, file='pre-requirements.txt', venv='/opt/edx'),
-                    'cat *requirements.txt')
-    _run_if_changed(pkg, 'requirements.txt', partial(
-                    pip_install, file='requirements.txt', venv='/opt/edx'),
-                    'cat *requirements.txt')
-    # end old-style requirements
-
     # Run new-style requirements
     for venv in VIRTUAL_ENVS:
         if not files.exists(venv):
@@ -409,7 +389,6 @@ def _install_requirements(pkg):
     if files.exists(AA_ENFORCE) and files.exists(AA_SANDBOX_POLICY):
         sudo('{0} {1}'.format(AA_ENFORCE, AA_SANDBOX_POLICY))
 
-
 @task
 @runs_once
 def deploy_with_puppet():

package_data.yaml

 # datadog integration configuration
-
 datadog_api: ''
 
 # repo_dirs : Exhaustive list of repo-dirs and their corresponding repo-names
@@ -21,12 +20,16 @@ repo_dirs:
   /opt/wwc/grading-controller: edx/grading-controller
   /opt/wwc/ease: edx/ease
   /opt/wwc/edx-platform: edx/edx-platform
+  /opt/wwc/data/cs50: MITx/cs50
+  /opt/wwc/data/edx4edx: MITx/edx4edx
+  /opt/wwc/data/700x: MITx/700x
   /opt/wwc/xqueue: edx/xqueue
   /opt/wwc/xserver: edx/xserver
   /opt/wwc/certificates: edx/certificates
+  /opt/wwc/drupal: MITx/drupal
   /opt/wwc/latex2edx: edx/latex2edx_xserver
   /opt/wwc/worker: edx/edx-platform
-
+  /opt/sysadmin: MITx/sysadmin
 # {pre,post}_checkout_regex:
 #      What actions to take before and after a repo checkout.
 #      Commands will be grouped together if multiple matches are made
@@ -134,22 +137,34 @@ post_checkout_regex: !!omap
 
   - ^edx-platform$|^content-.*$:
     - |
-      # Requirements files have been moved, so check the location first
-      if [[ -e /opt/wwc/edx-platform/local-requirements.txt ]]; then
+      #install local requirements for the platform
+      if [[ -e /opt/wwc/edx-platform/requirements/edx/local.txt ]]; then
         cd /opt/wwc/edx-platform
-        sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps -r \
-        /opt/wwc/edx-platform/local-requirements.txt
+        sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps --ignore-installed \
+        --exists-action w -r /opt/wwc/edx-platform/requirements/edx/local.txt
       fi
-      # Then check the new location
-      if [[ -e /opt/wwc/edx-platform/requirements/local.txt ]]; then
+      #install local requirements for the sandbox
+
+      if [[ -e /etc/apparmor.d/code.sandbox ]]; then
+        sudo /usr/sbin/aa-complain /etc/apparmor.d/code.sandbox
+
+        if [[ -e /opt/wwc/edx-platform/requirements/edx-sandbox/local.txt ]]; then
           cd /opt/wwc/edx-platform
-        sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps -r \
-        /opt/wwc/edx-platform/requirements/local.txt
+          sudo -E /opt/edx-sandbox/bin/pip install -q --upgrade --no-deps --ignore-installed \
+          --exists-action w -r /opt/wwc/edx-platform/requirements/edx-sandbox/local.txt
+          fi
+        sudo /usr/sbin/aa-enforce /etc/apparmor.d/code.sandbox
       fi
-
   - ^edx-platform$:
     - |
+      # On servers running the workers, which run from the same cloned
+      # repo, we don't want to run collect static.  This could cause
+      # problems because the cache's are shared with the edx-platform instances
+      # it's also simply not necessary for the workers as they have no
+      # view component.
+      edx_workers_status=$(service edx-workers status 2>/dev/null || true)
       # git.py already handle prerequisites, so don't let rake try to install them as well
+      if [[ -z $edx_workers_status ]]; then
         export NO_PREREQ_INSTALL=1
         # This is not a developer workspace, so we don't want to do the workspace migrations
         # Instead, we rely on having a clean checkout every time
@@ -177,7 +192,16 @@ post_checkout_regex: !!omap
           --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws \
           --noinput --verbosity=0
         fi
+      fi
     - |
+      # On servers running the workers, which run from the same cloned
+      # repo, we don't want to run collect static.  This could cause
+      # problems because the cache's are shared with the edx-platform instances
+      # it's also simply not necessary for the workers as they have no
+      # view component.
+      edx_workers_status=$(service edx-workers status 2>/dev/null || true)
+      # git.py already handle prerequisites, so don't let rake try to install them as well
+      if [[ -z $edx_workers_status ]]; then
         export NO_PREREQ_INSTALL=1
         export SKIP_WS_MIGRATIONS=1
         export RBENV_ROOT=/opt/www/.rbenv
@@ -203,7 +227,7 @@ post_checkout_regex: !!omap
           $RUN django-admin.py update_templates \
           --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws
         fi
-
+      fi
   - ^edx-platform$|^content-.*$:
     - |
       edxapp_status=$(service edxapp status 2>/dev/null || true)