Merge pull request #80 from edx/jarv/fabric-for-deploys

Jarv/fabric for deploys

.gitignore

 *.swp
+*.pyc
 .idea/

fabfile/__init__.py

+# Additional Tasks
+import cache
+import clean
+import ec2
+import audit
+import git
+import hosts
+import locks
+import os
+import ssh
+import status
+import migrate_check
+
+import yaml
+from dogapi import dog_stats_api, dog_http_api
+from timestamps import TSWrapper
+
+# Global tasks
+import logging
+from fabric.api import env, task, runs_once
+from output import squelch
+from datetime import datetime
+import sys
+import time
+from fabric.api import execute, local, task, runs_once
+from fabric.utils import fastprint
+from fabric.colors import blue
+from ssh_tunnel import setup_tunnel
+
+# These imports are to give aliases for these tasks
+from hosts import by_tags as tag
+from hosts import by_tags as tags
+from hosts import exemplar_from_tags as exemplar
+from git import default_deploy as deploy
+
+env.linewise = True
+env.noop = False
+env.use_ssh_config = True
+
+FORMAT = '[ %(asctime)s ] : %(message)s'
+logging.basicConfig(format=FORMAT, level=logging.WARNING)
+
+# add timestamps to output
+sys.stdout = TSWrapper(sys.stdout)
+sys.stderr = TSWrapper(sys.stderr)
+
+path = os.path.abspath(__file__)
+with open(os.path.join(
+        os.path.dirname(path), '../package_data.yaml')) as f:
+    package_data = yaml.load(f)
+    dog_stats_api.start(api_key=package_data['datadog_api'], statsd=True)
+    dog_http_api.api_key = package_data['datadog_api']
+
+
+@task
+def noop():
+    """
+    Disable modification of servers
+    """
+    env.noop = True
+    dog_stats_api.stop()
+
+
+@task
+def quiet():
+    """
+    Disables verbose output
+    """
+    squelch()
+
+
+@runs_once
+@task()
+def log(fname=None):
+    """
+    Writes a logfile to disk of the run
+    """
+
+    if not fname:
+        d = datetime.now()
+        fname = d.strftime('/var/tmp/fab-%Y%m%d-%H%M%S-{0}.log'.format(
+            os.getpid()))
+
+    env.logfile = fname
+    sys.stdout.log_to_file(fname)
+    sys.stderr.log_to_file(fname)

fabfile/alias.py

+import time
+from fabric.api import execute, local, task, runs_once
+from fabric.utils import fastprint
+from fabric.colors import blue
+from ssh_tunnel import setup_tunnel
+
+# These imports are to give aliases for these tasks
+from hosts import by_name as name
+from hosts import by_tags as tag
+from hosts import by_tags as tags
+from hosts import exemplar_from_tags as exemplar
+from git import default_deploy as deploy

fabfile/audit.py

+import logging
+from fabric.api import serial, task, parallel, env, execute, runs_once, settings,sudo
+from fabfile.safety import noopable
+from multiprocessing import Manager
+from timestamps import no_ts
+from packages import PackageInfo
+import tempfile
+from output import notify
+
+
+@task
+@parallel
+def collect_installed_packages(results):
+    """
+    Collect all installed packages for the selected hosts and store them in env
+    """
+    print env.host
+    pkg_info = PackageInfo()
+    results[env.host] = pkg_info.installed_packages()
+
+
+@task
+@serial
+def display_installed_packages(installed_packages):
+    """
+    Print all installed packages collected by collect_installed_packages
+    """
+
+    # FIXME: env.hosts loses the port information here, not sure why
+    with no_ts():
+        for pkg in installed_packages['{0}:22'.format(env.host)]:
+            notify("{pkg.name} = {pkg.revision}".format(pkg=pkg))
+
+
+@task(default=True)
+@runs_once
+def installed_packages(from_links=False):
+    """
+    List all of the installed packages on the selected packages
+    """
+    installed_packages = Manager().dict()
+    execute(collect_installed_packages, installed_packages)
+    execute(display_installed_packages, installed_packages)
+
+
+@task
+def audit_user(user, audit_output=None):
+    """
+    Logs on provided hosts and runs id for the supplied user with sudo.  Output
+    is logged to the provided file argument or a default using the
+    python gettempdir() function and the following file name format:
+
+    /tmp/audit-user-{user}.csv
+
+    The contents of this file are
+
+    host,user,command output
+
+    Note that if the file already exists, output will be appended to the
+    existing file.
+
+    """
+    logging.info("Auditing {host}.".format(host=env.host_string))
+
+    if not audit_output:
+        audit_output = tempfile.gettempdir() + "/audit-user-{user}.csv".format(
+            user=user)
+
+    with settings(warn_only=True):
+        with open(audit_output, 'a') as audit:
+            output = noopable(sudo)("id {user}".format(user=user))
+            audit.write("{host},{user},{output}\n".format(
+                host=env.host_string,
+                user=user,
+                output=output
+                )
+        )
+
+@task
+def remove_user(user):
+    """
+    Logs on to provided hosts and runs userdel for the supplied user with sudo.
+    The user's home directory is preserved.
+    """
+
+    logging.info("Removing {user} user from {host}.".format(
+        user=user,host=env.host_string))
+
+    with settings(warn_only=True):
+        output = noopable(sudo)("userdel {user}".format(user=user))
+        logging.info("Output of userdel command on host {host} was {out}".format(
+            host=env.host_string,out=output
+            )
+        )
+

fabfile/cache.py

+from fabric.api import task, runs_once, env, serial, puts, settings
+from fabric.utils import fastprint
+from fabric.colors import blue, red, white
+
+from output import notify
+from packages import PackageDescriptor
+from output import unsquelched
+from hosts import exemplar
+from ssh_tunnel import setup_tunnel
+from packages import PackageInfo
+
+
+@task
+@runs_once
+def from_exemplar(**tags):
+    """
+    Cache the set of packages installed on one host from the specified tags.
+
+    """
+    host_string = setup_tunnel([exemplar(**tags)])[0]
+    with settings(host_string=host_string):
+        installed_packages()
+
+
+@task
+@runs_once
+def limit_prefix(*prefix_list):
+    """
+    Limits cached packages to those that
+    match one or more prefix strings
+    """
+    env.package_descriptors = filter(
+        lambda pkg: any(pkg.name.startswith(prefix)
+                        for prefix in prefix_list), env.package_descriptors)
+
+
+@task(default=True)
+@runs_once
+def installed_packages(prefix=None):
+    """
+    Cache the set of packages installed on the selected host.
+    """
+    pkg_info = PackageInfo()
+    env.package_descriptors = [
+        package for package in pkg_info.installed_packages()
+        if prefix is None or package.name.startswith(prefix)
+    ]
+
+
+@task
+@runs_once
+def from_strings(**pkg_revs):
+    """
+    Cache packages based on strings, that can be either checked with confirm
+    or deployed with deploy.
+
+    Each named argument specifies a package by name, and the revision of
+    the package to deploy
+    """
+    packages = []
+    for pkg_name, pkg_rev in pkg_revs.items():
+        packages.append(PackageDescriptor(pkg_name, pkg_rev))
+
+    env.package_descriptors = packages
+
+    notify(env.package_descriptors)
+
+
+@task
+@runs_once
+def from_stdin(prefix=None):
+    """
+    Cache a list of packages from stdin.
+    Package names must start with prefix, if specified (any that don't
+    will be skipped). Package names and revisions should be separated
+    by = signs, and should be one per line.
+    """
+
+    if prefix:
+        prefix_msg = white('pkg_name', bold=True) + white(
+            ' must start with ') + blue(prefix)
+    else:
+        prefix_msg = ''
+    fastprint('\n')
+    fastprint('\n'.join([
+        white('Please enter pkg_name=pkg_rev, one per line\n', bold=True),
+        white('pkg_rev', bold=True) + white(' is a git revision hash'),
+        prefix_msg,
+        white('Complete your selections by entering a blank line.'),
+    ]))
+    fastprint('\n\n')
+
+    packages = {}
+    while True:
+        line = raw_input("> ")
+        if not line:
+            break
+
+        if '=' not in line:
+            fastprint(red("Expected = in '{line}'. Skipping...".format(
+                          line=line)) + white('\n'))
+            continue
+
+        pkg_name, _, pkg_rev = line.partition('=')
+        pkg_name = pkg_name.strip()
+        pkg_rev = pkg_rev.strip()
+
+        if prefix and not pkg_name.startswith(prefix):
+            fastprint(red("'{0}' does not start with '{1}'".format(
+                          pkg_name, prefix)) + white('\n'))
+            continue
+
+        packages[pkg_name] = pkg_rev
+
+    from_strings(**packages)
+
+
+@task
+@serial
+@runs_once
+def prompt(*pkg_names):
+    packages = {}
+    with unsquelched():
+        puts("Please supply git revisions to "
+             "deploy for the following packages:")
+        for pkg in pkg_names:
+            packages[pkg] = raw_input("{pkg} = ".format(pkg=pkg)).strip()
+
+        from_strings(**packages)

fabfile/choose.py

+from output import notify
+from fabric.api import abort
+from fabric.colors import blue, cyan, green, red, white
+from fabric.utils import fastprint
+
+
+def choose(msg, options):
+    choices = range(len(options))
+
+    fastprint(white(msg, bold=True) + white("\n"))
+    for i, target in enumerate(options):
+        fastprint("{0}. {1}\n".format(i, target))
+    fastprint("x. Cancel\n")
+
+    user_input = raw_input("> ")
+    if user_input == 'x':
+        abort("Cancelled")
+
+    try:
+        choice = int(user_input)
+    except:
+        fastprint(red("Choice must be an integer"))
+        return None
+
+    if choice not in choices:
+        fastprint(red("Choice must be one of {0}".format(choices)))
+        return None
+
+    return options[choice]
+
+
+def multi_choose_with_input(msg, options):
+    """
+    Options:
+        msg - header message for the chooser
+        options - dictionary of options to select
+
+
+    User selects one of the keys in the dictionary,
+    a new value is read from stdin
+    """
+
+    selections = options.keys()
+    user_input = None
+
+    while True:
+        fastprint('\n{0}{1}'.format(white(msg, bold=True), white("\n")))
+
+        # The extra white("\n") prints are to reset
+        # the color for the timestamp line prefix
+
+        fastprint(white("\n"))
+        for i, item in enumerate(selections):
+            fastprint(" {0}. {1} : {2}".format(white(i, bold=True),
+                cyan(item), cyan(options[item], bold=True)) + white("\n"))
+        fastprint(blue("  a. Select all") + white("\n"))
+        fastprint(blue("  c. Continue") + white("\n"))
+        fastprint(blue("  x. Cancel") + white("\n"))
+        fastprint(white("\n"))
+        user_input = raw_input("> ")
+
+        try:
+            if user_input == 'c':
+                break
+            elif user_input == 'x':
+                return None
+            elif int(user_input) in range(len(selections)):
+                name = selections[int(user_input)]
+                fastprint(green('Enter new msg for ') +
+                        cyan(name))
+                options[name] = raw_input(white(": "))
+        except:
+            notify("Invalid selection ->" + user_input + "<-")
+    return options
+
+
+def multi_choose(msg, options):
+
+    fastprint(white(msg, bold=True) + white("\n"))
+    selected = [" " for option in options]
+
+    user_input = None
+
+    while True:
+
+        # The extra white("\n") prints are to reset
+        # the color for the timestamp line prefix
+
+        fastprint(white("\n"))
+        for i, target in enumerate(options):
+            fastprint(green(selected[i]))
+            fastprint(cyan(" {0}. {1}".format(i, target)) + white("\n"))
+        fastprint(blue("  a. Select all") + white("\n"))
+        fastprint(blue("  c. Deploy selections") + white("\n"))
+        fastprint(blue("  x. Cancel") + white("\n"))
+        fastprint(white("\n"))
+
+        user_input = raw_input("> ")
+
+        try:
+            if user_input == 'c':
+                break
+            elif user_input == 'a':
+                selected = ['*' for i in range(len(selected))]
+            elif user_input == 'x':
+                return None
+            elif int(user_input) in range(len(options)):
+                if selected[int(user_input)] == " ":
+                    selected[int(user_input)] = "*"
+                else:
+                    selected[int(user_input)] = " "
+        except:
+            notify("Invalid selection ->" + user_input + "<-")
+
+    pkgs = [options[s] for s in range(len(selected)) if selected[s] == '*']
+    return pkgs

fabfile/clean.py

+from fabric.api import sudo, task, parallel
+from safety import noopable
+from modifiers import rolling
+
+@task
+@parallel
+def apt_get_clean():
+    """ Runs apt-get clean on a remote server """
+    noopable(sudo)('apt-get clean')   
+
+@task
+@rolling
+def mako_template_cache():
+    noopable(sudo)('service gunicorn stop')
+    noopable(sudo)('rm -rf /tmp/tmp*mako')
+    noopable(sudo)('service gunicorn start')

fabfile/ec2.py

+import boto
+from fabric.api import run, task, parallel, env
+
+env.instance_ids = {}
+
+
+def instance_id():
+    if env.host_string not in env.instance_ids:
+        env.instance_ids[env.host_string] = run('wget -q -O - http://169.254.169.254/latest/meta-data/instance-id')
+
+    return env.instance_ids[env.host_string]

fabfile/git.py

+import itertools
+import os
+import re
+import socket
+
+from functools import partial
+from dogapi import dog_stats_api
+from fabric.api import task, sudo, runs_once, execute
+from fabric.api import cd, env, abort, parallel, prefix
+from fabric.colors import white, green, red
+from fabric.contrib import console, files
+from fabric.operations import put
+from fabric.utils import fastprint
+from multiprocessing import Manager
+
+from .choose import multi_choose
+from .metrics import instance_tags_for_current_host
+from .modifiers import rolling
+from .output import notify
+from .packages import PackageInfo
+from .safety import noopable
+from .timestamps import no_ts
+
+REPO_URL = 'git@github.com:{}/{}'
+REPO_DIRNAME = '/opt/wwc'
+GIT_USER = "www-data"
+AA_COMPLAIN = '/usr/sbin/aa-complain'
+AA_ENFORCE = '/usr/sbin/aa-enforce'
+AA_SANDBOX_POLICY = '/etc/apparmor.d/code.sandbox'
+VIRTUAL_ENVS = ['/opt/edx', '/opt/edx-sandbox']
+
+
+@task(default=True, aliases=['deploy'])
+@runs_once
+def default_deploy(**pkg_revs):
+    """
+    Deploys the cached or specified packages to the
+    specified hosts
+
+    Packages are installed while the server is out of the
+    loadbalancer pool
+    """
+
+    if pkg_revs:
+        execute('cache.from_strings', **pkg_revs)
+    if socket.gethostname() != 'buildmaster-001':
+        execute('git.confirm')
+    execute('git.deploy')
+    execute('locks.remove_all_locks')
+
+
+def diff_link(pkg_org, pkg_name, old_revision, new_revision):
+    if '~' in pkg_name:
+        pkg_name = re.sub('~.*', '', pkg_name)
+
+    return 'Show on github: https://github.com/{org}/{pkg}/compare/{old}' \
+           '...{new}'.format(org=pkg_org,
+                             pkg=pkg_name,
+                             old=old_revision,
+                             new=new_revision)
+
+
+@task
+@parallel
+def diff_installed_packages(results):
+    pkg_info = PackageInfo()
+    # { basename(repo_dir): PackageDescriptor()) ... }
+    old_pkgs = {pkg.name: pkg
+                for pkg in pkg_info.installed_packages()}
+    change_list = []
+    for new_pkg in env.package_descriptors:
+        if new_pkg.name in old_pkgs:
+            old = old_pkgs[new_pkg.name].revision
+            new = new_pkg.revision
+            change_list.append((new_pkg.name, old, new))
+        else:
+            change_list.append((new_pkg.name, None, new_pkg.revision))
+    results.append((tuple(change_list), env.host))
+
+
+@task
+@runs_once
+def confirm():
+    """
+    Generate diffs comparing what's installed to what's about to be installed,
+    and ask the user to confirm to continue.
+
+    Answering no aborts the entire operation
+    """
+    execute('locks.wait_for_all_locks')
+    # turn off timestamps for the confirm prompt
+    with no_ts():
+        manager = Manager()
+        diffs = manager.list()
+        execute(diff_installed_packages, diffs)
+
+        local_diffs = list(diffs)
+
+        def sort_key(diff):
+            return diff[0]
+
+        local_diffs.sort(key=sort_key)
+
+        if not local_diffs:
+            execute('locks.remove_all_locks')
+            abort("Nothing to deploy")
+
+        choices = set()
+        pkg_info = PackageInfo()
+        servers_to_update = []
+        for key, group in itertools.groupby(local_diffs, key=sort_key):
+            servers = [d[1] for d in group]
+            servers_to_update.extend(servers)
+            for pkg, old, new in key:
+                choices.add(pkg)
+                notify(
+                    "    {name}: {diff}".format(
+                        name=pkg,
+                        old=old,
+                        new=new,
+                        diff=diff_link(pkg_info.org_from_name(pkg),
+                                       pkg_info.repo_from_name(pkg), old, new),
+                    ),
+                    show_prefix=False
+                )
+
+        choices = list(choices)
+
+        if len(choices) > 1:
+            selection = multi_choose("Select one or more item numbers to mark"
+                                     "them with a '*' for deployment.\n"
+                                     "Note: none are selected by default.\n"
+                                     "Select 'c' to deploy "
+                                     "the items that are marked with a '*'.",
+                                     choices)
+        else:
+            selection = choices
+
+        if not selection:
+            notify('Removing all locks and aborting')
+            execute('locks.remove_all_locks')
+            abort('Operation cancelled by user')
+
+        pre_post = display_pre_post(selection)
+        env.pre_post = pre_post
+        notify("{noop}Updating servers [{servers}]:".format(
+            servers=", ".join(servers_to_update),
+            noop="[noop] " if env.noop else ''
+        ), show_prefix=False)
+
+        if not console.confirm(
+                white('Please confirm the pre and post actions above',
+                      bold=True),
+                default=True):
+            execute('locks.remove_all_locks')
+            abort('Operation cancelled by user')
+
+        env.package_descriptors = [
+            pkg for pkg in env.package_descriptors if pkg.name in selection]
+
+
+def display_pre_post(choices):
+
+    """
+    Displays list of pre and post checkout commands,
+    returns the list of commands in a dictionary
+
+        return({
+                'pre': [ 'cmd1', 'cmd2', ... ],
+                'post': [ 'cmd1', 'cmd2', ... ]
+            })
+
+    """
+
+    pkg_info = PackageInfo()
+    pre_post = pkg_info.pre_post_actions(choices)
+
+    for stage in ['pre', 'post']:
+        if pre_post[stage]:
+            fastprint(green('{0}-checkout commands:\n'.format(stage),
+                      bold=True) + green('  -> ') + green('\n  -> '.join(
+                      pre_post[stage])) + white('\n\n'))
+        else:
+            fastprint(green('WARNING', bold=True) +
+                      green(' - no {0}-checkout commands for this set '
+                      'of packages : '.format(stage, choices)) +
+                      white('\n\n'))
+    return pre_post
+
+
+@task
+@rolling
+def deploy(auto_migrate=False):
+    """
+    Deploys the cached packages to the specified hosts.
+    Packages are installed while the server is out of the loadbalancer pool
+    """
+
+    packages = env.package_descriptors
+
+    # If these are not defined it means that the confirm
+    # step was skipped, in this case we figure out pre and
+    # post steps here
+    if not hasattr(env, 'pre_post'):
+        pkg_config = PackageInfo()
+        env.pre_post = pkg_config.pre_post_actions([pkg.name
+                                                   for pkg in packages])
+
+    contains_content = any(pkg.name.startswith('content') for pkg in packages)
+    contains_code = not all(pkg.name.startswith('content') for pkg in packages)
+
+    base_tags = [
+        'deploying_to_host:' + env.host,
+    ] + instance_tags_for_current_host()
+
+    if contains_content:
+        base_tags.append('type:content')
+    if contains_code:
+        base_tags.append('type:code')
+
+    package_tags = base_tags + ['package:' + pkg.name for pkg in packages]
+    metric_name = 'fabric.deployment'
+
+    # pre checkout commands
+    with dog_stats_api.timer(metric_name, tags=package_tags +
+                             ['step:pre_commands']):
+        with prefix("export GIT_SSH=/tmp/git.sh"):
+            for cmd in env.pre_post['pre']:
+                noopable(sudo)(cmd)
+
+    put(os.path.join(os.path.dirname(__file__), 'git.sh'),
+        '/tmp/git.sh', mode=0755, use_sudo=True)
+    for pkg in packages:
+        existing_repo = files.exists(pkg.repo_root, use_sudo=True)
+
+        repo_tags = base_tags + [
+            'package:' + pkg.name,
+            'existance:' + 'existing' if existing_repo else 'absent',
+        ]
+
+        with dog_stats_api.timer(metric_name, tags=repo_tags + ['step:clone']):
+            if existing_repo:
+                if not files.exists(os.path.join(pkg.repo_root, '.git'),
+                                    use_sudo=True):
+                    raise Exception("Repo root not a git repo - {0}".format(
+                        os.path.join(pkg.repo_root, '.git')))
+                with cd(pkg.repo_root):
+                    if pkg.revision == 'absent':
+                        noopable(sudo)('rm -rf {0}'.format(pkg.repo_root))
+                    else:
+                        checkout(pkg.revision)
+            else:
+                with cd(os.path.dirname(pkg.repo_root)):
+                    if pkg.revision != 'absent':
+                        clone(pkg.repo_org, pkg.repo_name, pkg.name, pkg.revision)
+            if '~' in pkg.name:
+                _update_course_xml(pkg, pkg.name.split('~')[1])
+
+        with dog_stats_api.timer(metric_name, tags=repo_tags +
+                                 ['step:requirements']):
+            _install_requirements(pkg)
+            _install_gemfile(pkg)
+            _install_npm_package(pkg)
+
+#        with dog_stats_api.timer(metric_name, tags=repo_tags + ['step:fact']):
+#            # drop a file for puppet so it knows that
+#            # code is installed for the service
+#            with cd('/etc/facter/facts.d'):
+#                pkg_config = PackageInfo()
+#                if pkg.repo_name in pkg_config.service_repos:
+#                    # facts can't have dashes so they are converted
+#                    # to underscores
+#                    noopable(sudo)(
+#                        'echo "{0}_installed=true" > {0}_installed.txt'.format(
+#                        pkg.repo_name.replace("-", "_")))
+
+#    with dog_stats_api.timer(metric_name, tags=package_tags +
+#                             ['step:pkg_version']):
+#        pkg_version()
+
+    with dog_stats_api.timer(metric_name, tags=package_tags +
+                             ['step:post_commands']):
+        # post checkout commands
+        with prefix("export GIT_SSH=/tmp/git.sh"):
+            for cmd in env.pre_post['post']:
+                noopable(sudo)(cmd)
+
+    if 'mitx' in [pkg.name for pkg in packages]:
+        # do not slow down content deploys by checking
+        # for migrations
+        execute('migrate_check.migrate_check', auto_migrate)
+
+
+@task
+def pkg_version():
+    """
+    Drops an html/json version file on the remote system
+    """
+    path = os.path.dirname(os.path.abspath(__file__))
+    with open(os.path.join(path, '../version-script/version.py')) as f:
+        put(f, '/tmp/version.py', use_sudo=True)
+
+    noopable(sudo)('/usr/bin/python /tmp/version.py')
+
+
+@task
+def clone(repo_org, repo_name, name, revision):
+    """
+    Parameters:
+        repo_name: github organization
+        repo_name: github repo name
+        name: basename(repo_dir) ex: content-mit-600x~Fall_2012
+        revision: commit hash
+    """
+
+    with no_ts():
+        with prefix("export GIT_SSH=/tmp/git.sh"):
+            noopable(sudo)("git clone {} {}".format(
+                REPO_URL.format(repo_org, repo_name), name))
+            with cd(name):
+                noopable(sudo)("git reset --hard {0}".format(revision))
+                if files.exists('.gitmodules', use_sudo=True):
+                    noopable(sudo)("git submodule update --init")
+                noopable(sudo)("chown -R {0}:{0} .".format(GIT_USER))
+
+
+@task
+def checkout(revision):
+    """
+    Parameters:
+        revision: commit hash
+    """
+
+    with no_ts():
+        with prefix("export GIT_SSH=/tmp/git.sh"):
+            noopable(sudo)("git remote prune origin")
+            noopable(sudo)("git fetch origin")
+            noopable(sudo)("git reset --hard {0}".format(revision))
+            if files.exists('.gitmodules', use_sudo=True):
+                noopable(sudo)("git submodule update --init")
+            noopable(sudo)("chown -R {0}:{0} .".format(GIT_USER))
+
+
+def _update_course_xml(pkg, root):
+    """
+    Parameters:
+        pkg: a single package descriptor
+        root: a root that must exist in the roots/ folder
+    """
+    with cd(pkg.repo_root):
+        if files.exists(
+                os.path.join(pkg.repo_root, 'roots/{0}.xml'.format(root)),
+                use_sudo=True):
+            noopable(sudo)('rm -f course.xml && '
+                           'ln -s roots/{0}.xml course.xml'.format(root))
+        else:
+            abort(red("There is a '~' in {0} but there is no roots/{1}.xml "
+                      "file in the repo!".format(pkg.name, root)))
+
+
+def _install_requirements(pkg):
+    """
+    Parameters:
+        pkg: single package descriptor
+
+    install pre-requirements and requirements
+    if they exists for the repo.
+    will not run pip install if the requirements file
+    has not changed since the last run
+
+    Turns off apparmor enforcement during pip install
+    for the sandbox virtualenv
+    """
+
+    def pip_install(file, venv='/opt/edx'):
+        with prefix("export GIT_SSH=/tmp/git.sh"):
+            with prefix('source {}'.format(os.path.join(venv, 'bin/activate'))):
+                with prefix('export PIP_DOWNLOAD_CACHE=/tmp/pip_download_cache'):
+                    noopable(sudo)('pip install --exists-action w -r {0}'.format(file))
+
+    if files.exists(AA_COMPLAIN) and files.exists(AA_SANDBOX_POLICY):
+        # in order to install code-sandbox requirements the
+        # code sandbox apparmor policy must be temporarily
+        # suspended
+        sudo('{0} {1}'.format(AA_COMPLAIN, AA_SANDBOX_POLICY))
+
+    # Run old-style requirements TODO: remove
+    _run_if_changed(pkg, 'pre-requirements.txt', partial(
+                    pip_install, file='pre-requirements.txt', venv='/opt/edx'),
+                    'cat *requirements.txt')
+    _run_if_changed(pkg, 'requirements.txt', partial(
+                    pip_install, file='requirements.txt', venv='/opt/edx'),
+                    'cat *requirements.txt')
+    # end old-style requirements
+
+    # Run new-style requirements
+    for venv in VIRTUAL_ENVS:
+        if not files.exists(venv):
+            # skip if the virtualenv doesn't exist
+            continue
+        venv_name = os.path.basename(venv)
+        pip_cmd_base = partial(pip_install, file='requirements/{}/base.txt'.format(venv_name), venv=venv)
+        pip_cmd_post = partial(pip_install, file='requirements/{}/post.txt'.format(venv_name), venv=venv)
+        _run_if_changed(pkg, 'requirements/{}/base.txt'.format(venv_name), pip_cmd_base,
+                        'cat requirements/{}/*.txt'.format(venv_name))
+        _run_if_changed(pkg, 'requirements/{}/post.txt'.format(venv_name), pip_cmd_post,
+                        'cat requirements/{}/*.txt'.format(venv_name))
+
+    if files.exists(AA_ENFORCE) and files.exists(AA_SANDBOX_POLICY):
+        sudo('{0} {1}'.format(AA_ENFORCE, AA_SANDBOX_POLICY))
+
+
+@task
+@runs_once
+def deploy_with_puppet():
+    execute('git.confirm')
+    execute(_deploy_with_puppet)
+    execute('locks.remove_all_locks')
+
+
+@task
+@rolling
+def _deploy_with_puppet():
+    execute('puppet')
+    execute('git.deploy')
+
+
+def _install_gemfile(pkg):
+    """
+    Parameters:
+        pkg: single package descriptor
+
+    Installs the Gemfile from the repo, if it exists.
+    Will not run if the Gemfile
+    has not changed since the last run
+    """
+    def bundle_install():
+        with prefix('export PATH=/opt/www/.rbenv/bin:$PATH'):
+            with prefix('RBENV_ROOT=/opt/www/.rbenv'):
+                with prefix('which rbenv'):
+                    with prefix('eval "$(rbenv init -)"'):
+                        noopable(sudo)('gem install bundler')
+                        noopable(sudo)('bundle install --binstubs')
+
+    _run_if_changed(pkg, 'Gemfile', bundle_install)
+
+
+def _install_npm_package(pkg):
+    """
+    Parameters:
+        pkg: single package descriptor
+
+    Installs the package.json from the repo, if it exists.
+    Will not run if the package.json has not changed since
+    the last run
+    """
+    def package_install():
+        noopable(sudo)('npm install')
+
+    _run_if_changed(pkg, 'package.json', package_install)
+
+
+def _run_if_changed(pkg, file, command, contents_command=None):
+    """
+    Runs command if the contents of file
+    inside pkg have changed since the last time the command was run.
+    Commands are executed inside pkg.repo_root
+    """
+    if contents_command is None:
+        contents_command = 'cat ' + file
+
+    with cd(pkg.repo_root):
+        path = os.path.join(pkg.repo_root, file)
+        if files.exists(path, use_sudo=True):
+            prev_md5_file = '/var/tmp/{0}-{1}.md5'.format(
+                pkg.repo_name.replace('/', '-'), file.replace('/', '-'))
+            md5_command = '{} | /usr/bin/md5sum'.format(contents_command)
+            if files.exists(prev_md5_file, use_sudo=True):
+                cur_md5 = sudo(md5_command)
+                prev_md5 = sudo('cat {0}'.format(prev_md5_file))
+                if cur_md5 == prev_md5:
+                    return
+
+            command()
+            noopable(sudo)('{} > {}'.format(md5_command, prev_md5_file))

fabfile/git.sh

+#!/bin/sh
+exec ssh -i "/etc/git-identity" -o "StrictHostKeyChecking no" "$@"

fabfile/hosts.py

+import boto
+from fabric.decorators import serial
+from ssh_tunnel import setup_tunnel
+import socket
+from fabric.api import env, task, abort
+from fabric.colors import red
+import logging
+
+
+def hosts_by_tag(tag, value):
+    """
+    Return a list of all hosts that have the specified value for the specified
+    tag
+    """
+    return hosts_by_tags(**{tag: value})
+
+
+def hosts_by_tags(**tags):
+    """
+    Return a list of all hosts that have the specified value for the specified
+    tags.
+
+    Tag values are allowed to include wildcards
+
+    If no variant tag is specified, this command will ignore all hosts
+    that have a variant specified.
+    """
+
+    if 'env' in tags:
+        tags['environment'] = tags['env']
+        del(tags['env'])
+
+    ec2 = boto.connect_ec2()
+    hosts = []
+    for res in ec2.get_all_instances(filters={'tag:' + tag: value
+                                        for tag, value in tags.iteritems()
+                                        if value != '*'}):
+        for inst in res.instances:
+            if inst.state == "running":
+                if (inst.public_dns_name):
+                    hosts.append(inst.public_dns_name)
+                else:
+                    hosts.append(inst.private_dns_name)
+    print hosts
+    return hosts
+
+def _fleet():
+    ec2 = boto.connect_ec2()
+    hosts = []
+    for res in ec2.get_all_instances():
+        for inst in res.instances:
+            if inst.state == "running":
+                try:
+                    instance_name = inst.tags['Name']
+                except:
+                    logging.warning("Instance with id {id} and {dns} has no assigned Name.".format(id=inst.id,dns=inst.public_dns_name))
+
+
+                host_to_add = instance_name + "." + DOMAIN
+
+                # fallback to the public hostname if the m.edx.org
+                # name doesn't exist
+                try:
+                    socket.gethostbyname(host_to_add.replace(':22',''))
+                except socket.error:
+                    if inst.public_dns_name:
+                        host_to_add = inst.public_dns_name
+
+                if host_to_add:
+                    hosts.append(host_to_add)
+    return hosts
+
+
+def exemplar(**tags):
+    """
+    Return the hostname of one host from the specified set
+    of tags, or None if there is no such host
+    """
+
+    hosts = hosts_by_tags(**tags)
+    if hosts:
+        return hosts[0]
+    else:
+        return None
+
+@task(alias='exemplar')
+def exemplar_from_tags(**tags):
+    env.hosts.append(exemplar(**tags))
+
+
+@task(aliases=['tag', 'tags'])
+def by_tags(**tags):
+    """
+    Add all running hosts that match the tag names provided
+    as keyword arguments.
+
+    """
+
+    env.hosts.extend(hosts_by_tags(**tags))
+    env.hosts.sort()
+    env.hosts = setup_tunnel(env.hosts)
+
+
+@task(aliases=['fleet'])
+def fleet():
+    """
+    Return a list of all hosts available  and running via the default AWS
+    credentials.
+
+    Your ability to operate on these hosts will depend upon the ssh credentials
+    that you are using to drive fab.  There is likely to be a mismatch between
+    what hosts you can see via IAM managed AWS credentials and which hosts
+    you can actually connect to even if you are using highly privileged
+    AWS pems.
+    """
+    hosts = _fleet()
+    env.hosts.extend(hosts)
+    env.hosts.sort()
+    env.hosts = setup_tunnel(env.hosts)

fabfile/locks.py

+import os
+import socket
+import time
+
+from output import notify
+from safety import noopable
+from fabric.api import task, run, env, settings, sudo, abort
+from fabric.api import runs_once, execute, serial, hide
+
+MAX_SLEEP_TIME = 10
+
+LOCK_FILE = '/opt/deploy/.lock'
+
+
+@task
+@runs_once
+def wait_for_all_locks():
+    execute('locks.wait_for_lock', hosts=sorted(env.hosts))
+
+
+@task
+@runs_once
+def remove_all_locks():
+    execute('locks.remove_lock', hosts=sorted(env.hosts, reverse=True))
+
+
+@task
+@serial
+def remove_lock():
+    noopable(sudo)("test ! -f {0} || rm {0}".format(LOCK_FILE))
+
+
+@task
+@serial
+def wait_for_lock():
+    if hasattr(env, 'deploy_user'):
+        lock_user = env.deploy_user
+    else:
+        lock_user = env.user
+
+    LOCK_ID = 'u:{user} h:{host} pid:{pid}'.format(user=lock_user,
+                                    host=socket.gethostname(),
+                                pid=str(os.getpid()))
+    sleep_time = 0.1
+    timeout = 120
+    start_time = time.time()
+
+    with settings(warn_only=True):
+        while True:
+            wait_time = time.time() - start_time
+
+            # break if the lockfile is removed or if it belongs to this pid
+            # if it exists lock_status will have the file's contents
+
+            with hide('running', 'stdout', 'stderr', 'warnings'):
+                lock_status = run("test ! -f {lfile} || "
+                                  "(cat {lfile} && "
+                                  'grep -q "{lid}" {lfile})'.format(
+                                      lfile=LOCK_FILE,
+                                      lid=LOCK_ID))
+
+                if lock_status.succeeded:
+                    noopable(sudo)('echo "{0}" > {1}'.format(
+                        LOCK_ID, LOCK_FILE))
+                    notify("Took lock")
+                    break
+
+                elif wait_time >= timeout:
+                    abort("Timeout expired, giving up")
+
+                lock_create_time = run("stat -c %Y {0}".format(LOCK_FILE))
+
+            delta = time.time() - float(lock_create_time)
+            (dhour, dsec) = divmod(delta, 3600)
+
+            notify("""
+
+        !! Deploy lockfile already exists ({lockfile}) !!
+            Waiting: {wait}s
+            Lockfile info: [ {owner} ]
+            Lock created: {dhour}h{dmin}m ago
+            """.format(
+                        lockfile=LOCK_FILE,
+                        wait=int(timeout - wait_time),
+                        owner=lock_status,
+                        dhour=int(dhour),
+                        dmin=int(dsec / 60),
+                        ))
+            time.sleep(sleep_time)
+            sleep_time *= 2
+            if sleep_time > MAX_SLEEP_TIME:
+                sleep_time = MAX_SLEEP_TIME

fabfile/maint.py

+from fabric.api import task, parallel, put, sudo
+from safety import noopable
+from .modifiers import rolling
+from StringIO import StringIO
+import json
+
+__all__ = ['on', 'off','maintain_service','unmaintain_service']
+
+services = ['lms','cms','lms-xml','lms-preview']
+
+
+def set_maintenance(value):
+    noopable(put)(StringIO(json.dumps({'maintenance': value})), '/etc/facter/facts.d/mitx_maintenance.json', use_sudo=True)
+
+
+@task
+@parallel
+def on():
+    """
+    Enable maintenance mode
+    """
+    set_maintenance(True)
+    puppet.checkin('maintenance')
+
+
+@task
+@parallel
+def off():
+    """
+    Disable maintenance mode
+    """
+    set_maintenance(False)
+    puppet.checkin('maintenance')
+
+
+@task
+@rolling
+def maintain_service(service):
+    """
+    Puts a specified edxapp service into maintenance mode by replacing
+    its nginx sites-enabled link with a link to the maintenance vhost.
+    """
+
+    if service not in services:
+        raise Exception("Provided service not in the service inventory. "
+                        "Acceptable values are {services}".format(
+            services=services
+        ))
+
+    noopable(sudo)("rm -f /etc/nginx/sites-enabled/{service}".format(
+        service=service))
+
+    noopable(sudo)("ln -s /etc/nginx/sites-available/{service}-maintenance"
+                   " /etc/nginx/sites-enabled/{service}-maintenance".format(
+        service=service))
+
+    noopable(sudo)("service nginx reload")
+
+@task
+@rolling
+def unmaintain_service(service):
+    """
+    Removes a specified edxapp service from maintenance mode by replacing
+    the appropriate link in /etc/nginx/sites-enabled.
+    """
+
+    if service not in services:
+        raise Exception("Provided service not in the service inventory. "
+                        "Acceptable values are {services}".format(
+            services=services
+        ))
+
+    noopable(sudo)("rm -f /etc/nginx/sites-enabled/{service}-maintenance".format(
+        service=service))
+
+    noopable(sudo)("ln -s /etc/nginx/sites-available/{service}"
+                   " /etc/nginx/sites-enabled/{service}".format(
+        service=service))
+
+    noopable(sudo)("service nginx reload")

fabfile/metrics.py

+import boto
+from .ec2 import instance_id
+
+
+def instance_tags_for_current_host():
+    """
+    Returns the datadog style tags for the active host
+    """
+    return instance_tags([instance_id()])
+
+
+def instance_tags(instance_ids):
+    """
+    Returns datadog style tags for the specified instances
+    """
+    ec2 = boto.connect_ec2()
+
+    tags = set()
+    for res in ec2.get_all_instances(instance_ids):
+        for instance in res.instances:
+            ec2_tags = instance.tags
+
+            tags.add('instance_id:' + instance.id)
+            if 'group' in ec2_tags:
+                tags.add('fab-group:' + ec2_tags['group'])
+            if 'environment' in ec2_tags:
+                tags.add('fab-environment:' + ec2_tags['environment'])
+            if 'variant' in ec2_tags:
+                tags.add('fab-variant:' + ec2_tags['variant'])
+
+    return list(tags)

fabfile/migrate_check.py

+from fabric.api import task, sudo, runs_once, prefix, hide, abort
+from fabric.contrib import console
+from fabric.colors import white, green
+from .safety import noopable
+
+
+@task()
+@runs_once
+def migrate_check(auto_migrate=False):
+    """
+    Checks to see whether migrations need to be run,
+    if they do it will prompt to run them before
+    continuing.
+
+    looks for " - Migrating" in the output of
+    the dry run
+
+    """
+
+    migration_cmd = "/opt/edx/bin/django-admin.py migrate --noinput " \
+                    "--settings=lms.envs.aws --pythonpath=/opt/wwc/edx-platform"
+
+    with prefix("export SERVICE_VARIANT=lms"):
+        with hide('running', 'stdout', 'stderr', 'warnings'):
+            dryrun_out = sudo(migration_cmd + " --db-dry-run", user="www-data")
+        migrate = False
+        for chunk in dryrun_out.split('Running migrations for '):
+            if 'Migrating' in chunk:
+                print "!!! Found Migration !!!\n" + chunk
+                migrate = True
+        if migrate:
+            if auto_migrate or console.confirm(
+                    green(migration_cmd) + white('\n') +
+                    white('Run migrations? ', bold=True), default=True):
+                noopable(sudo)(migration_cmd, user='www-data')

fabfile/modifiers.py

+import boto
+import time
+
+from collections import namedtuple
+from fabric.api import task, execute, serial
+from functools import wraps, partial
+from safety import noopable
+from output import notify
+from dogapi import dog_stats_api
+from .metrics import instance_tags
+from .ec2 import instance_id
+
+MAX_SLEEP_TIME = 1
+
+
+LockedElb = namedtuple('LockedElb', 'name elb lock')
+
+
+def await_elb_instance_state(lb, instance_id, awaited_state):
+
+    sleep_time = 0.1
+    start_time = time.time()
+    while True:
+        state = lb.get_instance_health([instance_id])[0].state
+        if state == awaited_state:
+            notify("Load Balancer {lb} is in awaited state {awaited_state}, proceeding.".format(
+                lb=lb.dns_name,
+                awaited_state=awaited_state
+            ))
+            break
+        else:
+
+            notify("Checking again in {0} seconds. Elapsed time: {1}".format(sleep_time, time.time() - start_time))
+            time.sleep(sleep_time)
+            sleep_time *= 2
+            if sleep_time > MAX_SLEEP_TIME:
+                sleep_time = MAX_SLEEP_TIME
+
+
+def rolling(func):
+
+    @task
+    @serial
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        elb = boto.connect_elb()
+
+        elbs = elb.get_all_load_balancers()
+        execute('locks.wait_for_all_locks')
+
+        inst_id = instance_id()
+        tags = ['task:' + func.__name__] + instance_tags(inst_id)
+        active_lbs = sorted(
+            lb
+            for lb in elbs
+            if inst_id in [info.id for info in lb.instances]
+        )
+
+        timer = partial(dog_stats_api.timer, tags=tags)
+
+        # Remove this node from the LB
+        for lb in active_lbs:
+            notify("Removing {id} from {lb}".format(id=inst_id, lb=lb))
+
+            with timer('rolling.deregister_instance'):
+                noopable(lb.deregister_instances)([inst_id])
+                noopable(await_elb_instance_state)(lb, inst_id, "OutOfService")
+
+        # Execute the operation
+        func(*args, **kwargs)
+
+        # Add this node back to the LBs
+        for lb in active_lbs:
+            notify("Adding {id} to {lb}".format(id=inst_id, lb=lb))
+            with timer('rolling.register_instance'):
+                noopable(lb.register_instances)([inst_id])
+
+        with timer('rolling.wait_for_start'):
+            # Wait for the node to come online in the LBs
+            for lb in active_lbs:
+                noopable(await_elb_instance_state)(lb, inst_id, "InService")
+
+    return wrapper

fabfile/output.py

+import sys
+from contextlib import contextmanager
+from fabric.api import puts
+
+class SquelchingStream(object):
+    def __init__(self, stream):
+        self.__dict__['stream'] = stream
+        self.__dict__['squelched'] = False
+        self.__dict__['needs_line_ending'] = False
+
+    def write(self, string):
+        if self.squelched:
+            self.stream.write('.')
+            self.stream.flush()
+            self.needs_line_ending = True
+        else:
+            if self.needs_line_ending:
+                self.needs_line_ending = False
+                self.stream.write('\n')
+            self.stream.write(string)
+
+    def __getattr__(self, attr):
+        return getattr(self.stream, attr)
+
+    def __setattr__(self, attr, val):
+        if attr in self.__dict__:
+            return object.__setattr__(self, attr, val)
+
+        return setattr(self.stream, attr, val)
+
+sys.stdout = SquelchingStream(sys.stdout)
+sys.stderr = SquelchingStream(sys.stderr)
+
+def squelch():
+    sys.stdout.squelched = sys.stderr.squelched = True
+
+def unsquelch():
+    sys.stdout.squelched = sys.stderr.squelched = False
+
+@contextmanager
+def unsquelched(stream=sys.stdout):
+    old_state = stream.squelched
+    stream.squelched = False
+    yield
+    stream.squelched = old_state
+
+def notify(msg, show_prefix=None, end='\n', flush=False):
+    with unsquelched():
+        puts(msg, show_prefix, end, flush)

fabfile/packages.py

+import os
+from fabric.api import run, settings, hide, sudo
+from collections import defaultdict
+import yaml
+import re
+
+MIN_REVISION_LENGTH = 7
+
+
+class PackageInfo:
+
+    def __init__(self):
+        path = os.path.abspath(__file__)
+        with open(os.path.join(
+                os.path.dirname(path), '../package_data.yaml')) as f:
+            package_data = yaml.load(f)
+        # exhaustive list of MITx repos
+        self.repo_dirs = package_data['repo_dirs']
+        self.cmd_list = {
+                'pre': package_data['pre_checkout_regex'],
+                'post': package_data['post_checkout_regex']}
+        self.service_repos = package_data['service_repos']
+
+    def repo_from_name(self, name):
+        repos = []
+        for repo_root in self.repo_dirs:
+            if os.path.basename(repo_root) == name:
+                repos.append(self.repo_dirs[repo_root])
+
+        if len(repos) > 1:
+            raise Exception['Multiple repos found for name']
+        elif len(repos) == 0:
+            raise Exception['Repo not found for name']
+        else:
+            return repos[0].split('/')[1]
+
+    def org_from_name(self, name):
+        repos = []
+        for repo_root in self.repo_dirs:
+            if os.path.basename(repo_root) == name:
+                repos.append(self.repo_dirs[repo_root])
+
+        if len(repos) > 1:
+            raise Exception['Multiple repos found for name']
+        elif len(repos) == 0:
+            raise Exception['Repo not found for name']
+        else:
+            return repos[0].split('/')[0]
+
+
+
+    def pre_post_actions(self, pkgs):
+        """
+        Returns a dictionary containing a list of
+        commands that need to be executed
+        pre and post checkout for one or more package names.
+
+        return({
+                'pre': [ 'cmd1', 'cmd2', ... ],
+                'post': [ 'cmd1', 'cmd2', ... ]
+            })
+
+        """
+
+        cmds = defaultdict(list)
+        for stage in ['pre', 'post']:
+            for regex, cmd_templates in self.cmd_list[stage]:
+                for pkg in pkgs:
+                    match = re.match(regex, pkg)
+                    if match is None:
+                        continue
+
+                    cmds[stage].extend(
+                        cmd.format(*match.groups(), **match.groupdict())
+                        for cmd in cmd_templates
+                        if cmd not in cmds[stage]
+                    )
+        return(cmds)
+
+    def installed_packages(self):
+        """
+        Returns the list of PackageDescriptors for the packages
+        installed on the system.
+
+        This is determined by looking at every package directory
+        we know about and checking its revision.
+        """
+
+        with settings(hide('running'), warn_only=True):
+            revisions = sudo(
+            """
+            for path in {0}; do
+                if [[ -d "$path/.git" ]]; then
+                    echo $path $(cd $path && git rev-parse HEAD 2>/dev/null)
+                fi
+            done
+            """.format(' '.join(self.repo_dirs))).split('\n')
+        packages = [revline.strip().split(' ') for revline in revisions
+                if ' ' in revline.strip()]
+
+        return [PackageDescriptor(os.path.basename(path), revision)
+                        for path, revision in packages]
+
+
+class PackageDescriptor(object):
+
+    def __init__(self, name, revision):
+
+        if revision != 'absent' and len(revision) < MIN_REVISION_LENGTH:
+            raise Exception("Must use at least {0} characters "
+            "in revision to pseudo-guarantee uniqueness".format(
+                MIN_REVISION_LENGTH))
+
+        self.name = name
+
+        # Find the repo_root by name
+        # This assumes that basename(repo_root) is unique
+        # for all repo_roots.  If this is not true an exception
+        # will be raised
+
+        pkg_info = PackageInfo()
+        repo_roots = []
+        for repo_dir in pkg_info.repo_dirs.keys():
+            if os.path.basename(repo_dir) == name:
+                repo_roots.append(repo_dir)
+        if len(repo_roots) != 1:
+            raise Exception("Unable to look up directory for repo")
+
+        self.repo_root = repo_roots[0]
+        self.repo_name = pkg_info.repo_dirs[self.repo_root].split('/')[1]
+        self.repo_org = pkg_info.repo_dirs[self.repo_root].split('/')[0]
+        self.revision = revision

fabfile/safety.py

+from fabric.api import env
+
+from output import notify
+
+def noopable(fun):
+    if env.noop:
+        def noop(*args, **kwargs):
+            notify("Would have called: {fun}({args}, {kwargs})".format(
+                fun=fun.__name__,
+                args=", ".join(repr(a) for a in args),
+                kwargs=", ".join("=".join([key, repr(val)]) for key, val in kwargs.items()),
+            ))
+        return noop
+    else:
+        return fun

fabfile/ssh.py

+from fabric.api import task, env, abort
+from fabric.colors import red
+import os
+import re
+
+
+@task(default=True)
+def ssh(user=None):
+
+    if user is None:
+        user = env.user
+    if len(env.hosts) != 1:
+        abort(red('Please specify one host for ssh'))
+
+    for host in env.hosts:
+        host = re.sub(':(\d+)', r' -p\1 ', host)
+        os.system('ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -l {0} {1}'.format(user, host))

fabfile/ssh_tunnel.py

+from fabric.api import abort, env, fastprint
+from fabric.colors import green, red, white
+import subprocess
+import shlex
+import atexit
+import time
+import boto
+import re
+import socket
+
+DOMAIN = 'm.edx.org:22'
+
+
+class SSHTunnel:
+
+    port = 9000  # default starting port
+    tunnels = {}
+
+    def __init__(self, host, phost, user, lport=None):
+
+        if lport is not None:
+            SSHTunnel.port = lport
+
+        while True:
+            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            s.settimeout(5)
+            try:
+                s.connect(('localhost', SSHTunnel.port))
+                s.shutdown(2)
+                # connection was successful so try a new port
+                SSHTunnel.port += 1
+            except:
+                self.lport = SSHTunnel.port
+                break
+        phost = re.sub(':(\d+)', r' -p\1 ', phost)
+
+        identities = ''
+        if env.key_filename:
+            # could be a list or a string
+            if isinstance(env.key_filename, basestring):
+                lst = [env.key_filename]
+            else:
+                lst = env.key_filename
+
+            identities = ' '.join('-i {f} '.format(f=f) for f in lst)
+
+        cmd = 'ssh -o UserKnownHostsFile=/dev/null ' \
+              '{ids}' \
+              '-o StrictHostKeyChecking=no -vAN -L {lport}:{host} ' \
+              '{user}@{phost}'.format(ids=identities, lport=self.lport,
+                                      host=host, user=user, phost=phost)
+
+        self.p = subprocess.Popen(shlex.split(cmd),
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE)
+
+        start_time = time.time()
+        atexit.register(self.p.kill)
+        while not 'Entering interactive session' in self.p.stderr.readline():
+            if time.time() > start_time + 10:
+                abort(red("Unable to create ssh tunnel - `{0}`".format(cmd)))
+
+    def local(self):
+        return 'localhost:{lport}'.format(lport=self.lport)
+
+
+def setup_tunnel(all_hosts, check_tag=True,
+                 proxy_name=None, user=None, lport=None):
+    """
+    Given a all_hosts it will check to see whether
+    any are proxy hosts if check_tag is True
+
+    returns a modified list
+    of hosts with localhost:port for tunneled hosts.
+    """
+
+    if user is None:
+        user = env.user
+    ec2 = boto.connect_ec2()
+
+    # the proxy hosts
+    proxies = {}
+    if check_tag:
+        for res in ec2.get_all_instances(filters={'tag-key': 'proxy'}):
+            for inst in res.instances:
+                host = ".".join([inst.tags['Name'], DOMAIN])
+                proxy = ".".join([inst.tags['proxy'], DOMAIN])
+                proxies.update({host: proxy})
+    else:
+        if not proxy_name:
+            raise Exception("Must specify a proxy_host")
+
+        proxies = {host: proxy_name for host in all_hosts}
+
+    # local tunneling ip:port
+    tunnels = {}
+    for host in all_hosts:
+        if host in proxies and host not in SSHTunnel.tunnels:
+            t = SSHTunnel(host=host, phost=proxies[host],
+                          user=user, lport=lport)
+            tunnels[host] = t.local()
+            fastprint(green('created {0} for {1} via {2}'.format(tunnels[host],
+                      host, proxies[host])) + white('\n'))
+            SSHTunnel.tunnels.update(tunnels)
+
+    return([SSHTunnel.tunnels[host] if host in SSHTunnel.tunnels else host
+           for host in all_hosts])

fabfile/status.py

+from fabric.api import task, sudo, abort, parallel, runs_once, execute
+from fabric.api import settings, hide
+from fabric.operations import put
+from fabric.utils import fastprint
+from safety import noopable
+from fabric.colors import blue, red
+from fabric.contrib import console
+from output import unsquelched
+from timestamps import no_ts
+from choose import multi_choose_with_input
+import json
+import tempfile
+
+status_file = '/opt/wwc/status_message.json'
+
+
+@task(default=True)
+@runs_once
+def status():
+    """
+    Drops {0} which is a json formatted file that contains a
+    status message that will be displayed to all users on the
+    on the courseware for a single course or for all courses
+    if 'global' is set.
+
+    Message(s) are entered or removed interactively on the console.
+
+    Example usage:
+
+        $ fab groups:prod_edx status
+
+    """.format(status_file)
+
+    with hide('running', 'stdout', 'stderr', 'warnings'):
+        env_json = sudo("cat /opt/wwc/lms-xml.env.json")
+    course_listings = json.loads(env_json)['COURSE_LISTINGS']
+    course_ids = [course_id for course_list in course_listings.itervalues()
+                  for course_id in course_list]
+    course_ids = ['global'] + course_ids
+
+    with no_ts():
+
+        course_status = None
+        with settings(warn_only=True):
+            cur_status = noopable(sudo)('cat {0}'.format(status_file))
+        try:
+            course_status = json.loads(cur_status)
+            # add empty entries for courses not in the list
+            empty_entries = set(course_ids) - set(course_status.keys())
+            course_status.update({entry: '' for entry in list(empty_entries)})
+
+        except ValueError:
+            fastprint(red("Not a valid json file, overwritting\n"))
+        if course_status is None:
+            course_status = {course: '' for course in course_ids}
+        new_status = multi_choose_with_input(
+                'Set the status message, blank to disable:',
+                course_status)
+
+        if new_status is not None:
+            # remove empty entries
+            new_status = {entry: new_status[entry]
+                    for entry in new_status if len(new_status[entry]) > 1}
+            with unsquelched():
+                if not console.confirm(
+                        'Setting new status message:\n{0}'.format(
+                            blue(str(new_status), bold=True)),
+                            default=False):
+                    abort('Operation cancelled by user')
+
+                with tempfile.NamedTemporaryFile(delete=True) as f:
+                    f.write(json.dumps(new_status))
+                    f.flush()
+                    execute(update_status, f.name)
+        else:
+            abort('Operation cancelled by user')
+
+
+@task
+@runs_once
+def remove():
+    """
+    Removes {0}, a status banner that is displayed to all
+    users on the front page.
+    """.format(status_file)
+
+    with unsquelched():
+        if not console.confirm(
+                blue('Remove /opt/wwc/status_message.html?', bold=True)):
+            abort('Operation cancelled by user')
+        execute(remove_status)
+
+
+@task
+@parallel
+def remove_status():
+    noopable(sudo)('rm -f {0}'.format(status_file))
+
+
+@task
+@parallel
+def update_status(fjson):
+    print status_file
+    noopable(put)(fjson, status_file, use_sudo=True)

fabfile/timestamps.py

+from datetime import datetime
+from contextlib import contextmanager
+import sys
+
+
+@contextmanager
+def no_ts():
+    sys.stdout.ts = False
+    yield
+    sys.stdout.ts = True
+
+
+class TSWrapper(object):
+
+    def __init__(self, stream):
+        self.o = stream
+        self.files = []
+        self.files.append(self.o)
+        self.newline = True
+        self.ts = True
+
+    def write(self, s):
+        d = datetime.now()
+        if self.ts:
+            buf = ""
+            lines = s.splitlines(True)
+
+            for line in lines:
+                if self.newline:
+                    buf += d.strftime('[ %Y%m%d %H:%M:%S ] : {0}'.format(line))
+                else:
+                    buf += str(line)
+
+                if line[-1] == '\n':
+                    self.newline = True
+                else:
+                    self.newline = False
+        else:
+            buf = s
+
+        for fh in self.files:
+            fh.write(buf)
+            fh.flush()
+
+    def log_to_file(self, fn):
+        fp = open(fn, 'a')
+        self.files.append(fp)
+
+    def __getattr__(self, attr):
+        return getattr(self.o, attr)

fabric-requirements.txt

+Fabric==1.5.1
+Jinja2==2.6
+PyYAML==3.10
+WebOb==1.2.3
+argparse==1.2.1
+beautifulsoup4==4.1.3
+boto==2.7.0
+cloudformation==0.0.0
+decorator==3.4.0
+distribute==0.6.30
+docopt==0.6.1
+dogapi==1.2.3
+ipython==0.13.1
+jenkinsapi==0.1.11
+lxml==3.1beta1
+newrelic==1.10.2.38
+paramiko==1.9.0
+path.py==3.0.1
+pingdom==0.2.0
+pycrypto==2.6
+pyparsing==1.5.6
+pyrelic==0.2.0
+python-dateutil==2.1
+requests==1.1.0
+schema==0.1.1
+simplejson==3.3.0
+simples3==1.0-alpha
+six==1.2.0
+-e git+https://github.com/bos/statprof.py.git@a17f7923b102c9039763583be9e377e8422e8f5f#egg=statprof-dev
+ujson==1.30
+wsgiref==0.1.2

package_data.yaml

+# datadog integration configuration
+
+datadog_api: ''
+
+# repo_dirs : Exhaustive list of repo-dirs and their corresponding repo-names
+# the repos are specified by basename(repo_dir), therefore basename(repo_dir)
+# under repo_dirs must be unique.
+
+# The following repos have upstart services with the same name
+service_repos:
+  - edx-platform
+  - grading-controller
+  - xqueue
+  - xserver
+  - certificates
+  - latex2edx
+  - analytics-experiments
+
+repo_dirs:
+  /opt/wwc/analytics-experiments: MITx/analytics-experiments
+  /opt/wwc/grading-controller: edx/grading-controller
+  /opt/wwc/ease: edx/ease
+  /opt/wwc/edx-platform: edx/edx-platform
+  /opt/wwc/xqueue: edx/xqueue
+  /opt/wwc/xserver: edx/xserver
+  /opt/wwc/certificates: edx/certificates
+  /opt/wwc/latex2edx: edx/latex2edx_xserver
+  /opt/wwc/worker: edx/edx-platform
+
+# {pre,post}_checkout_regex:
+#      What actions to take before and after a repo checkout.
+#      Commands will be grouped together if multiple matches are made
+#      with the duplicates removed.
+#
+#      For example a deploy of 'edx-platform' and 'grading-controller' will execute
+#      all pre_checkout_regex commands before the checkouts and then
+#      execute post_checkout_regex commands after.
+#
+#      Commmands will be displayed to the user during the confirm step.
+#
+#
+
+pre_checkout_regex: !!omap
+
+
+  - ^edx-platform$|^content-.*$:
+    - |
+
+      edxapp_status=$(service edxapp status 2>/dev/null || true)
+      edx_workers_status=$(service edx-workers status 2>/dev/null || true)
+      if [[ -n $edxapp_status ]]; then
+        if [[ $edxapp_status == *running* ]]; then
+          service edxapp stop;
+        fi
+      fi
+      if [[ -n $edx_workers_status ]]; then
+        if [[ $edx_workers_status == *running* ]]; then
+          service edx-workers stop;
+        fi
+      fi
+  - ^grading-controller$|^ease$:
+
+    - if [[ $(service grader status) == *running* ]]; then
+        service grader stop;
+      fi
+    - if [[ $(service pull_from_xqueue status) == *running* ]]; then
+        service pull_from_xqueue stop;
+      fi
+    - if [[ $(service ml_grader status) == *running* ]]; then
+        service ml_grader stop;
+      fi
+    - if [[ $(service expire_old status) == *running* ]]; then
+        service expire_old stop;
+      fi
+    - if [[ $(service ml_creator status) == *running* ]]; then
+        service ml_creator stop;
+      fi
+
+  - ^xqueue$:
+    - if [[ $(service xqueue status) == *running* ]]; then
+        service xqueue stop;
+      fi
+    - if [[ $(service xqueue_consumer status) == *running* ]]; then
+        service xqueue_consumer stop;
+      fi
+  - ^certificates$:
+    - if [[ $(service certificates status) == *running* ]]; then
+        service certificates stop;
+      fi
+  - ^xserver|^content-mit-600x.*$:
+    - if [[ $(service xserver status) == *running* ]]; then
+        service xserver stop;
+      fi
+  - ^latex2edx$:
+    - if [[ $(service latex2edx status) == *running* ]]; then
+        service latex2edx stop;
+      fi
+  - ^analytics-experiments$:
+    - if [[ $(service analytics status) == *running* ]]; then
+        service analytics stop;
+      fi
+
+
+post_checkout_regex: !!omap
+
+  - ^sandbox$|^staging$|^production$|^hiera$:
+    - chown -R puppet:puppet /etc/puppet
+
+  - ^(?P<repo>content-.*)$:
+    - |
+      COLLECT_STATIC_PKG_DIR=/opt/wwc/staticfiles/{repo}
+      COURSE_STATIC_PKG_DIR=/opt/wwc/course_static/{repo}
+      SOURCE_PKG_DIR=/opt/wwc/data/{repo}
+
+      # Move from course static data in the collectstatic root
+      # to being symlinked in a separate directory
+      if [[ -e $COLLECT_STATIC_PKG_DIR ]]; then
+        rm -rf "$COLLECT_STATIC_PKG_DIR"
+      fi
+
+      # If a course adds or removes a static subdirectory,
+      # we want to adjust the symlink
+      if [[ -e $COURSE_STATIC_PKG_DIR ]]; then
+        rm -rf "$COURSE_STATIC_PKG_DIR"
+      fi
+      if [[ -h $COURSE_STATIC_PKG_DIR ]]; then
+        rm "$COURSE_STATIC_PKG_DIR"
+      fi
+      if [[ -d $SOURCE_PKG_DIR/static ]]; then
+        ln -s $SOURCE_PKG_DIR/static $COURSE_STATIC_PKG_DIR
+      else
+        ln -s $SOURCE_PKG_DIR $COURSE_STATIC_PKG_DIR
+      fi
+
+  - ^edx-platform$|^content-.*$:
+    - |
+      # Requirements files have been moved, so check the location first
+      if [[ -e /opt/wwc/edx-platform/local-requirements.txt ]]; then
+        cd /opt/wwc/edx-platform
+        sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps -r \
+        /opt/wwc/edx-platform/local-requirements.txt
+      fi
+      # Then check the new location
+      if [[ -e /opt/wwc/edx-platform/requirements/local.txt ]]; then
+        cd /opt/wwc/edx-platform
+        sudo -E /opt/edx/bin/pip install -q --upgrade --no-deps -r \
+        /opt/wwc/edx-platform/requirements/local.txt
+      fi
+
+  - ^edx-platform$:
+    - |
+      # git.py already handle prerequisites, so don't let rake try to install them as well
+      export NO_PREREQ_INSTALL=1
+      # This is not a developer workspace, so we don't want to do the workspace migrations
+      # Instead, we rely on having a clean checkout every time
+      export SKIP_WS_MIGRATIONS=1
+
+      export RBENV_ROOT=/opt/www/.rbenv
+      # Use rbenv binaries
+      export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
+      # Use binaries installed in rbenv
+      export PATH=/opt/wwc/edx-platform/bin:$PATH
+      # Use binaries installed in virtualenv
+      export PATH=/opt/edx/bin:$PATH
+      eval "$(rbenv init -)"
+      # {{}} is used here because of python .format upstream
+      LMS_VARIANT="$(ls -1 /opt/wwc/lms*.env.json | tail -1 | xargs -i basename {{}} .env.json)"
+      export RUN="sudo -E -u www-data env SERVICE_VARIANT=$LMS_VARIANT PATH=$PATH"
+      cd /opt/wwc/edx-platform
+
+      # If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic)
+      if $(rake -T | grep --quiet gather_assets); then
+        $RUN rake lms:gather_assets:aws
+      # Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod]
+      elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws &>/dev/null); then
+        $RUN django-admin.py collectstatic \
+        --pythonpath=/opt/wwc/edx-platform --settings=lms.envs.aws \
+        --noinput --verbosity=0
+      fi
+    - |
+      export NO_PREREQ_INSTALL=1
+      export SKIP_WS_MIGRATIONS=1
+      export RBENV_ROOT=/opt/www/.rbenv
+      # Use rbenv binaries
+      export PATH=$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
+      # Use binaries installed in rbenv
+      export PATH=/opt/wwc/edx-platform/bin:$PATH
+      # Use binaries installed in virtualenv
+      export PATH=/opt/edx/bin:$PATH
+      eval "$(rbenv init -)"
+      export RUN="sudo -E -u www-data env SERVICE_VARIANT=cms PATH=$PATH"
+      cd /opt/wwc/edx-platform
+      # If we're gather_assets is available, run it (to compile coffee and sass, and then collectstatic)
+      if $(rake -T | grep --quiet gather_assets); then
+        $RUN rake cms:gather_assets:aws
+      # Otherwise, we're still using django pipeline, so just run collectstatic [TODO: Remove this clause when gather_assets gets to prod]
+      elif $($RUN django-admin.py help collectstatic --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then
+        $RUN django-admin.py collectstatic \
+        --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws \
+        --noinput --verbosity=0
+      fi
+      if $($RUN django-admin.py help update_templates --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws &>/dev/null) && [[ -r /opt/wwc/cms.auth.json ]]; then
+        $RUN django-admin.py update_templates \
+        --pythonpath=/opt/wwc/edx-platform --settings=cms.envs.aws
+      fi
+
+  - ^edx-platform$|^content-.*$:
+    - |
+      edxapp_status=$(service edxapp status 2>/dev/null || true)
+
+      edx_workers_status=$(service edx-workers status 2>/dev/null || true)
+      
+      if [[ -n $edxapp_status ]]; then
+        if [[ $edxapp_status == *stop* ]]; then
+          service edxapp start;
+        fi
+      fi
+      if [[ -n $edx_workers_status ]]; then
+        if [[ $edx_workers_status == *stop* ]]; then
+          service edx-workers start;
+        fi
+      fi
+  - ^grading-controller$|^ease$:
+    - |
+      if [[ ! -d /usr/share/nltk_data ]];then
+        source /opt/edx/bin/activate &&
+        python -m nltk.downloader -d /usr/share/nltk_data all;
+      fi
+    - |
+      if [[ -d /opt/wwc/.rbenv ]]; then
+        export RBENV_ROOT=/opt/wwc/.rbenv
+        export PATH=/opt/wwc/edx-platform/bin:$RBENV_ROOT/shims:$RBENV_ROOT/bin:$PATH
+        eval "$(rbenv init -)"
+      fi
+      if $(sudo -E -u www-data /opt/edx/bin/django-admin.py help collectstatic --pythonpath=/opt/wwc/grading-controller --settings=grading_controller.aws &>/dev/null); then
+        cd /opt/wwc/staticfiles
+        sudo -E -u www-data  /opt/edx/bin/django-admin.py collectstatic \
+        --pythonpath=/opt/wwc/grading-controller --settings=grading_controller.aws \
+        --noinput --verbosity=0
+      fi
+    - |
+      if [[ $(service grader status) == *stop* ]]; then
+        service grader start;
+      fi
+    - |
+      if [[ $(service expire_old status) == *stop* ]]; then
+        service expire_old start;
+      fi
+    - |
+      if [[ $(service ml_grader status) == *stop* ]]; then
+        service ml_grader start;
+      fi
+    - |
+      if [[ $(service ml_creator status) == *stop* ]]; then
+        service ml_creator start;
+      fi
+    - |
+      if [[ $(service pull_from_xqueue status) == *stop* ]]; then
+        service pull_from_xqueue start;
+      fi
+
+  - ^xqueue$:
+    - |
+      if [[ $(service xqueue status) == *stop* ]];then
+        service xqueue start;
+      fi
+    - |
+      if [[ $(service xqueue_consumer status) == *stop* ]];then
+        service xqueue_consumer start;
+      fi
+
+  - ^xserver|^content-mit-600x.*$:
+    - |
+      if [[ $(service xserver status) == *stop* ]]; then
+        service xserver start;
+      fi
+
+  - ^certificates$:
+    - |
+      if [[ $(service certificates status) == *stop* ]];then
+        service certificates start;
+      fi
+  - ^latex2edx$:
+    - |
+      if [[ $(service latex2edx status) == *stop* ]];then
+        service latex2edx start;
+      fi
+  - ^analytics-experiments$:
+    - |
+      if [[ $(service analytics status) == *stop* ]];then
+        service analytics start;
+      fi
+  - ^drupal$:
+    - |
+      if [[ -r /opt/wwc/settings.php ]]; then
+        cp /opt/wwc/settings.php /opt/wwc/drupal/sites/default/settings.php;
+      fi;
+      chown -R root:www-data /opt/wwc/drupal &&
+      find /opt/wwc/drupal -type d -exec chmod u=rwx,g=rx,o= '{}' \; &&
+      find /opt/wwc/drupal -type f -exec chmod u=rw,g=r,o= '{}' \; &&
+      find /opt/wwc/drupal/sites -type d -name files -exec chmod ug=rwx,o= '{}' \; &&
+      for x in /opt/wwc/drupal/sites/*/files; do
+              find ${x} -type d -exec chmod ug=rwx,o= '{}' \; &&
+              find ${x} -type f -exec chmod ug=rw,o= '{}' \;;
+      done;
+      chmod g+w /opt/wwc/drupal/sites/default/tmp &&
+      if [[ -r /opt/wwc/drupal/db/edx.sql ]]; then
+        cd /opt/wwc/drupal &&
+        `drush sql-connect` < db/edx.sql;
+      fi

version-script/version.py

+#!/usr/bin/env python
+import os
+import subprocess
+import re
+import json
+import glob
+
+ROOT_DIR = '/opt/wwc'
+VERSION_FILE = '/opt/wwc/versions.html'
+VERSION_JSON = '/opt/wwc/versions.json'
+GLOB_DIRS = [
+              os.path.join(ROOT_DIR, '*/.git'),
+              os.path.join(ROOT_DIR, '*/*/.git'),
+              os.path.join(ROOT_DIR, 'data/*/.git'),
+            ]
+
+
+TEMPLATE = """
+<html>
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <title></title>
+    <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.5.2/jquery.min.js"></script>
+
+    <style>
+      body {{
+         font-size: 2em;
+         color: #000;
+         font-family: monospace;
+      }}
+    </style>
+
+  </head>
+<body>
+<ul>
+  {BODY}
+</ul>
+</body>
+<script type="text/javascript">
+  $(".collapse").click(function () {{
+        $(this).parent().children().toggle();
+        $(this).toggle();
+
+        }});
+  $(document).ready(function() {{
+        $('.collapse').parent().children().toggle();
+        $('.collapse').toggle();
+  }});
+</script>
+</html>
+"""
+
+
+def main():
+    assert os.path.isdir(ROOT_DIR)
+    # using glob with fixed depths is much
+    # faster than os.walk for finding all .git repos
+    git_dirs = [git_dir for glob_dir in GLOB_DIRS
+                for git_dir in glob.glob(glob_dir)]
+    git_dirs = filter(lambda f: os.path.isdir(f), git_dirs)
+
+    version_info = ""
+    versions = {}
+
+    for git_dir in git_dirs:
+        repo_dir = git_dir.replace('/.git', '')
+        repo_dir_basename = os.path.basename(repo_dir)
+
+        # get the revision of the repo
+        p = subprocess.Popen(['/usr/bin/git', 'rev-parse', 'HEAD'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                    cwd=git_dir)
+        rev_output, rev_err = p.communicate()
+        revision = rev_output.splitlines()[0][:8]
+
+        # dictionary that will be written out as JSON
+        versions[repo_dir_basename] = revision
+
+        # use reflogs for the repo history
+        p = subprocess.Popen(
+          ['/usr/bin/git', 'log', '-g', '--abbrev-commit', '--pretty=oneline'],
+           stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=git_dir)
+
+        reflog_output, reflog_err = p.communicate()
+
+        # ignore lines that do not have 'HEAD'
+        reflog_lines = filter(lambda x: x.find('HEAD') >= 0,
+                                            reflog_output.splitlines())
+
+        # get the repo name, `git remote -v` seems like the fastest option
+        p = subprocess.Popen(['/usr/bin/git', 'remote', '-v'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                    cwd=git_dir)
+        remote_output, remote_err = p.communicate()
+        repo_url = None
+        for line in remote_output.splitlines():
+            if ' (fetch)' in line:
+                repo_url = re.search(
+                        'git@(.*) \(fetch\)', line).group(1).replace(
+                                                                ':', '/')
+                break
+        if not repo_url:
+            raise Exception("Unable to parse repo name")
+
+        version_info += """
+          <li> <span class="collapse"> <a href="http://{0}">{1}</a> - {2}
+                [ click for history (most recent last) ]</span>
+          <ul>""".format(repo_url, repo_dir_basename, revision)
+
+        ref_prev = None
+        for line in reflog_lines[:0:-1]:
+            ref = line.split()[0]
+            version_info += """
+              <li><span class="collapse">{ref} -
+              <a href="http://{repo}/compare/{ref}...{revision}">[diff current]</a>
+              """.format(ref=ref, repo=repo_url, revision=revision)
+
+            if ref_prev:
+                version_info += """
+                <a href="http://{repo}/compare/{ref_prev}...{ref}">[diff previous]</a>
+                """.format(repo=repo_url, ref=ref, ref_prev=ref_prev)
+            version_info += "</span></li>"
+            ref_prev = ref
+        version_info += """
+          </ul></li>"""
+
+    with open(VERSION_FILE, 'w') as f:
+        f.write(TEMPLATE.format(BODY=version_info))
+    with open(VERSION_JSON, 'w') as f:
+        f.write(json.dumps(versions, sort_keys=True, indent=4,
+                        separators=(',', ': ')))
+
+if __name__ == '__main__':
+    main()