git.py

import itertools
import os
import re
import socket

from functools import partial
from fabric.api import task, sudo, runs_once, execute
from fabric.api import cd, env, abort, parallel, prefix
from fabric.colors import white, green, red
from fabric.contrib import console, files
from fabric.operations import put
from fabric.utils import fastprint
from multiprocessing import Manager

from .choose import multi_choose
from .metrics import instance_tags_for_current_host
from .modifiers import rolling
from .output import notify
from .packages import PackageInfo
from .safety import noopable
from .timestamps import no_ts

REPO_URL = 'git@github.com:{}/{}'
REPO_DIRNAME = '/opt/wwc'
GIT_USER = "www-data"
AA_COMPLAIN = '/usr/sbin/aa-complain'
AA_ENFORCE = '/usr/sbin/aa-enforce'
AA_SANDBOX_POLICY = '/etc/apparmor.d/code.sandbox'
VIRTUAL_ENVS = ['/opt/edx', '/opt/edx-sandbox']


@task(default=True, aliases=['deploy'])
@runs_once
def default_deploy(**pkg_revs):
    """
    Deploys the cached or specified packages to the
    specified hosts

    Packages are installed while the server is out of the
    loadbalancer pool
    """

    if pkg_revs:
        execute('cache.from_strings', **pkg_revs)
    if socket.gethostname() != 'buildmaster-001':
        execute('git.confirm')
    execute('git.deploy')
    execute('locks.remove_all_locks')


def diff_link(pkg_org, pkg_name, old_revision, new_revision):
    if '~' in pkg_name:
        pkg_name = re.sub('~.*', '', pkg_name)

    return 'Show on github: https://github.com/{org}/{pkg}/compare/{old}' \
           '...{new}'.format(org=pkg_org,
                             pkg=pkg_name,
                             old=old_revision,
                             new=new_revision)


@task
@parallel
def diff_installed_packages(results):
    pkg_info = PackageInfo()
    # { basename(repo_dir): PackageDescriptor()) ... }
    old_pkgs = {pkg.name: pkg
                for pkg in pkg_info.installed_packages()}
    change_list = []
    for new_pkg in env.package_descriptors:
        if new_pkg.name in old_pkgs:
            old = old_pkgs[new_pkg.name].revision
            new = new_pkg.revision
            change_list.append((new_pkg.name, old, new))
        else:
            change_list.append((new_pkg.name, None, new_pkg.revision))
    results.append((tuple(change_list), env.host))


@task
@runs_once
def confirm():
    """
    Generate diffs comparing what's installed to what's about to be installed,
    and ask the user to confirm to continue.

    Answering no aborts the entire operation
    """
    execute('locks.wait_for_all_locks')
    # turn off timestamps for the confirm prompt
    with no_ts():
        manager = Manager()
        diffs = manager.list()
        execute(diff_installed_packages, diffs)

        local_diffs = list(diffs)

        def sort_key(diff):
            return diff[0]

        local_diffs.sort(key=sort_key)

        if not local_diffs:
            execute('locks.remove_all_locks')
            abort("Nothing to deploy")

        choices = set()
        pkg_info = PackageInfo()
        servers_to_update = []
        for key, group in itertools.groupby(local_diffs, key=sort_key):
            servers = [d[1] for d in group]
            servers_to_update.extend(servers)
            for pkg, old, new in key:
                choices.add(pkg)
                notify(
                    "    {name}: {diff}".format(
                        name=pkg,
                        old=old,
                        new=new,
                        diff=diff_link(pkg_info.org_from_name(pkg),
                                       pkg_info.repo_from_name(pkg), old, new),
                    ),
                    show_prefix=False
                )

        choices = list(choices)

        if len(choices) > 1:
            selection = multi_choose("Select one or more item numbers to mark"
                                     "them with a '*' for deployment.\n"
                                     "Note: none are selected by default.\n"
                                     "Select 'c' to deploy "
                                     "the items that are marked with a '*'.",
                                     choices)
        else:
            selection = choices

        if not selection:
            notify('Removing all locks and aborting')
            execute('locks.remove_all_locks')
            abort('Operation cancelled by user')

        pre_post = display_pre_post(selection)
        env.pre_post = pre_post
        notify("{noop}Updating servers [{servers}]:".format(
            servers=", ".join(servers_to_update),
            noop="[noop] " if env.noop else ''
        ), show_prefix=False)

        if not console.confirm(
                white('Please confirm the pre and post actions above',
                      bold=True),
                default=True):
            execute('locks.remove_all_locks')
            abort('Operation cancelled by user')

        env.package_descriptors = [
            pkg for pkg in env.package_descriptors if pkg.name in selection]


def display_pre_post(choices):

    """
    Displays list of pre and post checkout commands,
    returns the list of commands in a dictionary

        return({
                'pre': [ 'cmd1', 'cmd2', ... ],
                'post': [ 'cmd1', 'cmd2', ... ]
            })

    """

    pkg_info = PackageInfo()
    pre_post = pkg_info.pre_post_actions(choices)

    for stage in ['pre', 'post']:
        if pre_post[stage]:
            fastprint(green('{0}-checkout commands:\n'.format(stage),
                      bold=True) + green('  -> ') + green('\n  -> '.join(
                      pre_post[stage])) + white('\n\n'))
        else:
            fastprint(green('WARNING', bold=True) +
                      green(' - no {0}-checkout commands for this set '
                      'of packages : '.format(stage, choices)) +
                      white('\n\n'))
    return pre_post


@task
@rolling
def deploy(auto_migrate=False):
    """
    Deploys the cached packages to the specified hosts.
    Packages are installed while the server is out of the loadbalancer pool
    """

    packages = env.package_descriptors

    # If these are not defined it means that the confirm
    # step was skipped, in this case we figure out pre and
    # post steps here
    if not hasattr(env, 'pre_post'):
        pkg_config = PackageInfo()
        env.pre_post = pkg_config.pre_post_actions([pkg.name
                                                   for pkg in packages])

    contains_content = any(pkg.name.startswith('content') for pkg in packages)
    contains_code = not all(pkg.name.startswith('content') for pkg in packages)

    base_tags = [
        'deploying_to_host:' + env.host,
    ] + instance_tags_for_current_host()

    if contains_content:
        base_tags.append('type:content')
    if contains_code:
        base_tags.append('type:code')

    package_tags = base_tags + ['package:' + pkg.name for pkg in packages]
    metric_name = 'fabric.deployment'

    # pre checkout commands
    with prefix("export GIT_SSH=/tmp/git.sh"):
        for cmd in env.pre_post['pre']:
            noopable(sudo)(cmd)

    put(os.path.join(os.path.dirname(__file__), 'git.sh'),
        '/tmp/git.sh', mode=0755, use_sudo=True)
    for pkg in packages:
        existing_repo = files.exists(pkg.repo_root, use_sudo=True)

        repo_tags = base_tags + [
            'package:' + pkg.name,
            'existance:' + 'existing' if existing_repo else 'absent',
        ]

        if existing_repo:
            if not files.exists(os.path.join(pkg.repo_root, '.git'),
                                use_sudo=True):
                raise Exception("Repo root not a git repo - {0}".format(
                    os.path.join(pkg.repo_root, '.git')))
            with cd(pkg.repo_root):
                if pkg.revision == 'absent':
                    noopable(sudo)('rm -rf {0}'.format(pkg.repo_root))
                else:
                    checkout(pkg.revision)
        else:
            with cd(os.path.dirname(pkg.repo_root)):
                if pkg.revision != 'absent':
                    clone(pkg.repo_org, pkg.repo_name, pkg.name, pkg.revision)
        if '~' in pkg.name:
            _update_course_xml(pkg, pkg.name.split('~')[1])

        _install_requirements(pkg)
        _install_gemfile(pkg)
        _install_npm_package(pkg)

        # drop a file for puppet so it knows that
        # code is installed for the service
#        with cd('/etc/facter/facts.d'):
#            pkg_config = PackageInfo()
#            if pkg.repo_name in pkg_config.service_repos:
#                # facts can't have dashes so they are converted
#                # to underscores
#                noopable(sudo)(
#                    'echo "{0}_installed=true" > {0}_installed.txt'.format(
#                    pkg.repo_name.replace("-", "_")))

    pkg_version()

    # post checkout commands
    with prefix("export GIT_SSH=/tmp/git.sh"):
        for cmd in env.pre_post['post']:
            noopable(sudo)(cmd)

    if 'mitx' in [pkg.name for pkg in packages]:
        # do not slow down content deploys by checking
        # for migrations
        execute('migrate_check.migrate_check', auto_migrate)


@task
def pkg_version():
    """
    Drops an html/json version file on the remote system
    """
    path = os.path.dirname(os.path.abspath(__file__))
    with open(os.path.join(path, '../version-script/version.py')) as f:
        put(f, '/tmp/version.py', use_sudo=True)

    noopable(sudo)('/usr/bin/python /tmp/version.py')


@task
def clone(repo_org, repo_name, name, revision):
    """
    Parameters:
        repo_name: github organization
        repo_name: github repo name
        name: basename(repo_dir) ex: content-mit-600x~Fall_2012
        revision: commit hash
    """

    with no_ts():
        with prefix("export GIT_SSH=/tmp/git.sh"):
            noopable(sudo)("git clone {} {}".format(
                REPO_URL.format(repo_org, repo_name), name))
            with cd(name):
                noopable(sudo)("git reset --hard {0}".format(revision))
                if files.exists('.gitmodules', use_sudo=True):
                    noopable(sudo)("git submodule update --init")
                noopable(sudo)("chown -R {0}:{0} .".format(GIT_USER))


@task
def checkout(revision):
    """
    Parameters:
        revision: commit hash
    """

    with no_ts():
        with prefix("export GIT_SSH=/tmp/git.sh"):
            noopable(sudo)("git remote prune origin")
            noopable(sudo)("git fetch origin")
            noopable(sudo)("git reset --hard {0}".format(revision))
            if files.exists('.gitmodules', use_sudo=True):
                noopable(sudo)("git submodule update --init")
            noopable(sudo)("chown -R {0}:{0} .".format(GIT_USER))


def _update_course_xml(pkg, root):
    """
    Parameters:
        pkg: a single package descriptor
        root: a root that must exist in the roots/ folder
    """
    with cd(pkg.repo_root):
        if files.exists(
                os.path.join(pkg.repo_root, 'roots/{0}.xml'.format(root)),
                use_sudo=True):
            noopable(sudo)('rm -f course.xml && '
                           'ln -s roots/{0}.xml course.xml'.format(root))
        else:
            abort(red("There is a '~' in {0} but there is no roots/{1}.xml "
                      "file in the repo!".format(pkg.name, root)))


def _install_requirements(pkg):
    """
    Parameters:
        pkg: single package descriptor

    install pre-requirements and requirements
    if they exists for the repo.
    will not run pip install if the requirements file
    has not changed since the last run

    Turns off apparmor enforcement during pip install
    for the sandbox virtualenv
    """

    def pip_install(file, venv='/opt/edx'):
        with prefix("export GIT_SSH=/tmp/git.sh"):
            with prefix('source {}'.format(os.path.join(venv, 'bin/activate'))):
                with prefix('export PIP_DOWNLOAD_CACHE=/tmp/pip_download_cache'):
                    noopable(sudo)('pip install --exists-action w -r {0}'.format(file))

    if files.exists(AA_COMPLAIN) and files.exists(AA_SANDBOX_POLICY):
        # in order to install code-sandbox requirements the
        # code sandbox apparmor policy must be temporarily
        # suspended
        sudo('{0} {1}'.format(AA_COMPLAIN, AA_SANDBOX_POLICY))

    # Run new-style requirements
    for venv in VIRTUAL_ENVS:
        if not files.exists(venv):
            # skip if the virtualenv doesn't exist
            continue
        venv_name = os.path.basename(venv)
        pip_cmd_base = partial(pip_install, file='requirements/{}/base.txt'.format(venv_name), venv=venv)
        pip_cmd_post = partial(pip_install, file='requirements/{}/post.txt'.format(venv_name), venv=venv)
        _run_if_changed(pkg, 'requirements/{}/base.txt'.format(venv_name), pip_cmd_base,
                        'cat requirements/{}/*.txt'.format(venv_name))
        _run_if_changed(pkg, 'requirements/{}/post.txt'.format(venv_name), pip_cmd_post,
                        'cat requirements/{}/*.txt'.format(venv_name))

    if files.exists(AA_ENFORCE) and files.exists(AA_SANDBOX_POLICY):
        sudo('{0} {1}'.format(AA_ENFORCE, AA_SANDBOX_POLICY))

@task
@runs_once
def deploy_with_puppet():
    execute('git.confirm')
    execute(_deploy_with_puppet)
    execute('locks.remove_all_locks')


@task
@rolling
def _deploy_with_puppet():
    execute('puppet')
    execute('git.deploy')


def _install_gemfile(pkg):
    """
    Parameters:
        pkg: single package descriptor

    Installs the Gemfile from the repo, if it exists.
    Will not run if the Gemfile
    has not changed since the last run
    """
    def bundle_install():
        with prefix('export PATH=/opt/www/.rbenv/bin:$PATH'):
            with prefix('RBENV_ROOT=/opt/www/.rbenv'):
                with prefix('which rbenv'):
                    with prefix('eval "$(rbenv init -)"'):
                        noopable(sudo)('gem install bundler')
                        noopable(sudo)('bundle install --binstubs')

    _run_if_changed(pkg, 'Gemfile', bundle_install)


def _install_npm_package(pkg):
    """
    Parameters:
        pkg: single package descriptor

    Installs the package.json from the repo, if it exists.
    Will not run if the package.json has not changed since
    the last run
    """
    def package_install():
        noopable(sudo)('npm install')

    _run_if_changed(pkg, 'package.json', package_install)


def _run_if_changed(pkg, file, command, contents_command=None):
    """
    Runs command if the contents of file
    inside pkg have changed since the last time the command was run.
    Commands are executed inside pkg.repo_root
    """
    if contents_command is None:
        contents_command = 'cat ' + file

    with cd(pkg.repo_root):
        path = os.path.join(pkg.repo_root, file)
        if files.exists(path, use_sudo=True):
            prev_md5_file = '/var/tmp/{0}-{1}.md5'.format(
                pkg.repo_name.replace('/', '-'), file.replace('/', '-'))
            md5_command = '{} | /usr/bin/md5sum'.format(contents_command)
            if files.exists(prev_md5_file, use_sudo=True):
                cur_md5 = sudo(md5_command)
                prev_md5 = sudo('cat {0}'.format(prev_md5_file))
                if cur_md5 == prev_md5:
                    return

            command()
            noopable(sudo)('{} > {}'.format(md5_command, prev_md5_file))