prereqs.py 4.8 KB
Newer Older
Will Daly committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
"""
Install Python, Ruby, and Node prerequisites.
"""

import os
import hashlib
from distutils import sysconfig
from paver.easy import *
from .utils.envs import Env


PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
NPM_REGISTRY = "http://registry.npmjs.org/"
PYTHON_REQ_FILES = [
    'requirements/edx/pre.txt',
16 17
    'requirements/edx/github.txt',
    'requirements/edx/local.txt',
Will Daly committed
18
    'requirements/edx/base.txt',
19
    'requirements/edx/post.txt',
Will Daly committed
20 21
]

22 23 24 25 26 27
# Developers can have private requirements, for local copies of github repos,
# or favorite debugging tools, etc.
PRIVATE_REQS = 'requirements/private.txt'
if os.path.exists(PRIVATE_REQS):
    PYTHON_REQ_FILES.append(PRIVATE_REQS)

Will Daly committed
28

29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
def no_prereq_install():
    """
    Determine if NO_PREREQ_INSTALL should be truthy or falsy.
    """
    vals = {
        '0': False,
        '1': True,
        'true': True,
        'false': False,
    }

    val = os.environ.get("NO_PREREQ_INSTALL", 'False').lower()

    try:
        return vals[val]
44
    except KeyError:
45 46 47
        return False


Will Daly committed
48 49 50 51 52 53 54 55 56 57
def compute_fingerprint(path_list):
    """
    Hash the contents of all the files and directories in `path_list`.
    Returns the hex digest.
    """

    hasher = hashlib.sha1()

    for path in path_list:

58 59
        # For directories, create a hash based on the modification times
        # of first-level subdirectories
Will Daly committed
60
        if os.path.isdir(path):
61 62 63 64
            for dirname in sorted(os.listdir(path)):
                p = os.path.join(path, dirname)
                if os.path.isdir(p):
                    hasher.update(str(os.stat(p).st_mtime))
Will Daly committed
65 66 67 68

        # For files, hash the contents of the file
        if os.path.isfile(path):
            with open(path, "rb") as file_handle:
69
                hasher.update(file_handle.read())
Will Daly committed
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106

    return hasher.hexdigest()


def prereq_cache(cache_name, paths, install_func):
    """
    Conditionally execute `install_func()` only if the files/directories
    specified by `paths` have changed.

    If the code executes successfully (no exceptions are thrown), the cache
    is updated with the new hash.
    """
    # Retrieve the old hash
    cache_filename = cache_name.replace(" ", "_")
    cache_file_path = os.path.join(PREREQS_MD5_DIR, "{}.sha1".format(cache_filename))
    old_hash = None
    if os.path.isfile(cache_file_path):
        with open(cache_file_path) as cache_file:
            old_hash = cache_file.read()

    # Compare the old hash to the new hash
    # If they do not match (either the cache hasn't been created, or the files have changed),
    # then execute the code within the block.
    new_hash = compute_fingerprint(paths)
    if new_hash != old_hash:
        install_func()

        # Update the cache with the new hash
        # If the code executed within the context fails (throws an exception),
        # then this step won't get executed.
        try:
            os.makedirs(PREREQS_MD5_DIR)
        except OSError:
            if not os.path.isdir(PREREQS_MD5_DIR):
                raise

        with open(cache_file_path, "w") as cache_file:
107 108 109 110
            # Since the pip requirement files are modified during the install
            # process, we need to store the hash generated AFTER the installation
            post_install_hash = compute_fingerprint(paths)
            cache_file.write(post_install_hash)
Will Daly committed
111
    else:
112
        print('{cache} unchanged, skipping...'.format(cache=cache_name))
Will Daly committed
113 114


115
def ruby_prereqs_installation():
Will Daly committed
116 117 118 119 120 121
    """
    Installs Ruby prereqs
    """
    sh('bundle install --quiet')


122
def node_prereqs_installation():
Will Daly committed
123
    """
124
    Configures npm and installs Node prerequisites
Will Daly committed
125
    """
126 127 128
    sh("test `npm config get registry` = \"{reg}\" || "
       "(echo setting registry; npm config set registry"
       " {reg})".format(reg=NPM_REGISTRY))
Will Daly committed
129 130 131
    sh('npm install')


132
def python_prereqs_installation():
Will Daly committed
133 134 135 136
    """
    Installs Python prerequisites
    """
    for req_file in PYTHON_REQ_FILES:
137
        sh("pip install -q --exists-action w -r {req_file}".format(req_file=req_file))
Will Daly committed
138 139 140


@task
141 142 143 144
def install_ruby_prereqs():
    """
    Installs Ruby prereqs
    """
145
    if no_prereq_install():
146 147
        return

148 149 150 151 152 153 154 155
    prereq_cache("Ruby prereqs", ["Gemfile"], ruby_prereqs_installation)


@task
def install_node_prereqs():
    """
    Installs Node prerequisites
    """
156
    if no_prereq_install():
157 158
        return

159 160 161 162 163 164 165 166
    prereq_cache("Node prereqs", ["package.json"], node_prereqs_installation)


@task
def install_python_prereqs():
    """
    Installs Python prerequisites
    """
167
    if no_prereq_install():
168 169
        return

170 171 172 173
    prereq_cache("Python prereqs", PYTHON_REQ_FILES + [sysconfig.get_python_lib()], python_prereqs_installation)


@task
Will Daly committed
174 175 176 177
def install_prereqs():
    """
    Installs Ruby, Node and Python prerequisites
    """
178
    if no_prereq_install():
179 180
        return

181 182 183
    install_ruby_prereqs()
    install_node_prereqs()
    install_python_prereqs()