release.py 18 KB
Newer Older
David Baumgold committed
1 2 3 4
#!/usr/bin/env python
"""
a release-master multitool
"""
5 6
from __future__ import print_function, unicode_literals
import sys
David Baumgold committed
7 8 9
import argparse
from datetime import date, timedelta
import re
David Baumgold committed
10 11
import collections
import functools
David Baumgold committed
12
import textwrap
13 14
import json
import getpass
15 16 17 18 19 20 21 22 23 24 25 26

try:
    from path import path
    from git import Repo, Commit
    from git.refs.symbolic import SymbolicReference
    from dateutil.parser import parse as parse_datestring
    import requests
except ImportError:
    print("Error: missing dependencies! Please run this command to install them:")
    print("pip install path.py requests python-dateutil GitPython==0.3.2.RC1")
    sys.exit(1)

David Baumgold committed
27 28 29 30
try:
    from pygments.console import colorize
except ImportError:
    colorize = lambda color, text: text
David Baumgold committed
31 32

JIRA_RE = re.compile(r"\b[A-Z]{2,}-\d+\b")
33
PR_BRANCH_RE = re.compile(r"remotes/edx/pr/(\d+)")
David Baumgold committed
34 35 36 37 38
PROJECT_ROOT = path(__file__).abspath().dirname()
repo = Repo(PROJECT_ROOT)
git = repo.git


David Baumgold committed
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
class memoized(object):
    """
    Decorator. Caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned
    (not reevaluated).

    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
    """
    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):
        if not isinstance(args, collections.Hashable):
            # uncacheable. a list, for instance.
            # better to not cache than blow up.
            return self.func(*args)
        if args in self.cache:
            return self.cache[args]
        else:
            value = self.func(*args)
            self.cache[args] = value
            return value

    def __repr__(self):
        '''Return the function's docstring.'''
        return self.func.__doc__

    def __get__(self, obj, objtype):
        '''Support instance methods.'''
        return functools.partial(self.__call__, obj)


David Baumgold committed
72 73 74
def make_parser():
    parser = argparse.ArgumentParser(description="release master multitool")
    parser.add_argument(
75 76
        '--previous', '--prev', '-p', metavar="GITREV", default="edx/release",
        help="previous release [%(default)s]")
David Baumgold committed
77 78
    parser.add_argument(
        '--current', '--curr', '-c', metavar="GITREV", default="HEAD",
79
        help="current release candidate [%(default)s]")
David Baumgold committed
80 81 82 83 84 85 86 87 88 89 90 91 92
    parser.add_argument(
        '--date', '-d',
        help="expected release date: defaults to "
        "next Tuesday [{}]".format(default_release_date()))
    parser.add_argument(
        '--merge', '-m', action="store_true", default=False,
        help="include merge commits")
    parser.add_argument(
        '--table', '-t', action="store_true", default=False,
        help="only print table")
    return parser


93
def ensure_pr_fetch():
94 95 96 97 98 99
    """
    Make sure that the git repository contains a remote called "edx" that has
    two fetch URLs; one for the main codebase, and one for pull requests.
    Returns True if the environment was modified in any way, False otherwise.
    """
    modified = False
100 101 102
    remotes = git.remote().splitlines()
    if not "edx" in remotes:
        git.remote("add", "edx", "https://github.com/edx/edx-platform.git")
103
        modified = True
104 105
    # it would be nice to use the git-python API to do this, but it doesn't seem
    # to support configurations with more than one value per key. :(
106 107 108 109
    edx_fetches = git.config("remote.edx.fetch", get_all=True).splitlines()
    pr_fetch = '+refs/pull/*/head:refs/remotes/edx/pr/*'
    if pr_fetch not in edx_fetches:
        git.config("remote.edx.fetch", pr_fetch, add=True)
110
        modified = True
111
    git.fetch("edx")
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
    return modified


def get_github_creds():
    """
    Returns Github credentials if they exist, as a two-tuple of (username, token).
    Otherwise, return None.
    """
    netrc_auth = requests.utils.get_netrc_auth("https://api.github.com")
    if netrc_auth:
        return netrc_auth
    config_file = path("~/.config/edx-release").expand()
    if config_file.isfile():
        with open(config_file) as f:
            config = json.load(f)
        github_creds = config.get("credentials", {}).get("api.github.com", {})
        username = github_creds.get("username", "")
        token = github_creds.get("token", "")
        if username and token:
            return (username, token)
    return None


135
def create_github_creds():
David Baumgold committed
136 137 138
    """
    https://developer.github.com/v3/oauth_authorizations/#create-a-new-authorization
    """
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
    headers = {"User-Agent": "edx-release"}
    payload = {"note": "edx-release"}
    username = raw_input("Github username: ")
    password = getpass.getpass("Github password: ")
    response = requests.post(
        "https://api.github.com/authorizations",
        auth=(username, password),
        headers=headers, data=json.dumps(payload),
    )
    # is the user using two-factor authentication?
    otp_header = response.headers.get("X-GitHub-OTP")
    if not response.ok and otp_header and otp_header.startswith("required;"):
        # get two-factor code, redo the request
        headers["X-GitHub-OTP"] = raw_input("Two-factor authentication code: ")
        response = requests.post(
            "https://api.github.com/authorizations",
            auth=(username, password),
            headers=headers, data=json.dumps(payload),
        )
    if not response.ok:
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
        message = response.json()["message"]
        if message != "Validation Failed":
            raise requests.exceptions.RequestException(message)
        else:
            # A token called "edx-release" already exists on Github.
            # Delete it, and try again.
            token_id = get_github_auth_id(username, password, "edx-release")
            if token_id:
                delete_github_auth_token(username, password, token_id)
            response = requests.post(
                "https://api.github.com/authorizations",
                auth=(username, password),
                headers=headers, data=json.dumps(payload),
            )
    if not response.ok:
        message = response.json()["message"]
        raise requests.exceptions.RequestException(message)

177 178 179
    return (username, response.json()["token"])


180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
def get_github_auth_id(username, password, note):
    """
    Return the ID associated with the Github auth token with the given note.
    If no such auth token exists, return None.
    """
    response = requests.get(
        "https://api.github.com/authorizations",
        auth=(username, password),
        headers={"User-Agent": "edx-release"},
    )
    if not response.ok:
        message = response.json()["message"]
        raise requests.exceptions.RequestException(message)

    for auth_token in response.json():
        if auth_token["note"] == "edx-release":
            return auth_token["id"]
    return None


def delete_github_auth_token(username, password, token_id):
    response = requests.delete(
        "https://api.github.com/authorizations/{id}".format(id=token_id),
        auth=(username, password),
        headers={"User-Agent": "edx-release"},
    )
    if not response.ok:
        message = response.json()["message"]
        raise requests.exceptions.RequestException(message)


David Baumgold committed
211
def ensure_github_creds(attempts=3):
212 213 214 215 216 217 218 219 220 221 222 223 224
    """
    Make sure that we have Github OAuth credentials. This will check the user's
    .netrc file, as well as the ~/.config/edx-release file. If no credentials
    exist in either place, it will prompt the user to create OAuth credentials,
    and store them in ~/.config/edx-release.

    Returns False if we found credentials, True if we had to create them.
    """
    if get_github_creds():
        return False

    # Looks like we need to create the OAuth creds
    print("We need to set up OAuth authentication with Github's API. "
225 226
          "Your password will not be stored.", file=sys.stderr)
    token = None
David Baumgold committed
227
    for _ in range(attempts):
228 229 230
        try:
            username, token = create_github_creds()
        except requests.exceptions.RequestException as e:
231
            print(
232
                "Invalid authentication: {}".format(e.message),
233 234 235 236 237
                file=sys.stderr,
            )
            continue
        else:
            break
238 239 240
    if token:
        print("Successfully authenticated to Github", file=sys.stderr)
    if not token:
241
        print("Too many invalid authentication attempts.", file=sys.stderr)
David Baumgold committed
242
        return False
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264

    config_file = path("~/.config/edx-release").expand()
    # make sure parent directory exists
    config_file.parent.makedirs_p()
    # read existing config if it exists
    if config_file.isfile():
        with open(config_file) as f:
            config = json.load(f)
    else:
        config = {}
    # update config
    if not "credentials" in config:
        config["credentials"] = {}
    if not "api.github.com" in config["credentials"]:
        config["credentials"]["api.github.com"] = {}
    config["credentials"]["api.github.com"]["username"] = username
    config["credentials"]["api.github.com"]["token"] = token
    # write it back out
    with open(config_file, "w") as f:
        json.dump(config, f)

    return True
265 266


David Baumgold committed
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
def default_release_date():
    """
    Returns a date object corresponding to the expected date of the next release:
    normally, this Tuesday.
    """
    today = date.today()
    TUESDAY = 2
    days_until_tuesday = (TUESDAY - today.isoweekday()) % 7
    return today + timedelta(days=days_until_tuesday)


def parse_ticket_references(text):
    """
    Given a commit message, return a list of all JIRA ticket references in that
    message. If there are no ticket references, return an empty list.
    """
283
    return set(JIRA_RE.findall(text))
David Baumgold committed
284 285


286 287 288 289 290 291 292
class DoesNotExist(Exception):
    def __init__(self, message, commit, branch):
        self.message = message
        self.commit = commit
        self.branch = branch


293 294 295 296 297 298 299 300 301 302 303 304 305 306
def get_merge_commit(commit, branch="master"):
    """
    Given a commit that was merged into the given branch, return the merge commit
    for that event.

    http://stackoverflow.com/questions/8475448/find-merge-commit-which-include-a-specific-commit
    """
    commit_range = "{}..{}".format(commit, branch)
    ancestry_paths = git.rev_list(commit_range, ancestry_path=True).splitlines()
    first_parents = git.rev_list(commit_range, first_parent=True).splitlines()
    both = set(ancestry_paths) & set(first_parents)
    for commit_hash in reversed(ancestry_paths):
        if commit_hash in both:
            return repo.commit(commit_hash)
307 308
    # no merge commit!
    msg = "No merge commit for {commit} in {branch}!".format(
309
        commit=commit, branch=branch,
310 311
    )
    raise DoesNotExist(msg, commit, branch)
312

313 314 315 316 317 318

def get_pr_info(num):
    """
    Returns the info from the Github API
    """
    url = "https://api.github.com/repos/edx/edx-platform/pulls/{num}".format(num=num)
319
    username, token = get_github_creds()
320
    headers = {
321
        "Authorization": "token {}".format(token),
322 323 324
        "User-Agent": "edx-release",
    }
    response = requests.get(url, headers=headers)
325 326 327 328 329 330 331
    result = response.json()
    if not response.ok:
        raise requests.exceptions.RequestException(result["message"])
    return result


def get_merged_prs(start_ref, end_ref):
332
    """
333 334
    Return the set of all pull requests (as integers) that were merged between
    the start_ref and end_ref.
335
    """
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
    ensure_pr_fetch()
    start_unmerged_branches = set(
        branch.strip() for branch in
        git.branch(all=True, no_merged=start_ref).splitlines()
    )
    end_merged_branches = set(
        branch.strip() for branch in
        git.branch(all=True, merged=end_ref).splitlines()
    )
    merged_between_refs = start_unmerged_branches & end_merged_branches
    merged_prs = set()
    for branch in merged_between_refs:
        match = PR_BRANCH_RE.search(branch)
        if match:
            merged_prs.add(int(match.group(1)))
    return merged_prs
352 353


David Baumgold committed
354
@memoized
355
def prs_by_email(start_ref, end_ref):
356 357 358 359 360 361
    """
    Returns an ordered dictionary of {email: pr_list}
    Email is the email address of the person who merged the pull request
    The dictionary is alphabetically ordered by email address
    The pull request list is ordered by merge date
    """
David Baumgold committed
362
    unordered_data = collections.defaultdict(set)
363 364
    for pr_num in get_merged_prs(start_ref, end_ref):
        ref = "refs/remotes/edx/pr/{num}".format(num=pr_num)
365
        branch = SymbolicReference(repo, ref)
366 367
        try:
            merge = get_merge_commit(branch.commit, end_ref)
368 369
        except DoesNotExist:
            pass  # this commit will be included in the commits_without_prs table
370 371
        else:
            unordered_data[merge.author.email].add((pr_num, merge))
372

David Baumgold committed
373
    ordered_data = collections.OrderedDict()
374 375 376 377 378 379
    for email in sorted(unordered_data.keys()):
        ordered = sorted(unordered_data[email], key=lambda pair: pair[1].authored_date)
        ordered_data[email] = [num for num, merge in ordered]
    return ordered_data


380
def generate_pr_table(start_ref, end_ref):
381
    """
382
    Return a string corresponding to a pull request table to embed in Confluence
383
    """
David Baumgold committed
384
    header = "|| Merged By || Author || Title || PR || JIRA || Verified? ||"
385
    pr_link = "[#{num}|https://github.com/edx/edx-platform/pull/{num}]"
David Baumgold committed
386
    user_link = "[@{user}|https://github.com/{user}]"
387
    rows = [header]
388
    prbe = prs_by_email(start_ref, end_ref)
389 390 391 392 393 394
    for email, pull_requests in prbe.items():
        for i, pull_request in enumerate(pull_requests):
            try:
                pr_info = get_pr_info(pull_request)
                title = pr_info["title"] or ""
                body = pr_info["body"] or ""
David Baumgold committed
395
                author = pr_info["user"]["login"]
396
            except requests.exceptions.RequestException as e:
David Baumgold committed
397 398 399
                message = (
                    "Warning: could not fetch data for #{num}: "
                    "{message}".format(num=pull_request, message=e.message)
400
                )
David Baumgold committed
401
                print(colorize("red", message), file=sys.stderr)
402 403
                title = "?"
                body = "?"
David Baumgold committed
404 405
                author = ""
            rows.append("| {merged_by} | {author} | {title} | {pull_request} | {jira} | {verified} |".format(
406
                merged_by=email if i == 0 else "",
David Baumgold committed
407
                author=user_link.format(user=author) if author else "",
408 409 410 411 412 413
                title=title.replace("|", "\|"),
                pull_request=pr_link.format(num=pull_request),
                jira=", ".join(parse_ticket_references(body)),
                verified="",
            ))
    return "\n".join(rows)
David Baumgold committed
414 415


416 417 418
@memoized
def get_commits_not_in_prs(start_ref, end_ref):
    """
David Baumgold committed
419
    Return a tuple of commits that exist between start_ref and end_ref,
420
    but were not merged to the end_ref. If everyone is following the
David Baumgold committed
421
    pull request process correctly, this should return an empty tuple.
422
    """
David Baumgold committed
423
    return tuple(Commit.iter_items(
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
        repo,
        "{start}..{end}".format(start=start_ref, end=end_ref),
        first_parent=True, no_merges=True,
    ))


def generate_commit_table(start_ref, end_ref):
    """
    Return a string corresponding to a commit table to embed in Comfluence.
    The commits in the table should only be commits that are not in the
    pull request table.
    """
    header = "|| Author || Summary || Commit || JIRA || Verified? ||"
    commit_link = "[commit|https://github.com/edx/edx-platform/commit/{sha}]"
    rows = [header]
    commits = get_commits_not_in_prs(start_ref, end_ref)
    for commit in commits:
        rows.append("| {author} | {summary} | {commit} | {jira} | {verified} |".format(
            author=commit.author.email,
            summary=commit.summary.replace("|", "\|"),
            commit=commit_link.format(sha=commit.hexsha),
            jira=", ".join(parse_ticket_references(commit.message)),
            verified="",
        ))
    return "\n".join(rows)


451
def generate_email(start_ref, end_ref, release_date=None):
David Baumgold committed
452 453 454 455 456
    """
    Returns a string roughly approximating an email.
    """
    if release_date is None:
        release_date = default_release_date()
457
    prbe = prs_by_email(start_ref, end_ref)
David Baumgold committed
458 459 460 461

    email = """
        To: {emails}

David Baumgold committed
462 463 464 465
        You merged at least one pull request for edx-platform that is going out
        in this upcoming release, and you are responsible for verifying those
        changes on the staging servers before the code is released. Please go
        to the release page to do so:
David Baumgold committed
466

467
        https://openedx.atlassian.net/wiki/display/ENG/{date}+Release
David Baumgold committed
468

469
        The staging server is: https://www.stage.edx.org
David Baumgold committed
470 471 472 473 474 475 476

        Note that you are responsible for verifying any pull requests that you
        merged, whether you wrote the code or not. (If you didn't write the code,
        you can and should try to get the person who wrote the code to help
        verify the changes -- but even if you can't, you're still responsible!)
        If you find any bugs, please notify me and record the bugs on the
        release page. Thanks!
David Baumgold committed
477
    """.format(
478
        emails=", ".join(prbe.keys()),
David Baumgold committed
479 480 481 482 483 484 485 486 487 488 489 490
        date=release_date.isoformat(),
    )
    return textwrap.dedent(email).strip()


def main():
    parser = make_parser()
    args = parser.parse_args()
    if isinstance(args.date, basestring):
        # user passed in a custom date, so we need to parse it
        args.date = parse_datestring(args.date).date()

491 492
    ensure_github_creds()

David Baumgold committed
493
    if args.table:
494
        print(generate_pr_table(args.previous, args.current))
David Baumgold committed
495 496 497
        return

    print("EMAIL:")
498
    print(generate_email(args.previous, args.current, release_date=args.date).encode('UTF-8'))
David Baumgold committed
499 500 501 502 503 504 505
    print("\n")
    print("Wiki Table:")
    print(
        "Type Ctrl+Shift+D on Confluence to embed the following table "
        "in your release wiki page"
    )
    print("\n")
506
    print(generate_pr_table(args.previous, args.current))
David Baumgold committed
507
    commits_without_prs = get_commits_not_in_prs(args.previous, args.current)
508 509 510 511 512 513 514 515 516 517 518 519 520 521
    if commits_without_prs:
        num = len(commits_without_prs)
        plural = num > 1
        print("\n")
        print(
            "There {are} {num} {commits} in this release that did not come in "
            "through pull requests!".format(
                num=num, are="are" if plural else "is",
                commits="commits" if plural else "commit"
            )
        )
        print("\n")
        print(generate_commit_table(args.previous, args.current))

David Baumgold committed
522 523 524

if __name__ == "__main__":
    main()