release.py 19.9 KB
Newer Older
David Baumgold committed
1 2 3 4
#!/usr/bin/env python
"""
a release-master multitool
"""
5
from __future__ import print_function, unicode_literals
6

David Baumgold committed
7
import argparse
David Baumgold committed
8 9
import collections
import functools
10
import getpass
11 12 13 14 15
import json
import re
import sys
import textwrap
from datetime import date, timedelta
16 17

try:
18
    from path import Path as path
19 20 21 22
    from git import Repo, Commit
    from git.refs.symbolic import SymbolicReference
    from dateutil.parser import parse as parse_datestring
    import requests
23
    import yaml
24 25
except ImportError:
    print("Error: missing dependencies! Please run this command to install them:")
26
    print("pip install path.py requests python-dateutil GitPython PyYAML")
27 28
    sys.exit(1)

David Baumgold committed
29 30 31 32
try:
    from pygments.console import colorize
except ImportError:
    colorize = lambda color, text: text
David Baumgold committed
33 34

JIRA_RE = re.compile(r"\b[A-Z]{2,}-\d+\b")
35
PR_BRANCH_RE = re.compile(r"remotes/edx/pr/(\d+)")
36 37 38 39 40 41 42 43 44 45


def project_root():
    directory = path(__file__).abspath().dirname()
    while not (directory / ".git").exists():
        directory = directory.parent
    return directory


PROJECT_ROOT = project_root()
David Baumgold committed
46 47 48
repo = Repo(PROJECT_ROOT)
git = repo.git

49
PEOPLE_YAML = "https://raw.githubusercontent.com/edx/repo-tools-data/master/people.yaml"
David Baumgold committed
50

51

David Baumgold committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
class memoized(object):
    """
    Decorator. Caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned
    (not reevaluated).

    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
    """
    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):
        if not isinstance(args, collections.Hashable):
            # uncacheable. a list, for instance.
            # better to not cache than blow up.
            return self.func(*args)
        if args in self.cache:
            return self.cache[args]
        else:
            value = self.func(*args)
            self.cache[args] = value
            return value

    def __repr__(self):
        '''Return the function's docstring.'''
        return self.func.__doc__

    def __get__(self, obj, objtype):
        '''Support instance methods.'''
        return functools.partial(self.__call__, obj)


David Baumgold committed
85 86 87
def make_parser():
    parser = argparse.ArgumentParser(description="release master multitool")
    parser.add_argument(
88 89
        '--previous', '--prev', '-p', metavar="GITREV", default="edx/release",
        help="previous release [%(default)s]")
David Baumgold committed
90 91
    parser.add_argument(
        '--current', '--curr', '-c', metavar="GITREV", default="HEAD",
92
        help="current release candidate [%(default)s]")
David Baumgold committed
93 94 95 96 97 98 99 100 101 102 103 104 105
    parser.add_argument(
        '--date', '-d',
        help="expected release date: defaults to "
        "next Tuesday [{}]".format(default_release_date()))
    parser.add_argument(
        '--merge', '-m', action="store_true", default=False,
        help="include merge commits")
    parser.add_argument(
        '--table', '-t', action="store_true", default=False,
        help="only print table")
    return parser


106
def ensure_pr_fetch():
107 108 109 110 111 112
    """
    Make sure that the git repository contains a remote called "edx" that has
    two fetch URLs; one for the main codebase, and one for pull requests.
    Returns True if the environment was modified in any way, False otherwise.
    """
    modified = False
113
    remotes = git.remote().splitlines()
stv committed
114
    if 'edx' not in remotes:
115
        git.remote("add", "edx", "https://github.com/edx/edx-platform.git")
116
        modified = True
117 118
    # it would be nice to use the git-python API to do this, but it doesn't seem
    # to support configurations with more than one value per key. :(
119 120 121 122
    edx_fetches = git.config("remote.edx.fetch", get_all=True).splitlines()
    pr_fetch = '+refs/pull/*/head:refs/remotes/edx/pr/*'
    if pr_fetch not in edx_fetches:
        git.config("remote.edx.fetch", pr_fetch, add=True)
123
        modified = True
124
    git.fetch("edx")
125 126 127 128 129
    return modified


def get_github_creds():
    """
130
    Returns GitHub credentials if they exist, as a two-tuple of (username, token).
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
    Otherwise, return None.
    """
    netrc_auth = requests.utils.get_netrc_auth("https://api.github.com")
    if netrc_auth:
        return netrc_auth
    config_file = path("~/.config/edx-release").expand()
    if config_file.isfile():
        with open(config_file) as f:
            config = json.load(f)
        github_creds = config.get("credentials", {}).get("api.github.com", {})
        username = github_creds.get("username", "")
        token = github_creds.get("token", "")
        if username and token:
            return (username, token)
    return None


148
def create_github_creds():
David Baumgold committed
149 150 151
    """
    https://developer.github.com/v3/oauth_authorizations/#create-a-new-authorization
    """
152
    headers = {"User-Agent": "edx-release"}
153 154 155 156
    payload = {
        "note": "edx-release",
        "scopes": ["repo"],
    }
157 158
    username = raw_input("GitHub username: ")
    password = getpass.getpass("GitHub password: ")
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
    response = requests.post(
        "https://api.github.com/authorizations",
        auth=(username, password),
        headers=headers, data=json.dumps(payload),
    )
    # is the user using two-factor authentication?
    otp_header = response.headers.get("X-GitHub-OTP")
    if not response.ok and otp_header and otp_header.startswith("required;"):
        # get two-factor code, redo the request
        headers["X-GitHub-OTP"] = raw_input("Two-factor authentication code: ")
        response = requests.post(
            "https://api.github.com/authorizations",
            auth=(username, password),
            headers=headers, data=json.dumps(payload),
        )
    if not response.ok:
175 176 177 178
        message = response.json()["message"]
        if message != "Validation Failed":
            raise requests.exceptions.RequestException(message)
        else:
179
            # A token called "edx-release" already exists on GitHub.
180 181 182 183 184 185 186 187 188 189 190 191 192
            # Delete it, and try again.
            token_id = get_github_auth_id(username, password, "edx-release")
            if token_id:
                delete_github_auth_token(username, password, token_id)
            response = requests.post(
                "https://api.github.com/authorizations",
                auth=(username, password),
                headers=headers, data=json.dumps(payload),
            )
    if not response.ok:
        message = response.json()["message"]
        raise requests.exceptions.RequestException(message)

193 194 195
    return (username, response.json()["token"])


196 197
def get_github_auth_id(username, password, note):
    """
198
    Return the ID associated with the GitHub auth token with the given note.
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
    If no such auth token exists, return None.
    """
    response = requests.get(
        "https://api.github.com/authorizations",
        auth=(username, password),
        headers={"User-Agent": "edx-release"},
    )
    if not response.ok:
        message = response.json()["message"]
        raise requests.exceptions.RequestException(message)

    for auth_token in response.json():
        if auth_token["note"] == "edx-release":
            return auth_token["id"]
    return None


def delete_github_auth_token(username, password, token_id):
    response = requests.delete(
        "https://api.github.com/authorizations/{id}".format(id=token_id),
        auth=(username, password),
        headers={"User-Agent": "edx-release"},
    )
    if not response.ok:
        message = response.json()["message"]
        raise requests.exceptions.RequestException(message)


David Baumgold committed
227
def ensure_github_creds(attempts=3):
228
    """
229
    Make sure that we have GitHub OAuth credentials. This will check the user's
230 231 232 233 234 235 236 237 238 239
    .netrc file, as well as the ~/.config/edx-release file. If no credentials
    exist in either place, it will prompt the user to create OAuth credentials,
    and store them in ~/.config/edx-release.

    Returns False if we found credentials, True if we had to create them.
    """
    if get_github_creds():
        return False

    # Looks like we need to create the OAuth creds
240
    print("We need to set up OAuth authentication with GitHub's API. "
241 242
          "Your password will not be stored.", file=sys.stderr)
    token = None
David Baumgold committed
243
    for _ in range(attempts):
244 245 246
        try:
            username, token = create_github_creds()
        except requests.exceptions.RequestException as e:
247
            print(
248
                "Invalid authentication: {}".format(e.message),
249 250 251 252 253
                file=sys.stderr,
            )
            continue
        else:
            break
254
    if token:
255
        print("Successfully authenticated to GitHub", file=sys.stderr)
256
    if not token:
257
        print("Too many invalid authentication attempts.", file=sys.stderr)
David Baumgold committed
258
        return False
259 260 261 262 263 264 265 266 267 268 269

    config_file = path("~/.config/edx-release").expand()
    # make sure parent directory exists
    config_file.parent.makedirs_p()
    # read existing config if it exists
    if config_file.isfile():
        with open(config_file) as f:
            config = json.load(f)
    else:
        config = {}
    # update config
stv committed
270
    if 'credentials' not in config:
271
        config["credentials"] = {}
stv committed
272
    if 'api.github.com' not in config['credentials']:
273 274 275 276 277 278 279 280
        config["credentials"]["api.github.com"] = {}
    config["credentials"]["api.github.com"]["username"] = username
    config["credentials"]["api.github.com"]["token"] = token
    # write it back out
    with open(config_file, "w") as f:
        json.dump(config, f)

    return True
281 282


David Baumgold committed
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
def default_release_date():
    """
    Returns a date object corresponding to the expected date of the next release:
    normally, this Tuesday.
    """
    today = date.today()
    TUESDAY = 2
    days_until_tuesday = (TUESDAY - today.isoweekday()) % 7
    return today + timedelta(days=days_until_tuesday)


def parse_ticket_references(text):
    """
    Given a commit message, return a list of all JIRA ticket references in that
    message. If there are no ticket references, return an empty list.
    """
299
    return set(JIRA_RE.findall(text))
David Baumgold committed
300 301


302 303 304 305 306
class DoesNotExist(Exception):
    def __init__(self, message, commit, branch):
        self.message = message
        self.commit = commit
        self.branch = branch
307
        Exception.__init__(self, message)
308 309


310 311 312 313 314 315 316 317 318 319 320 321 322 323
def get_merge_commit(commit, branch="master"):
    """
    Given a commit that was merged into the given branch, return the merge commit
    for that event.

    http://stackoverflow.com/questions/8475448/find-merge-commit-which-include-a-specific-commit
    """
    commit_range = "{}..{}".format(commit, branch)
    ancestry_paths = git.rev_list(commit_range, ancestry_path=True).splitlines()
    first_parents = git.rev_list(commit_range, first_parent=True).splitlines()
    both = set(ancestry_paths) & set(first_parents)
    for commit_hash in reversed(ancestry_paths):
        if commit_hash in both:
            return repo.commit(commit_hash)
324 325
    # no merge commit!
    msg = "No merge commit for {commit} in {branch}!".format(
326
        commit=commit, branch=branch,
327 328
    )
    raise DoesNotExist(msg, commit, branch)
329

330 331 332

def get_pr_info(num):
    """
333
    Returns the info from the GitHub API
334 335
    """
    url = "https://api.github.com/repos/edx/edx-platform/pulls/{num}".format(num=num)
336
    username, token = get_github_creds()
337
    headers = {
338
        "Authorization": "token {}".format(token),
339 340 341
        "User-Agent": "edx-release",
    }
    response = requests.get(url, headers=headers)
342 343 344 345 346 347 348
    result = response.json()
    if not response.ok:
        raise requests.exceptions.RequestException(result["message"])
    return result


def get_merged_prs(start_ref, end_ref):
349
    """
350 351
    Return the set of all pull requests (as integers) that were merged between
    the start_ref and end_ref.
352
    """
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
    ensure_pr_fetch()
    start_unmerged_branches = set(
        branch.strip() for branch in
        git.branch(all=True, no_merged=start_ref).splitlines()
    )
    end_merged_branches = set(
        branch.strip() for branch in
        git.branch(all=True, merged=end_ref).splitlines()
    )
    merged_between_refs = start_unmerged_branches & end_merged_branches
    merged_prs = set()
    for branch in merged_between_refs:
        match = PR_BRANCH_RE.search(branch)
        if match:
            merged_prs.add(int(match.group(1)))
    return merged_prs
369 370


David Baumgold committed
371
@memoized
372
def prs_by_email(start_ref, end_ref):
373 374 375 376 377 378
    """
    Returns an ordered dictionary of {email: pr_list}
    Email is the email address of the person who merged the pull request
    The dictionary is alphabetically ordered by email address
    The pull request list is ordered by merge date
    """
379 380 381 382 383
    username, token = get_github_creds()
    headers = {
        "Authorization": "token {}".format(token),
        "User-Agent": "edx-release",
    }
384 385
    # `emails` maps from other_emails to primary email, based on people.yaml.
    emails = {}
386 387 388 389 390 391 392
    people_resp = requests.get(PEOPLE_YAML, headers=headers)
    people_resp.raise_for_status()
    people = yaml.safe_load(people_resp.text)
    for person in people.itervalues():
        if 'other_emails' in person:
            for other_email in person['other_emails']:
                emails[other_email] = person['email']
393

David Baumgold committed
394
    unordered_data = collections.defaultdict(set)
395 396
    for pr_num in get_merged_prs(start_ref, end_ref):
        ref = "refs/remotes/edx/pr/{num}".format(num=pr_num)
397
        branch = SymbolicReference(repo, ref)
398 399
        try:
            merge = get_merge_commit(branch.commit, end_ref)
400 401
        except DoesNotExist:
            pass  # this commit will be included in the commits_without_prs table
402
        else:
403
            email = emails.get(merge.author.email, merge.author.email)
404 405 406 407 408 409 410 411
            if email.endswith("@users.noreply.github.com"):
                # A bogus GitHub address, look up their GitHub name in
                # people.yaml
                username = email.split("@")[0]
                try:
                    email = people[username]['email']
                except KeyError:
                    pass
412
            unordered_data[email].add((pr_num, merge))
413

David Baumgold committed
414
    ordered_data = collections.OrderedDict()
415 416 417 418 419 420
    for email in sorted(unordered_data.keys()):
        ordered = sorted(unordered_data[email], key=lambda pair: pair[1].authored_date)
        ordered_data[email] = [num for num, merge in ordered]
    return ordered_data


421
def generate_pr_table(start_ref, end_ref):
422
    """
423
    Return a UTF-8 string corresponding to a pull request table to embed in Confluence.
424
    """
425
    header = "|| Merged By || Author || Title || PR || JIRA || Release Notes? || Verified? ||"
426
    pr_link = "[#{num}|https://github.com/edx/edx-platform/pull/{num}]"
David Baumgold committed
427
    user_link = "[@{user}|https://github.com/{user}]"
428
    rows = [header]
429
    prbe = prs_by_email(start_ref, end_ref)
430 431 432 433 434 435
    for email, pull_requests in prbe.items():
        for i, pull_request in enumerate(pull_requests):
            try:
                pr_info = get_pr_info(pull_request)
                title = pr_info["title"] or ""
                body = pr_info["body"] or ""
David Baumgold committed
436
                author = pr_info["user"]["login"]
437
            except requests.exceptions.RequestException as e:
David Baumgold committed
438 439 440
                message = (
                    "Warning: could not fetch data for #{num}: "
                    "{message}".format(num=pull_request, message=e.message)
441
                )
David Baumgold committed
442
                print(colorize("red", message), file=sys.stderr)
443 444
                title = "?"
                body = "?"
David Baumgold committed
445
                author = ""
446
            rows.append("| {merged_by} | {author} | {title} | {pull_request} | {jira} | {release_notes} | {verified} |".format(
447
                merged_by=email if i == 0 else "",
David Baumgold committed
448
                author=user_link.format(user=author) if author else "",
449
                title=title.replace("|", "\|").replace('{', '\{').replace('}', '\}'),
450 451
                pull_request=pr_link.format(num=pull_request),
                jira=", ".join(parse_ticket_references(body)),
452
                release_notes="",
453 454
                verified="",
            ))
455
    return "\n".join(rows).encode("utf8")
David Baumgold committed
456 457


458 459 460
@memoized
def get_commits_not_in_prs(start_ref, end_ref):
    """
David Baumgold committed
461
    Return a tuple of commits that exist between start_ref and end_ref,
462
    but were not merged to the end_ref. If everyone is following the
David Baumgold committed
463
    pull request process correctly, this should return an empty tuple.
464
    """
David Baumgold committed
465
    return tuple(Commit.iter_items(
466 467 468 469 470 471 472 473 474 475 476 477
        repo,
        "{start}..{end}".format(start=start_ref, end=end_ref),
        first_parent=True, no_merges=True,
    ))


def generate_commit_table(start_ref, end_ref):
    """
    Return a string corresponding to a commit table to embed in Comfluence.
    The commits in the table should only be commits that are not in the
    pull request table.
    """
478
    header = "|| Author || Summary || Commit || JIRA || Release Notes? || Verified? ||"
479 480 481 482
    commit_link = "[commit|https://github.com/edx/edx-platform/commit/{sha}]"
    rows = [header]
    commits = get_commits_not_in_prs(start_ref, end_ref)
    for commit in commits:
483
        rows.append("| {author} | {summary} | {commit} | {jira} | {release_notes} | {verified} |".format(
484 485 486 487
            author=commit.author.email,
            summary=commit.summary.replace("|", "\|"),
            commit=commit_link.format(sha=commit.hexsha),
            jira=", ".join(parse_ticket_references(commit.message)),
488
            release_notes="",
489 490 491 492 493
            verified="",
        ))
    return "\n".join(rows)


494
def generate_email(start_ref, end_ref, release_date=None):
David Baumgold committed
495 496 497 498 499
    """
    Returns a string roughly approximating an email.
    """
    if release_date is None:
        release_date = default_release_date()
500
    prbe = prs_by_email(start_ref, end_ref)
David Baumgold committed
501 502 503 504

    email = """
        To: {emails}

David Baumgold committed
505 506 507 508
        You merged at least one pull request for edx-platform that is going out
        in this upcoming release, and you are responsible for verifying those
        changes on the staging servers before the code is released. Please go
        to the release page to do so:
David Baumgold committed
509

510
        https://openedx.atlassian.net/wiki/display/ENG/{date}+Release
David Baumgold committed
511

512
        The staging server is: https://stage.edx.org
David Baumgold committed
513 514 515 516 517 518 519

        Note that you are responsible for verifying any pull requests that you
        merged, whether you wrote the code or not. (If you didn't write the code,
        you can and should try to get the person who wrote the code to help
        verify the changes -- but even if you can't, you're still responsible!)
        If you find any bugs, please notify me and record the bugs on the
        release page. Thanks!
520 521 522

        By the way, if you have an @edx.org email address and are having trouble logging
        into stage, you may need to reset your password.
523 524 525 526

        If you would prefer this email be sent to a different email address of yours,
        send a request to oscm@edx.org with the details.

David Baumgold committed
527
    """.format(
528
        emails=", ".join(prbe.keys()),
David Baumgold committed
529 530 531 532 533 534 535 536 537 538 539 540
        date=release_date.isoformat(),
    )
    return textwrap.dedent(email).strip()


def main():
    parser = make_parser()
    args = parser.parse_args()
    if isinstance(args.date, basestring):
        # user passed in a custom date, so we need to parse it
        args.date = parse_datestring(args.date).date()

541 542
    ensure_github_creds()

David Baumgold committed
543
    if args.table:
544
        print(generate_pr_table(args.previous, args.current))
David Baumgold committed
545 546
        return

547
    print("Generating stage verification email and its list of recipients. This may take around a minute...")
548
    print(generate_email(args.previous, args.current, release_date=args.date).encode('UTF-8'))
David Baumgold committed
549 550 551 552 553 554 555
    print("\n")
    print("Wiki Table:")
    print(
        "Type Ctrl+Shift+D on Confluence to embed the following table "
        "in your release wiki page"
    )
    print("\n")
556
    print(generate_pr_table(args.previous, args.current))
David Baumgold committed
557
    commits_without_prs = get_commits_not_in_prs(args.previous, args.current)
558 559 560 561 562 563 564 565 566 567 568 569 570 571
    if commits_without_prs:
        num = len(commits_without_prs)
        plural = num > 1
        print("\n")
        print(
            "There {are} {num} {commits} in this release that did not come in "
            "through pull requests!".format(
                num=num, are="are" if plural else "is",
                commits="commits" if plural else "commit"
            )
        )
        print("\n")
        print(generate_commit_table(args.previous, args.current))

David Baumgold committed
572 573 574

if __name__ == "__main__":
    main()