Merge pull request #545 from edx/feanil/monitor_repos

Feanil/monitor repos

requirements.txt

@@ -9,3 +9,5 @@ ecdsa==0.10
 paramiko==1.12.0
 pycrypto==2.6.1
 wsgiref==0.1.2
+GitPython==0.3.2.RC1
+pymongo==2.4.1

util/jenkins/monitor_repos.py

+import argparse
+import json
+import logging as log
+import pickle
+import requests
+import yaml
+from datetime import datetime
+from git import Repo
+from os import path
+from pprint import pformat
+from pymongo import MongoClient, DESCENDING
+from stage_release import uri_from
+
+def releases(repo):
+    """
+    Yield a list of all release candidates from the origin.
+    """
+    for ref in repo.refs:
+        if ref.name.startswith('origin/rc/'):
+            yield ref
+
+def candidates_since(repo, time):
+    """
+    Given a repo yield a list of release candidate refs that have a
+    commit on them after the passed in time
+    """
+    for rc in releases(repo):
+        last_update = datetime.utcfromtimestamp(rc.commit.committed_date)
+        if last_update > time:
+            # New or updated RC
+            yield rc
+
+def stage_release(url, token, repo, rc):
+    """
+    Submit a job to stage a new release for the new rc of the repo.
+    """
+    # Setup the Jenkins params.
+    params = []
+    params.append({'name': "{}_REF".format(repo), 'value': True})
+    params.append({'name': repo, 'value': rc.commit.hexsha})
+    build_params = {'parameter': params}
+    log.info("New rc found{}, staging new release.".format(rc.name))
+    r = requests.post(url,
+                      data={"token", token},
+                      params={"json": json.dumps(build_params)})
+    if r.status_code != 201:
+        msg = "Failed to submit request with params: {}"
+        raise Exception(msg.format(pformat(build_params)))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Monitor git repos for new rc branches.")
+    parser.add_argument('-c', '--config', required=True,
+        help="Config file.")
+    parser.add_argument('-p', '--pickle', default="data.pickle",
+        help="Pickle of presistent data.")
+
+    args = parser.parse_args()
+
+    config = yaml.safe_load(open(args.config))
+
+    if path.exists(args.pickle):
+        data = pickle.load(open(args.pickle))
+    else:
+        data = {}
+
+    # Presist the last time we made this check.
+    if 'last_check' not in data:
+        last_check = datetime.utcnow()
+    else:
+        last_check = data['last_check']
+
+    data['last_check'] = datetime.utcnow()
+
+    # Find plays that are affected by this repo.
+    repos_with_changes = {}
+    for repo in config['repos']:
+        # Check for new rc candidates.
+        for rc in candidates_since(Repo(repo), last_check):
+            # Notify stage-release to build for the new repo.
+            stage_release(config['abbey_url'], config['abbey_token'], repo, rc)
+
+    pickle.dump(data, open(args.pickle, 'w'))

util/jenkins/stage_release.py

+"""
+Take in a YAML file with the basic data of all the things we could
+deploy and command line hashes for the repos that we want to deploy
+right now.
+
+Example Config YAML file:
+---
+DOC_STORE_CONFIG:
+    hosts: [ list, of, mongo, hosts]
+    port: #
+    db: 'db'
+    user: 'jenkins'
+    password: 'password'
+
+configuration_repo: "/path/to/configuration/repo"
+configuration_secure_repo: "/path/to/configuration-secure"
+
+repos:
+    edxapp:
+        plays:
+        - edxapp
+        - worker
+    xqueue:
+        plays:
+        - xqueue
+    6.00x:
+        plays:
+        - xserver
+    xserver:
+        plays:
+        - xserver
+
+deployments:
+    edx:
+    - stage
+    - prod
+    edge:
+    - stage
+    - prod
+    loadtest:
+    - stage
+
+# A jenkins URL to post requests for building AMIs
+abbey_url: "http://...."
+abbey_token: "API_TOKEN"
+---
+"""
+import argparse
+import json
+import yaml
+import logging as log
+from datetime import datetime
+from git import Repo
+from pprint import pformat
+from pymongo import MongoClient, DESCENDING
+
+log.basicConfig(level=log.DEBUG)
+
+def uri_from(doc_store_config):
+    """
+    Convert the below structure to a mongodb uri.
+
+    DOC_STORE_CONFIG:
+      hosts:
+        - 'host1.com'
+        - 'host2.com'
+      port: 10012
+      db: 'devops'
+      user: 'username'
+      password: 'password'
+    """
+
+    uri_format = "mongodb://{user}:{password}@{hosts}/{db}"
+    host_format = "{host}:{port}"
+
+    port = doc_store_config['port']
+    host_uris = [host_format.format(host=host,port=port) for host in doc_store_config['hosts']]
+    return uri_format.format(
+        user=doc_store_config['user'],
+        password=doc_store_config['password'],
+        hosts=",".join(host_uris),
+        db=doc_store_config['db'])
+
+def prepare_release(args):
+    config = yaml.safe_load(open(args.config))
+    client = MongoClient(uri_from(config['DOC_STORE_CONFIG']))
+    db = client[config['DOC_STORE_CONFIG']['db']]
+
+    # Get configuration repo versions
+    config_repo_ver = Repo(config['configuration_repo']).commit().hexsha
+    config_secure_ver = Repo(config['configuration_secure_repo']).commit().hexsha
+
+    # Parse the vars.
+    var_array = map(lambda key_value: key_value.split('='), args.REPOS)
+    update_repos = { item[0]:item[1] for item in var_array }
+    log.info("Update repos: {}".format(pformat(update_repos)))
+
+    release = {}
+    now = datetime.utcnow()
+    release['_id'] = args.release_id
+    release['date_created'] = now
+    release['date_modified'] = now
+    release['build_status'] = 'Unknown'
+    release['build_user'] = args.user
+
+
+    release_coll = db[args.deployment]
+    releases = release_coll.find({'build_status': 'Succeeded'}).sort('_id', DESCENDING)
+    all_plays = {}
+
+    try:
+        last_successful = releases.next()
+        all_plays = last_successful['plays']
+    except StopIteration:
+        # No successful builds.
+        log.warn("No Previously successful builds.")
+
+    # For all repos that were updated
+    for repo, ref in update_repos.items():
+        var_name = "{}_version".format(repo.replace('-','_'))
+        if repo not in config['repos']:
+            raise Exception("No info for repo with name '{}'".format(repo))
+
+        # For any play that uses the updated repo
+        for play in config['repos'][repo]:
+            if play not in all_plays:
+                all_plays[play] = {}
+
+            if 'vars' not in all_plays[play]:
+                all_plays[play]['vars'] = {}
+
+            all_plays[play]['vars'][var_name] = ref
+            # Configuration to use to build these AMIs
+            all_plays[play]['configuration_ref'] = config_repo_ver
+            all_plays[play]['configuration_secure_ref'] = config_secure_ver
+
+            # Set amis to None for all envs of this deployment
+            all_plays[play]['amis'] = {}
+            for env in config['deployments'][args.deployment]:
+                # Check the AMIs collection to see if an ami already exist
+                # for this configuration.
+                potential_ami = ami_for(db, env,
+                                        args.deployment,
+                                        play, config_repo_ver,
+                                        config_secure_ver,
+                                        ref)
+                if potential_ami:
+                    all_plays[play]['amis'][env] = potential_ami['_id']
+                else:
+                    all_plays[play]['amis'][env] = None
+
+    release['plays'] = all_plays
+    release_coll.insert(release)
+    # All plays that need new AMIs have been updated.
+    notify_abbey(config['abbey_url'], config['abbey_token'], args.deployment, all_plays, args.release_id)
+
+def ami_for(db, env, deployment, play, configuration,
+    configuration_secure, ansible_vars):
+
+    ami_signature = {
+        'env': env,
+        'deployment': deployment,
+        'play': play,
+        'configuration_ref': configuration,
+        'configuration_secure_ref': configuration_secure,
+        'vars': ansible_vars,
+    }
+
+    return db.amis.find_one(ami_signature)
+
+import requests
+def notify_abbey(abbey_url, abbey_token, deployment, all_plays, release_id):
+    for play_name, play in all_plays.items():
+        for env, ami in play['amis'].items():
+            if ami is None:
+                params = []
+                params.append({ 'name': 'play', 'value': play_name})
+                params.append({ 'name': 'deployment', 'value': deployment})
+                params.append({ 'name': 'environment', 'value': env})
+                params.append({ 'name': 'vars', 'value': yaml.dump(play['vars'], default_flow_style=False)})
+                params.append({ 'name': 'release_id', 'value': release_id})
+                build_params = {'parameter': params}
+
+                log.info("Need ami for {}".format(pformat(build_params)))
+                r = requests.post(abbey_url,
+                                  data={"token": abbey_token},
+                                  params={"json": json.dumps(build_params)})
+
+                log.info("Sent request got {}".format(r))
+                if r.status_code != 201:
+                    # Something went wrong.
+                    msg = "Failed to submit request with params: {}"
+                    raise Exception(msg.format(pformat(build_params)))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Prepare a new release.")
+    parser.add_argument('-c', '--config', required=True, help="Configuration for deploys")
+    parser.add_argument('-u', '--user', required=True, help="User staging the release.")
+    msg = "The deployment to build for eg. edx, edge, loadtest"
+    parser.add_argument('-d', '--deployment', required=True, help=msg)
+    parser.add_argument('-r', '--release-id', required=True, help="Id of Release.")
+    parser.add_argument('REPOS', nargs='+',
+        help="Any number of var=value(no spcae around '='" + \
+             " e.g. 'edxapp=3233bac xqueue=92832ab'")
+
+    args = parser.parse_args()
+    log.debug(args)
+    prepare_release(args)