resolving conflicts, updating from master

playbooks/roles/jenkins_master/defaults/main.yml

@@ -4,6 +4,9 @@ jenkins_group: "edx"
 jenkins_server_name: "jenkins.testeng.edx.org"
 jenkins_port: 8080
 
+jenkins_deb_url: "http://pkg.jenkins-ci.org/debian/binary/jenkins_1.530_all.deb"
+jenkins_deb: "jenkins_1.530_all.deb"
+
 jenkins_plugins:
     - { name: "build-name-setter", version: "1.3" }
     - { name: "build-pipeline-plugin", version: "1.4" }
@@ -38,6 +41,12 @@ jenkins_plugins:
     - { name: "multiple-scms", version: "0.2" }
     - { name: "timestamper", version: "1.5.7" }
 
+jenkins_bundled_plugins:
+    - "credentials"
+    - "git"
+    - "ssh-credentials"
+    - "ssh-slaves"
+
 jenkins_custom_plugins:
     - { repo_name: "git-client-plugin",
         repo_url: "https://github.com/edx/git-client-plugin.git",
@@ -53,4 +62,5 @@ jenkins_debian_pkgs:
     - nginx
     - git
     - maven
+    - daemon
     - python-pycurl

playbooks/roles/jenkins_master/tasks/main.yml

@@ -13,14 +13,11 @@
 - name: jenkins_master | Add the jenkins user to the group
   user: name={{ jenkins_user }} append=yes groups={{ jenkins_group }}
 
-- name: jenkins_master | Install Jenkins apt key
-  apt_key: url=http://pkg.jenkins-ci.org/debian/jenkins-ci.org.key state=present
+- name: jenkins_master | Download Jenkins package
+  get_url: url="{{ jenkins_deb_url }}" dest="/tmp/{{ jenkins_deb }}"
 
-- name: jenkins_master | Add apt repository
-  apt_repository: repo='deb http://pkg.jenkins-ci.org/debian binary/' state=present
-
-- name: jenkins_master | Install jenkins package
-  apt: pkg=jenkins state=present update_cache=yes
+- name: jenkins_master | Install Jenkins package
+  command: dpkg -i --force-depends "/tmp/{{ jenkins_deb }}"
 
 - name: jenkins_master | Stop Jenkins
   service: name=jenkins state=stopped
@@ -89,6 +86,16 @@
         owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
   with_items: "{{ jenkins_custom_plugins }}"
 
+
+# Plugins that are bundled with Jenkins are "pinned".
+# Jenkins will overwrite updated plugins with its built-in version
+# unless we create a ".pinned" file for the plugin.
+# See https://issues.jenkins-ci.org/browse/JENKINS-13129
+- name: jenkins_master | Create plugin pin files
+  command: touch {{ jenkins_home }}/plugins/${item}.jpi.pinned
+           creates={{ jenkins_home }}/plugins/${item}.jpi.pinned
+  with_items: "{{ jenkins_bundled_plugins }}"
+
 - name: jenkins_master | Setup nginix vhost
   template:
     src=etc/nginx/sites-available/jenkins.j2

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -31,6 +31,12 @@
         owner={{ jenkins_user }} group={{ jenkins_group }}
         mode=400
 
+# Ensure that we get a current version of Git
+# GitHub requires version 1.7.10 or later
+# https://help.github.com/articles/https-cloning-errors
+- name: jenkins_worker | Add git apt repository
+  apt_repository: repo='ppa:git-core/ppa'
+
 - name: jenkins_worker | Install system packages
   apt: pkg={{','.join(jenkins_debian_pkgs)}}
        state=present update_cache=yes

playbooks/roles/nginx/templates/xqueue.j2

@@ -14,6 +14,10 @@ server {
     try_files $uri @proxy_to_app;
   }
 
+  # No basic auth security on the heartbeat url, so that ELB can use it
+  location /xqueue/status/{
+    try_files $uri @proxy_to_app;
+  }
 
 location @proxy_to_app {
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;

playbooks/roles/xqueue/defaults/main.yml

@@ -68,6 +68,8 @@ xqueue_auth_config:
       PASSWORD: $XQUEUE_MYSQL_PASSWORD
       HOST: $XQUEUE_MYSQL_HOST
       PORT: $XQUEUE_MYSQL_PORT
+  RABBITMQ_USER: $XQUEUE_RABBITMQ_USER
+  RABBITMQ_PASS: $XQUEUE_RABBITMQ_PASS
 
 xqueue_create_db: 'yes'
 xqueue_source_repo: https://github.com/edx/xqueue.git

playbooks/roles/xserver/defaults/main.yml

@@ -4,10 +4,10 @@
 XSERVER_NGINX_PORT: 18050
 XSERVER_BASIC_AUTH: False
 
-RABBIT_RUN_URL: ''
-RABBIT_GRADER_ROOT: ''
-RABBIT_LOGGING_ENV: 'sandbox'
-RABBIT_SYSLOG_SERVER: ''
+XSERVER_RUN_URL: ''
+XSERVER_GRADER_ROOT: ''
+XSERVER_LOGGING_ENV: 'sandbox'
+XSERVER_SYSLOG_SERVER: ''
 # by default do not check out the content
 # repo needed on the xserver for grading
 # python submissions, TODO: replace with an open
@@ -15,11 +15,11 @@ RABBIT_SYSLOG_SERVER: ''
 XSERVER_GRADER_CHECKOUT: False
 
 xserver_env_config:
-  RUN_URL: $RABBIT_RUN_URL
-  GRADER_ROOT: $RABBIT_GRADER_ROOT
-  LOGGING_ENV: $RABBIT_LOGGING_ENV
+  RUN_URL: $XSERVER_RUN_URL
+  GRADER_ROOT: $XSERVER_GRADER_ROOT
+  LOGGING_ENV: $XSERVER_LOGGING_ENV
   LOG_DIR: "{{ data_dir }}/logs/xserver"
-  SYSLOG_SERVER: $RABBIT_SYSLOG_SERVER
+  SYSLOG_SERVER: $XSERVER_SYSLOG_SERVER
   SANDBOX_PYTHON: '/opt/edx_apparmor_sandbox/bin/python'
 
 xserver_git_identity_path: "{{ secure_dir }}/files/git-identity"
@@ -38,7 +38,7 @@ xserver_sandbox_venv_dir: "{{ venv_dir }}_apparmor_sandbox"
 xserver_requirements_file: "{{ xserver_code_dir }}/requirements.txt"
 
 xserver_gunicorn_port: 8050
-xserver_gunicorn_host: 8050
+xserver_gunicorn_host: 'localhost'
 
 xserver_debian_pkgs:
   - build-essential

util/vpc-tools/create_stack.py

+import argparse
+import boto
+from os.path import basename
+from time import sleep
+
+FAILURE_STATES = [
+    'CREATE_FAILED',
+    'ROLLBACK_IN_PROGRESS',
+    'ROLLBACK_FAILED',
+    'ROLLBACK_COMPLETE',
+    'DELETE_IN_PROGRESS',
+    'DELETE_FAILED',
+    'DELETE_COMPLETE',
+    ]
+
+def upload_file(file_path, bucket_name, key_name):
+    """
+    Upload a file to the given s3 bucket and return a template url.
+    """
+    conn = boto.connect_s3()
+    try:
+        bucket = conn.get_bucket(bucket_name)
+    except boto.exception.S3ResponseError as e:
+        conn.create_bucket(bucket_name)
+        bucket = conn.get_bucket(bucket_name, validate=False)
+
+    key = boto.s3.key.Key(bucket)
+    key.key = key_name 
+    key.set_contents_from_filename(file_path)
+
+    url = 'https://s3.amazonaws.com/{}/{}'.format(bucket_name, key_name)
+    return url
+
+def create_stack(stack_name, template, region='us-east-1', blocking=True, temp_bucket='edx-sandbox-devops'):
+    cfn = boto.connect_cloudformation()
+
+    # Upload the template to s3
+    key_name = 'cloudformation/auto/{}_{}'.format(stack_name, basename(template))
+    template_url = upload_file(template, temp_bucket, key_name)
+
+    # Reference the stack.
+    try:
+        stack_id = cfn.create_stack(stack_name,
+            template_url=template_url,
+            capabilities=['CAPABILITY_IAM'],
+            tags={'autostack':'true'},
+            parameters=[('KeyName', 'continuous-integration')])
+    except Exception as e:
+        print(e.message)
+        raise e
+
+   
+    status = None 
+    while blocking:
+        sleep(5)
+        stack_instance = cfn.describe_stacks(stack_id)[0]
+        status = stack_instance.stack_status
+        print(status)
+        if 'COMPLETE' in status:
+            break
+
+    if status in FAILURE_STATES:
+        raise Exception('Creation Failed. Stack Status: {}, ID:{}'.format(
+            status, stack_id))
+
+    return stack_id
+
+
+if __name__ == '__main__':
+        description = 'Create a cloudformation stack from a template.'
+        parser = argparse.ArgumentParser(description=description)
+
+        msg = 'Name for the cloudformation stack.'
+        parser.add_argument('-n', '--stackname', required=True, help=msg)
+
+        msg = 'Name of the bucket to use for temporarily uploading the \
+            template.'
+        parser.add_argument('-b', '--bucketname', default="edx-sandbox-devops",
+            help=msg)
+
+        msg = 'The path to the cloudformation template.'
+        parser.add_argument('-t', '--template', required=True, help=msg)
+       
+        msg = 'The AWS region to build this stack in.'
+        parser.add_argument('-r', '--region', default='us-east-1', help=msg)
+       
+        args = parser.parse_args() 
+        stack_name = args.stackname
+        template = args.template
+        region = args.region
+        bucket_name = args.bucketname
+
+        create_stack(stack_name, template, region, bucket_name)
+        print('Stack({}) created.'.format(stack_name))

util/vpc-tools/vpc-dns.py

-"""vpc-dns.py
-
-Usage:
-    vpc-dns.py create-zone (vpc <vpc_id> | stack-name <stack_name>)
-    vpc-dns.py (-h --help)
-    vpc-dns.py (-v --version)
-
-Options:
-    -h --help       Show this screen.
-    -v --version    Show version.
-"""
-import boto
-from boto.route53.record import ResourceRecordSets
-from docopt import docopt
-from vpcutil import vpc_for_stack_name
-
-class VPCDns:
-    BACKEND_ZONE = "Z4AI6ADZTL3HN"
-    DNS_SUFFIX = ".vpc.edx.org"
-    ZONE = "{name}" + DNS_SUFFIX
-
-    def __init__(self,vpc_id=None):
-        self.vpc_id = vpc_id
-        self.elb = boto.connect_elb()
-        self.r53 = boto.connect_route53()
-
-    def create_zone(self, vpc_id):
-        zone_name = self.ZONE.format(name=vpc_id)
-        print zone_name
-        hosted_zone = self.get_or_create_hosted_zone(zone_name)
-
-        elbs = self.elb.get_all_load_balancers()
-
-        for elb in [x for x in elbs if x.vpc_id == self.vpc_id]:
-            self.create_service_dns(elb,self.get_zone_id_from_retval(
-                hosted_zone.Id),self.vpc_id)
-
-    def get_zone_id_from_retval(self,retval):
-        """
-        The data structure returned by the create_hosted_zone call
-        pre-pends the string /hostedzone/ to the Id for some reason.
-        """
-        return retval.replace("/hostedzone/","")
-
-
-    def get_or_create_hosted_zone(self, zone_name):
-
-        hosted_zone = self.r53.get_hosted_zone_by_name(zone_name)
-
-        if not hosted_zone:
-            zone_data = self.r53.create_hosted_zone(zone_name,
-                                        comment="Created by automation.")
-            hosted_zone = self.r53.get_hosted_zone_by_name(zone_name)
-
-        return hosted_zone
-
-
-    def get_elb_service(self, elb):
-
-        services = ["edxapp","rabbit","xqueue","xserver","worker"]
-
-        for service in services:
-            if service in elb.dns_name.lower():
-                return service
-
-        raise Exception("No service mapping for " + elb.dns_name)
-
-
-    def create_service_dns(self, elb, zone, vpc_id):
-        """
-        """
-        records = self.r53.get_all_rrsets(zone)
-
-        old_names = [r.name for r in records]
-
-        HOST_TEMPLATE = "{service}.{vpc_id}" + self.DNS_SUFFIX
-
-        service = self.get_elb_service(elb)
-
-        dns_name = HOST_TEMPLATE.format(service=service,
-                                        vpc_id=vpc_id)
-
-        change_set = ResourceRecordSets()
-
-        if dns_name + '.' in old_names:
-            print "adding delete"
-            change = change_set.add_change(
-                'DELETE',
-                dns_name,
-                'CNAME',
-                600)
-
-            change.add_value(elb.dns_name)
-
-        change = change_set.add_change(
-            'CREATE',
-            dns_name,
-            'CNAME',
-            600 )
-
-        change.add_value(elb.dns_name)
-
-        print change_set.to_xml()
-
-        self.r53.change_rrsets(zone, change_set.to_xml())
-
-VERSION="0.1"
-
-def dispatch(args):
-    if args.get("vpc"):
-      vpc_id = args.get("<vpc_id>")
-    elif args.get("stack-name"):
-      stack_name = args.get("<stack_name>")
-      vpc_id = vpc_for_stack_name(stack_name)
-    else:
-      raise Exception("No vpc_id or stack_name provided.")
-
-    c = VPCDns(vpc_id=vpc_id)
-
-    if args.get("create-zone"):
-        c.create_zone(vpc_id)
-
-if __name__ == "__main__":
-    args = docopt(__doc__, version=VERSION)
-    dispatch(args)
-

util/vpc-tools/vpc_dns.py

+import argparse
+import boto
+from vpcutil import vpc_for_stack_name
+from pprint import pprint
+
+r53 = boto.connect_route53()
+
+
+# Utility Functions
+def add_or_update_record(zone, record_name, record_type, record_ttl, record_values):
+    zone_id = zone.Id.replace("/hostedzone/","")
+
+    records = r53.get_all_rrsets(zone_id)
+
+    old_records = { r.name[:-1] : r for r in records }
+    pprint(old_records)
+
+    change_set = boto.route53.record.ResourceRecordSets()
+
+    # If the record name already points to something.
+    # Delete the existing connection.
+    if record_name in old_records.keys():
+        print "adding delete"
+        change = change_set.add_change(
+            'DELETE',
+            record_name,
+            record_type,
+            record_ttl)
+
+        for value in old_records[record_name].resource_records:
+            change.add_value(value)
+
+    change = change_set.add_change(
+        'CREATE',
+        record_name,
+        record_type,
+        record_ttl)
+
+    for value in record_values:
+        change.add_value(value)
+     
+    print(change_set.to_xml())
+    r53.change_rrsets(zone_id, change_set.to_xml())
+
+
+def add_zone_to_parent(zone, parent):
+    #Add a reference for the new zone to its parent zone.
+    parent_name = parent.Name[:-1]
+    zone_name = zone.Name[:-1]
+
+    add_or_update_record(parent, zone_name, 'NS', 900, zone.NameServers)
+
+
+def get_or_create_hosted_zone(zone_name):
+    # Get the parent zone.
+    parent_zone_name = ".".join(zone_name.split('.')[1:])
+    parent_zone = r53.get_hosted_zone_by_name(parent_zone_name)
+    if not parent_zone:
+        msg = "Parent zone({}) does not exist."
+        raise Exception(msg.format(parent_zone_name))
+
+    hosted_zone = r53.get_hosted_zone_by_name(zone_name)
+
+    if not hosted_zone:
+        r53.create_hosted_zone(zone_name,
+                               comment="Created by automation.")
+        hosted_zone = r53.get_hosted_zone_by_name(zone_name)
+
+    add_zone_to_parent(hosted_zone, parent_zone)
+
+    return hosted_zone
+
+
+def elbs_for_stack_name(stack_name):
+    vpc_id = vpc_for_stack_name(stack_name)
+    elbs = boto.connect_elb()
+    for elb in elbs.get_all_load_balancers():
+        if elb.vpc_id == vpc_id:
+            yield elb
+
+
+def ensure_service_dns(elb, prefix, zone):
+    dns_template = "{prefix}.{zone_name}"
+
+    # Have to remove the trailing period that is on zone names.
+    zone_name = zone.Name[:-1]
+    dns_name = dns_template.format(prefix=prefix,
+                                   zone_name=zone_name)
+
+    add_or_update_record(zone, dns_name, 'CNAME', 600, [elb.dns_name])
+
+
+if __name__ == "__main__":
+    description = "Give a cloudformation stack name, for an edx stack, setup \
+        DNS names for the ELBs in the stack."
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('-n', '--stackname',
+        help="The name of the cloudformation stack.",
+        required=True)
+    args = parser.parse_args()
+    stack_name = args.stackname
+
+    # Create DNS for edxapp and xqueue.
+    dns_settings = {
+        'edxapp': ['courses', 'studio'],
+        'xqueue': ['xqueue'],
+        'rabbit': ['rabbit'],
+        'xserver': ['xserver'],
+        'worker': ['worker'],
+        'forum': ['forum'],
+    }
+
+    # Create a zone for the stack.
+    zone_name = "{}.vpc.edx.org".format(stack_name)
+
+    zone = get_or_create_hosted_zone(zone_name)
+
+    stack_elbs = elbs_for_stack_name(stack_name)
+    for elb in stack_elbs:
+        for role, dns_prefixes in dns_settings.items():
+            #FIXME this breaks when the service name is in the stack name ie. testforumstack.
+            # Get the tags for the instances in this elb and compare the service against the role tag.
+            if role in elb.dns_name.lower():
+                for prefix in dns_prefixes:
+                    ensure_service_dns(elb, prefix, zone)