Merge remote-tracking branch 'origin' into jarv/deploy

playbooks/ec2.py

@@ -137,8 +137,8 @@ class Ec2Inventory(object):
         self.index = {}
 
         # Read settings and parse CLI arguments
-        self.read_settings()
         self.parse_cli_args()
+        self.read_settings()
 
         # Cache
         if self.args.refresh_cache:
@@ -177,7 +177,7 @@ class Ec2Inventory(object):
         ''' Reads the settings from the ec2.ini file '''
 
         config = ConfigParser.SafeConfigParser()
-        config.read(os.path.dirname(os.path.realpath(__file__)) + '/ec2.ini')
+        config.read(self.args.inifile)
 
         # is eucalyptus?
         self.eucalyptus_host = None
@@ -230,6 +230,10 @@ class Ec2Inventory(object):
                            help='Get all the variables about a specific instance')
         parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)')
+
+        default_inifile = os.environ.get("ANSIBLE_EC2_INI", os.path.dirname(os.path.realpath(__file__))+'/ec2.ini')
+
+        parser.add_argument('--inifile', dest='inifile', help='Path to init script to use', default=default_inifile)
         self.args = parser.parse_args()
 
 

playbooks/edx-west/ec2.ini

 [ec2]
 regions=us-west-1
+regions_exclude = us-gov-west-1
 destination_variable=public_dns_name
 vpc_destination_variable=private_dns_name
 cache_path=/tmp
 cache_max_age=300
+route53=False

playbooks/roles/edxlocal/defaults/main.yml

+---
+edxlocal_debian_pkgs:
+  - pymongo
+  - python-mysqldb
+  - mysql-server-5.5
+  - postfix 

playbooks/roles/edxlocal/tasks/main.yml

@@ -11,14 +11,8 @@
 #  http://downloads.mysql.com/archives/mysql-5.1/mysql-5.1.62.tar.gz
 #
 ---
-- name: edxlocal | install python-pymongo (req for ansible)
-  pip: name=pymongo
-
-- name: edxlocal | install python-mysqldb (req for ansible)
-  apt: pkg=python-mysqldb state=present
-
-- name: edxlocal | install mysql server and recommends
-  apt: pkg=mysql-server-5.5 state=present install_recommends=yes
+- name: edxlocal| install packages needed for single server 
+  apt: pkg={{','.join(edxlocal_debian_pkgs)}} install_recommends=yes state=present
 
 - name: edxlocal | create a database for edxapp
   mysql_db: >

util/jenkins/ansible-provision.sh

@@ -96,8 +96,7 @@ cat $extra_vars
 cd playbooks/edx-east
 # run the tasks to launch an ec2 instance from AMI
 ansible-playbook -vvvv edx_provision.yml  -i inventory.ini -e "@${extra_vars}"  --user ubuntu
-# run tasks to update application config files that 
-# for the hostnames
+# run tasks to update application config files for the sandbox hostname
 if [[ $server_type == "full_edx_installation" ]]; then
     ansible-playbook -vvvv edx_continuous_integration.yml  -i "${dns_name}.${dns_zone}," -e "@${extra_vars}" --user ubuntu --tags "lms-env,cms-env,lms-preview-env"
 fi

util/vpc-tools/vpc-tools.py

@@ -13,6 +13,8 @@ Options:
 import boto
 from docopt import docopt
 from vpcutil import vpc_for_stack_name
+from vpcutil import stack_name_for_vpc
+from collections import defaultdict
 
 
 VERSION="vpc tools 0.1"
@@ -29,6 +31,7 @@ JUMPBOX_CONFIG = """
     """
 
 HOST_CONFIG = """
+    # Instance ID: {instance_id}
     Host {name}
       ProxyCommand ssh {config_file} -W %h:%p {jump_box}
       HostName {ip}
@@ -47,6 +50,7 @@ def dispatch(args):
 def _ssh_config(args):
     if args.get("vpc"):
       vpc_id = args.get("<vpc_id>")
+      stack_name = stack_name_for_vpc(vpc_id)
     elif args.get("stack-name"):
       stack_name = args.get("<stack_name>")
       vpc_id = vpc_for_stack_name(stack_name)
@@ -71,17 +75,23 @@ def _ssh_config(args):
     else:
       config_file = ""
 
-    jump_box = "{vpc_id}-jumpbox".format(vpc_id=vpc_id)
-    friendly = "{vpc_id}-{logical_id}-{instance_id}"
+    jump_box = "{stack_name}-jumpbox".format(stack_name=stack_name)
+    friendly = "{stack_name}-{logical_id}-{instance_number}"
+    id_type_counter = defaultdict(int)
 
     reservations = vpc.get_all_instances(filters={'vpc-id' : vpc_id})
 
     for reservation in reservations:
         for instance in reservation.instances:
 
-            logical_id = instance.__dict__['tags']['aws:cloudformation:logical-id']
+            if 'group' in instance.tags:
+                logical_id = instance.tags['group']
+            else:
+                logical_id = instance.tags['aws:cloudformation:logical-id']
+            instance_number = id_type_counter[logical_id]
+            id_type_counter[logical_id] += 1
 
-            if logical_id == "BastionHost":
+            if logical_id == "BastionHost" or logical_id == 'bastion':
 
                 print JUMPBOX_CONFIG.format(
                     jump_box=jump_box,
@@ -90,33 +100,32 @@ def _ssh_config(args):
                     identity_file=identity_file,
                     strict_host_check=strict_host_check)
 
-            else:
-                print HOST_CONFIG.format(
-                    name=instance.private_ip_address,
-                    vpc_id=vpc_id,
-                    jump_box=jump_box,
-                    ip=instance.private_ip_address,
-                    user=user,
-                    logical_id=logical_id,
-                    identity_file=identity_file,
-                    config_file=config_file,
-                    strict_host_check=strict_host_check)
+            # Print host config even for the bastion box because that is how
+            # ansible accesses it.
+            print HOST_CONFIG.format(
+                name=instance.private_ip_address,
+                jump_box=jump_box,
+                ip=instance.private_ip_address,
+                user=user,
+                identity_file=identity_file,
+                config_file=config_file,
+                strict_host_check=strict_host_check,
+                instance_id=instance.id)
 
             #duplicating for convenience with ansible
-            name = friendly.format(vpc_id=vpc_id,
+            name = friendly.format(stack_name=stack_name,
                                    logical_id=logical_id,
-                                   instance_id=instance.id)
+                                   instance_number=instance_number)
+
             print HOST_CONFIG.format(
                 name=name,
-                vpc_id=vpc_id,
                 jump_box=jump_box,
                 ip=instance.private_ip_address,
                 user=user,
-                logical_id=logical_id,
                 identity_file=identity_file,
                 config_file=config_file,
-                strict_host_check=strict_host_check)
-
+                strict_host_check=strict_host_check,
+                instance_id=instance.id)
 
 if __name__ == '__main__':
     args = docopt(__doc__, version=VERSION)

util/vpc-tools/vpcutil.py

@@ -7,3 +7,15 @@ def vpc_for_stack_name(stack_name):
       if resource.resource_type == 'AWS::EC2::VPC':
         return resource.physical_resource_id
 
+def stack_name_for_vpc(vpc_name):
+    cfn_tag_key = 'aws:cloudformation:stack-name'
+    vpc = boto.connect_vpc()
+    resource = vpc.get_all_vpcs(vpc_ids=[vpc_name])[0]
+    if cfn_tag_key in resource.tags:
+        return resource.tags[cfn_tag_key]
+    else:
+        msg = "VPC({}) is not part of a cloudformation stack.".format(vpc_name)
+        raise Exception(msg)
+    
+    
+