Merge pull request #227 from edx/feanil/provision-vpc

Feanil/provision vpc

cloudformation_templates/edx-reference-architecture.json

@@ -1506,17 +1506,11 @@
         "SourceDestCheck":"false",
         "ImageId":{
           "Fn::FindInMap":[
-            "AWSRegionArch2AMI",
+            "AWSNATAMI",
             {
               "Ref":"AWS::Region"
             },
-            {
-              "Fn::FindInMap":[
-                "AWSInstanceType2Arch",
-                "t1.micro",
-                "Arch"
-              ]
-            }
+            "AMI"
           ]
         },
         "SecurityGroupIds":[

playbooks/edx-east/feanil_deploy.yml

@@ -2,7 +2,6 @@
 - hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_edxapp
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/edx_continuous_integration_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
@@ -13,7 +12,6 @@
 - hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_xserver
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/edx_continuous_integration_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
@@ -23,7 +21,6 @@
   serial: 1
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/edx_continuous_integration_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
@@ -31,7 +28,6 @@
 - hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_xqueue
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/edx_continuous_integration_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common

playbooks/roles/jenkins/tasks/jenkins.yml

@@ -53,3 +53,13 @@
     #- maven-plugin.hpi
   notify:
     - restart Jenkins
+
+- name: jenkins | install ssh key for private git repos
+  copy: >
+    src="{{jenkins_git_identity_path}}"
+    dest="{{jenkins_user_home}}/.ssh/id_rsa"
+    force=yes
+    owner="{{jenkins_user}}"
+    group="{{jenkins_user}}"
+    mode=600
+  when: c_install_ssh_key is defined

playbooks/roles/jenkins/vars/main.yml

@@ -6,6 +6,8 @@ jenkins_phantomjs_folder: phantomjs-1.9.1-linux-x86_64
 jenkins_user: "jenkins"
 jenkins_user_home: /home/jenkins
 jenkins_ruby_version: "1.9.3-p448"
+jenkins_git_identity_path: "{{secure_dir}}/files/git-identity"
+
 jenkins_debian_pkgs:
     - ruby-bundler
     - rubygems
@@ -62,4 +64,6 @@ jenkins_plugins:
     - tmpcleaner.hpi
     - token-macro.hpi
     - translation.hpi
-    - violations.hpi
\ No newline at end of file
+    - violations.hpi
+    - multiple-scms.hpi
+    - timestamper.hpi

playbooks/roles/rbenv/tasks/main.yml

@@ -60,6 +60,7 @@
     owner={{ rbenv_user }}
     group={{ rbenv_user }}
     state=directory
+    recurse=yes
 
 - name: rbenv | install build depends
   apt: pkg={{ item }} state=present install_recommends=no
@@ -179,7 +180,7 @@
   - install
 
 - name: rbenv | install ruby {{ rbenv_ruby_version }}
-  shell: "{{ rbenv_user_home }}/.rbenv/bin/rbenv install {{ rbenv_ruby_version }}"
+  shell: "{{ rbenv_user_home }}/.rbenv/bin/rbenv install {{ rbenv_ruby_version }} creates={{rbenv_user_home}}/.rbenv/versions/{{rbenv_ruby_version}}"
   when: ruby_installed|failed
   sudo: true
   sudo_user: "{{ rbenv_user }}" 
@@ -191,7 +192,6 @@
   shell: "{{rbenv_user_home }}/.rbenv/bin/rbenv global {{ rbenv_ruby_version }}"
   sudo: true
   sudo_user: "{{ rbenv_user }}"
-  when: ruby_installed|failed
   tags:
   - ruby
   - install
@@ -208,7 +208,6 @@
   shell: "{{rbenv_user_home}}/.rbenv/bin/rbenv rehash"
   sudo: true
   sudo_user: "{{ rbenv_user }}" 
-  when: ruby_installed|failed
   tags:
   - ruby
   - install

util/vpc-tools/vpc-tools.py

 """VPC Tools.
 
 Usage:
-    vpc-tools.py ssh-config vpc <vpc_id> identity-file <identity_file> user <user>
+    vpc-tools.py ssh-config vpc <vpc_id> identity-file <identity_file> user <user> [config-file <config_file>] [strict-host-check <strict_host_check>]
     vpc-tools.py (-h --help)
     vpc-tools.py (-v --version)
 
@@ -16,6 +16,7 @@ from docopt import docopt
 
 VERSION="vpc tools 0.1"
 DEFAULT_USER="ubuntu"
+DEFAULT_HOST_CHECK="yes"
 
 JUMPBOX_CONFIG = """
     Host {jump_box}
@@ -23,15 +24,17 @@ JUMPBOX_CONFIG = """
       IdentityFile {identity_file}
       ForwardAgent yes
       User {user}
+      StrictHostKeyChecking {strict_host_check}
     """
 
 HOST_CONFIG = """
     Host {name}
-      ProxyCommand ssh -W %h:%p {jump_box}
+      ProxyCommand ssh {config_file} -W %h:%p {jump_box}
       HostName {ip}
       IdentityFile {identity_file}
       ForwardAgent yes
       User {user}
+      StrictHostKeyChecking {strict_host_check}
     """
 
 
@@ -45,8 +48,21 @@ def _ssh_config(args):
     vpc = boto.connect_vpc()
 
     identity_file = args.get("<identity_file>")
-    user = args.get("<user>",DEFAULT_USER)
+    user = args.get("<user>")
     vpc_id = args.get("<vpc_id>")
+    config_file = args.get("<config_file>")
+    strict_host_check = args.get("<strict_host_check>")
+
+    if not user:
+      user = DEFAULT_USER
+
+    if not strict_host_check:
+      strict_host_check = DEFAULT_HOST_CHECK
+
+    if config_file:
+      config_file = "-F {}".format(config_file)
+    else:
+      config_file = "nothing"
 
     jump_box = "{vpc_id}-jumpbox".format(vpc_id=vpc_id)
     friendly = "{vpc_id}-{logical_id}-{instance_id}"
@@ -64,7 +80,8 @@ def _ssh_config(args):
                     jump_box=jump_box,
                     ip=instance.ip_address,
                     user=user,
-                    identity_file=identity_file)
+                    identity_file=identity_file,
+                    strict_host_check=strict_host_check)
 
             else:
                 print HOST_CONFIG.format(
@@ -74,7 +91,9 @@ def _ssh_config(args):
                     ip=instance.private_ip_address,
                     user=user,
                     logical_id=logical_id,
-                    identity_file=identity_file)
+                    identity_file=identity_file,
+                    config_file=config_file,
+                    strict_host_check=strict_host_check)
 
             #duplicating for convenience with ansible
             name = friendly.format(vpc_id=vpc_id,
@@ -87,9 +106,11 @@ def _ssh_config(args):
                 ip=instance.private_ip_address,
                 user=user,
                 logical_id=logical_id,
-                identity_file=identity_file)
+                identity_file=identity_file,
+                config_file=config_file,
+                strict_host_check=strict_host_check)
 
 
 if __name__ == '__main__':
     args = docopt(__doc__, version=VERSION)
-    dispatch(args)
\ No newline at end of file
+    dispatch(args)