Merge pull request #453 from edx/feanil/pre_stage

Feanil/pre stage

cloudformation_templates/edx-reference-architecture.json

@@ -1974,6 +1974,12 @@
             "CidrIp":"0.0.0.0/0"
           },
           {
+            "IpProtocol":"tcp",
+            "FromPort":"11371",
+            "ToPort":"11371",
+            "CidrIp":"0.0.0.0/0"
+          },
+          {
             "IpProtocol":"icmp",
             "FromPort":"-1",
             "ToPort":"-1",
@@ -2018,6 +2024,12 @@
             "FromPort":"10016",
             "ToPort":"10016",
             "CidrIp":"0.0.0.0/0"
+          },
+          {
+            "IpProtocol":"tcp",
+            "FromPort":"11371",
+            "ToPort":"11371",
+            "CidrIp":"0.0.0.0/0"
           }
         ]
       }
@@ -4864,52 +4876,6 @@
       "Value":{
         "Ref":"EdxappServerSecurityGroup"
       }
-    },
-    "DatabaseConfigurationString":{
-      "Description":"JDBC connection string for database",
-      "Value":{
-        "Fn::Join":[
-          "",
-          [
-            "'DATABASES': {\n",
-            "    'default': {\n",
-            "        'ENGINE': 'django.db.backends.mysql',\n",
-            "        'NAME': '",
-            {
-              "Ref":"DBName"
-            },
-            "',\n",
-            "        'USER': '",
-            {
-              "Ref":"DBUsername"
-            },
-            "',\n",
-            "        'PASSWORD': '",
-            {
-              "Ref":"DBPassword"
-            },
-            "',\n",
-            "        'HOST': '",
-            {
-              "Fn::GetAtt":[
-                "EdxDB",
-                "Endpoint.Address"
-              ]
-            },
-            "',\n",
-            "        'PORT': '",
-            {
-              "Fn::GetAtt":[
-                "EdxDB",
-                "Endpoint.Port"
-              ]
-            },
-            "'\n",
-            "    }\n",
-            "}\n"
-          ]
-        ]
-      }
     }
   }
 }

playbooks/edx-east/edx_vpc.yml

@@ -2,29 +2,31 @@
 - hosts: first_in_tag_role_mongo
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
-    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/common/common.yml"
   roles:
     - common
+    - gh_users
     - role: 'mongo'
       mongo_create_users: yes
 #- hosts: tag_role_mongo:!first_in_tag_role_mongo
 #  sudo: True
 #  vars_files:
-#    - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
-#    - "{{ secure_dir }}/vars/users.yml"
+#    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
+#    - "{{ secure_dir }}/vars/common/common.yml"
 #  roles:
 #    - common
+#    - gh_users
 #    - mongo
 - hosts: first_in_tag_role_edxapp
   sudo: True
   serial: 1
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - datadog
     - supervisor
     - role: nginx
@@ -36,17 +38,16 @@
       edxapp_lms_env: 'lms.envs.load_test'
       migrate_db: 'yes'
       openid_workaround: 'yes'
-      edx_platform_commit: 'HEAD'
     - splunkforwarder
 - hosts: tag_role_edxapp:!first_in_tag_role_edxapp
   sudo: True
   serial: 1
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - datadog
     - supervisor
     - role: nginx
@@ -56,16 +57,15 @@
       - lms-preview
     - role: 'edxapp'
       edxapp_lms_env: 'lms.envs.load_test'
-      edx_platform_commit: 'HEAD'
     - splunkforwarder
 - hosts: tag_role_worker
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - datadog
     - supervisor
     - role: nginx
@@ -76,16 +76,15 @@
     - role: 'edxapp'
       edxapp_lms_env: 'lms.envs.load_test'
       celery_worker: True
-      edx_platform_commit: 'HEAD'
     - splunkforwarder
 - hosts: tag_role_xserver
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - supervisor
     - role: nginx
       nginx_sites:
@@ -96,22 +95,22 @@
   serial: 1
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - supervisor
     - rabbitmq
     - splunkforwarder
 - hosts: first_in_tag_role_xqueue
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - supervisor
     - role: nginx
       nginx_sites:
@@ -122,11 +121,11 @@
 - hosts: tag_role_xqueue:!first_in_tag_role_xqueue
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - supervisor
     - role: nginx
       nginx_sites:
@@ -136,11 +135,11 @@
 - hosts: tag_role_forum
   sudo: True
   vars_files:
-    - "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
+    - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
-    - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - gh_users
     - supervisor
     - oraclejdk
     - elasticsearch

playbooks/jenkins_master.yml

@@ -9,4 +9,6 @@
   vars:
     COMMON_DATA_DIR: "/mnt"
   roles:
+    - common
+    - gh_users
     - jenkins_master

playbooks/roles/edxapp/tasks/deploy.yml

@@ -237,7 +237,7 @@
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
   sudo_user: "{{ supervisor_service_user }}"
-  when: not devstack
+  when: celery_worker is not defined and not devstack
   with_items: service_variants_enabled
   tags: deploy
 

playbooks/roles/jenkins_master/defaults/main.yml

@@ -68,3 +68,6 @@ jenkins_debian_pkgs:
     - maven
     - daemon
     - python-pycurl
+
+# Extra packages need for a specific jenkins instance.
+JENKINS_EXTRA_PKGS: []

playbooks/roles/jenkins_master/tasks/main.yml

@@ -7,6 +7,13 @@
   tags:
   - jenkins
 
+- name: jenkins_master | install jenkins extra system packages
+  apt:
+    pkg={{','.join(JENKINS_EXTRA_PKGS)}}
+    state=present update_cache=yes
+  tags:
+  - jenkins
+
 - name: jenkins_master | create jenkins group
   group: name={{ jenkins_group }} state=present
 

util/vpc-tools/create_stack.py

 import argparse
 import boto
+import yaml
 from os.path import basename
 from time import sleep
+from pprint import pprint
+
 
 FAILURE_STATES = [
     'CREATE_FAILED',
@@ -28,28 +31,39 @@ def upload_file(file_path, bucket_name, key_name):
     key.key = key_name
     key.set_contents_from_filename(file_path)
 
-    url = 'https://s3.amazonaws.com/{}/{}'.format(bucket_name, key_name)
+    key.set_acl('public-read')
+    url = key.generate_url(300, query_auth=False)
     return url
 
-def create_stack(stack_name, template, region='us-east-1', blocking=True, temp_bucket='edx-sandbox-devops'):
+def create_stack(stack_name, template, region='us-east-1', blocking=True,
+                 temp_bucket='edx-sandbox-devops', parameters=[],
+                 update=False):
+
     cfn = boto.connect_cloudformation()
 
     # Upload the template to s3
-    key_name = 'cloudformation/auto/{}_{}'.format(stack_name, basename(template))
+    key_pattern = 'devops/cloudformation/auto/{}_{}'
+    key_name = key_pattern.format(stack_name, basename(template))
     template_url = upload_file(template, temp_bucket, key_name)
 
     # Reference the stack.
     try:
+        if update:
+            stack_id = cfn.update_stack(stack_name,
+                template_url=template_url,
+                capabilities=['CAPABILITY_IAM'],
+                tags={'autostack':'true'},
+                parameters=parameters)
+        else:
             stack_id = cfn.create_stack(stack_name,
                 template_url=template_url,
                 capabilities=['CAPABILITY_IAM'],
                 tags={'autostack':'true'},
-            parameters=[('KeyName', 'continuous-integration')])
+                parameters=parameters)
     except Exception as e:
         print(e.message)
         raise e
 
-   
     status = None
     while blocking:
         sleep(5)
@@ -65,6 +79,9 @@ def create_stack(stack_name, template, region='us-east-1', blocking=True, temp_b
 
     return stack_id
 
+def cfn_params_from(filename):
+    params_dict = yaml.safe_load(open(filename))
+    return [ (key,value) for key,value in params_dict.items() ]
 
 if __name__ == '__main__':
         description = 'Create a cloudformation stack from a template.'
@@ -73,6 +90,9 @@ if __name__ == '__main__':
         msg = 'Name for the cloudformation stack.'
         parser.add_argument('-n', '--stackname', required=True, help=msg)
 
+        msg = 'Pass this argument if we are updating an existing stack.'
+        parser.add_argument('-u', '--update', action='store_true')
+
         msg = 'Name of the bucket to use for temporarily uploading the \
             template.'
         parser.add_argument('-b', '--bucketname', default="edx-sandbox-devops",
@@ -84,11 +104,16 @@ if __name__ == '__main__':
         msg = 'The AWS region to build this stack in.'
         parser.add_argument('-r', '--region', default='us-east-1', help=msg)
 
+        msg = 'YAML file containing stack build parameters'
+        parser.add_argument('-p', '--parameters', help=msg)
+
         args = parser.parse_args()
         stack_name = args.stackname
         template = args.template
         region = args.region
         bucket_name = args.bucketname
+        parameters = cfn_params_from(args.parameters)
+        update = args.update
 
-        create_stack(stack_name, template, region, bucket_name)
+        create_stack(stack_name, template, region, temp_bucket=bucket_name, parameters=parameters, update=update)
         print('Stack({}) created.'.format(stack_name))

util/vpc-tools/vpc_dns.py

@@ -104,6 +104,9 @@ if __name__ == "__main__":
     parser.add_argument('-n', '--stackname',
         help="The name of the cloudformation stack.",
         required=True)
+
+    parser.add_argument('-z', '--parent-zone',
+        help="The parent zone under which the dns for this vpc resides.")
     args = parser.parse_args()
     stack_name = args.stackname
 
@@ -118,7 +121,10 @@ if __name__ == "__main__":
     }
 
     # Create a zone for the stack.
-    zone_name = "{}.vpc.edx.org".format(stack_name)
+    parent_zone = 'vpc.edx.org'
+    if args.parent_zone:
+        parent_zone = args.parent_zone
+    zone_name = "{}.{}".format(stack_name, parent_zone)
 
     zone = get_or_create_hosted_zone(zone_name)