Skip to content

C
configuration
Commit 63da379d by Feanil Patel
Options

Merge pull request #453 from edx/feanil/pre_stage 

Feanil/pre stage
parents 53b7baa0 298b2a4b
Showing with 95 additions and 87 deletions
cloudformation_templates/edx-reference-architecture.json
......@@ -1974,6 +1974,12 @@
"CidrIp":"0.0.0.0/0"
},
{
"IpProtocol":"tcp",
"FromPort":"11371",
"ToPort":"11371",
"CidrIp":"0.0.0.0/0"
},
{
"IpProtocol":"icmp",
"FromPort":"-1",
"ToPort":"-1",
......@@ -2018,6 +2024,12 @@
"FromPort":"10016",
"ToPort":"10016",
"CidrIp":"0.0.0.0/0"
},
{
"IpProtocol":"tcp",
"FromPort":"11371",
"ToPort":"11371",
"CidrIp":"0.0.0.0/0"
}
]
}
......@@ -4864,52 +4876,6 @@
"Value":{
"Ref":"EdxappServerSecurityGroup"
}
},
"DatabaseConfigurationString":{
"Description":"JDBC connection string for database",
"Value":{
"Fn::Join":[
"",
[
"'DATABASES': {\n",
" 'default': {\n",
" 'ENGINE': 'django.db.backends.mysql',\n",
" 'NAME': '",
{
"Ref":"DBName"
},
"',\n",
" 'USER': '",
{
"Ref":"DBUsername"
},
"',\n",
" 'PASSWORD': '",
{
"Ref":"DBPassword"
},
"',\n",
" 'HOST': '",
{
"Fn::GetAtt":[
"EdxDB",
"Endpoint.Address"
]
},
"',\n",
" 'PORT': '",
{
"Fn::GetAtt":[
"EdxDB",
"Endpoint.Port"
]
},
"'\n",
" }\n",
"}\n"
]
]
}
}
}
}
}
playbooks/edx-east/edx_vpc.yml
......@@ -2,29 +2,31 @@
- hosts: first_in_tag_role_mongo
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/users.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
roles:
- common
- gh_users
- role: 'mongo'
mongo_create_users: yes
#- hosts: tag_role_mongo:!first_in_tag_role_mongo
# sudo: True
# vars_files:
# - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
# - "{{ secure_dir }}/vars/users.yml"
# - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
# - "{{ secure_dir }}/vars/common/common.yml"
# roles:
# - common
# - gh_users
# - mongo
- hosts: first_in_tag_role_edxapp
sudo: True
serial: 1
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- datadog
- supervisor
- role: nginx
......@@ -36,17 +38,16 @@
edxapp_lms_env: 'lms.envs.load_test'
migrate_db: 'yes'
openid_workaround: 'yes'
edx_platform_commit: 'HEAD'
- splunkforwarder
- hosts: tag_role_edxapp:!first_in_tag_role_edxapp
sudo: True
serial: 1
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- datadog
- supervisor
- role: nginx
......@@ -56,16 +57,15 @@
- lms-preview
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
edx_platform_commit: 'HEAD'
- splunkforwarder
- hosts: tag_role_worker
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- datadog
- supervisor
- role: nginx
......@@ -76,16 +76,15 @@
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
celery_worker: True
edx_platform_commit: 'HEAD'
- splunkforwarder
- hosts: tag_role_xserver
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- supervisor
- role: nginx
nginx_sites:
......@@ -96,22 +95,22 @@
serial: 1
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- supervisor
- rabbitmq
- splunkforwarder
- hosts: first_in_tag_role_xqueue
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- supervisor
- role: nginx
nginx_sites:
......@@ -122,11 +121,11 @@
- hosts: tag_role_xqueue:!first_in_tag_role_xqueue
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- supervisor
- role: nginx
nginx_sites:
......@@ -136,11 +135,11 @@
- hosts: tag_role_forum
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- supervisor
- oraclejdk
- elasticsearch
......
playbooks/jenkins_master.yml
......@@ -9,4 +9,6 @@
vars:
COMMON_DATA_DIR: "/mnt"
roles:
- common
- gh_users
- jenkins_master
playbooks/roles/edxapp/tasks/deploy.yml
......@@ -237,7 +237,7 @@
config={{ supervisor_cfg }}
name="edxapp:{{ item }}"
sudo_user: "{{ supervisor_service_user }}"
when: not devstack
when: celery_worker is not defined and not devstack
with_items: service_variants_enabled
tags: deploy
......
playbooks/roles/jenkins_master/defaults/main.yml
......@@ -68,3 +68,6 @@ jenkins_debian_pkgs:
- maven
- daemon
- python-pycurl
# Extra packages need for a specific jenkins instance.
JENKINS_EXTRA_PKGS: []
playbooks/roles/jenkins_master/tasks/main.yml
......@@ -7,6 +7,13 @@
tags:
- jenkins
- name: jenkins_master | install jenkins extra system packages
apt:
pkg={{','.join(JENKINS_EXTRA_PKGS)}}
state=present update_cache=yes
tags:
- jenkins
- name: jenkins_master | create jenkins group
group: name={{ jenkins_group }} state=present
......
util/vpc-tools/create_stack.py
import argparse
import boto
import yaml
from os.path import basename
from time import sleep
from pprint import pprint
FAILURE_STATES = [
'CREATE_FAILED',
......@@ -25,32 +28,43 @@ def upload_file(file_path, bucket_name, key_name):
bucket = conn.get_bucket(bucket_name, validate=False)
key = boto.s3.key.Key(bucket)
key.key = key_name
key.key = key_name
key.set_contents_from_filename(file_path)
url = 'https://s3.amazonaws.com/{}/{}'.format(bucket_name, key_name)
key.set_acl('public-read')
url = key.generate_url(300, query_auth=False)
return url
def create_stack(stack_name, template, region='us-east-1', blocking=True, temp_bucket='edx-sandbox-devops'):
def create_stack(stack_name, template, region='us-east-1', blocking=True,
temp_bucket='edx-sandbox-devops', parameters=[],
update=False):
cfn = boto.connect_cloudformation()
# Upload the template to s3
key_name = 'cloudformation/auto/{}_{}'.format(stack_name, basename(template))
key_pattern = 'devops/cloudformation/auto/{}_{}'
key_name = key_pattern.format(stack_name, basename(template))
template_url = upload_file(template, temp_bucket, key_name)
# Reference the stack.
try:
stack_id = cfn.create_stack(stack_name,
template_url=template_url,
capabilities=['CAPABILITY_IAM'],
tags={'autostack':'true'},
parameters=[('KeyName', 'continuous-integration')])
if update:
stack_id = cfn.update_stack(stack_name,
template_url=template_url,
capabilities=['CAPABILITY_IAM'],
tags={'autostack':'true'},
parameters=parameters)
else:
stack_id = cfn.create_stack(stack_name,
template_url=template_url,
capabilities=['CAPABILITY_IAM'],
tags={'autostack':'true'},
parameters=parameters)
except Exception as e:
print(e.message)
raise e
status = None
status = None
while blocking:
sleep(5)
stack_instance = cfn.describe_stacks(stack_id)[0]
......@@ -65,6 +79,9 @@ def create_stack(stack_name, template, region='us-east-1', blocking=True, temp_b
return stack_id
def cfn_params_from(filename):
params_dict = yaml.safe_load(open(filename))
return [ (key,value) for key,value in params_dict.items() ]
if __name__ == '__main__':
description = 'Create a cloudformation stack from a template.'
......@@ -73,6 +90,9 @@ if __name__ == '__main__':
msg = 'Name for the cloudformation stack.'
parser.add_argument('-n', '--stackname', required=True, help=msg)
msg = 'Pass this argument if we are updating an existing stack.'
parser.add_argument('-u', '--update', action='store_true')
msg = 'Name of the bucket to use for temporarily uploading the \
template.'
parser.add_argument('-b', '--bucketname', default="edx-sandbox-devops",
......@@ -80,15 +100,20 @@ if __name__ == '__main__':
msg = 'The path to the cloudformation template.'
parser.add_argument('-t', '--template', required=True, help=msg)
msg = 'The AWS region to build this stack in.'
parser.add_argument('-r', '--region', default='us-east-1', help=msg)
args = parser.parse_args()
msg = 'YAML file containing stack build parameters'
parser.add_argument('-p', '--parameters', help=msg)
args = parser.parse_args()
stack_name = args.stackname
template = args.template
region = args.region
bucket_name = args.bucketname
parameters = cfn_params_from(args.parameters)
update = args.update
create_stack(stack_name, template, region, bucket_name)
create_stack(stack_name, template, region, temp_bucket=bucket_name, parameters=parameters, update=update)
print('Stack({}) created.'.format(stack_name))
util/vpc-tools/vpc_dns.py
......@@ -104,6 +104,9 @@ if __name__ == "__main__":
parser.add_argument('-n', '--stackname',
help="The name of the cloudformation stack.",
required=True)
parser.add_argument('-z', '--parent-zone',
help="The parent zone under which the dns for this vpc resides.")
args = parser.parse_args()
stack_name = args.stackname
......@@ -118,7 +121,10 @@ if __name__ == "__main__":
}
# Create a zone for the stack.
zone_name = "{}.vpc.edx.org".format(stack_name)
parent_zone = 'vpc.edx.org'
if args.parent_zone:
parent_zone = args.parent_zone
zone_name = "{}.{}".format(stack_name, parent_zone)
zone = get_or_create_hosted_zone(zone_name)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment