Skip to content

C
configuration
Commit 50041fa8 by John Jarvis
Options

Merge pull request #561 from edx/rc/empanada 

Rc/empanada
parents 3e037702 4b80d36d
Showing with 1274 additions and 607 deletions
.gitignore
......@@ -5,7 +5,7 @@
\#*\#
*~
.#*
vagrant/devstack/cs_comments_service
vagrant/devstack/edx-platform
vagrant/release/*/devstack/cs_comments_service
vagrant/release/*/devstack/edx-platform
vagrant/*/devstack/edx-platform
vagrant/*/devstack/cs_comments_service
vagrant/*/devstack/ora
cloudformation_templates/edx-reference-architecture.json
......@@ -16,7 +16,7 @@
"Description":"Name of an existing EC2 KeyPair to enable SSH access to the web server",
"Default":"deployment"
},
"InstanceType":{
"EdxappInstanceType":{
"Description":"WebServer EC2 instance type",
"Type":"String",
"Default":"m1.small",
......@@ -131,6 +131,52 @@
],
"ConstraintDescription":"must be a valid EC2 instance type."
},
"XserverInstanceType":{
"Description":"RabbitMQ server EC2 instance type",
"Type":"String",
"Default":"m1.small",
"AllowedValues":[
"t1.micro",
"m1.small",
"m1.medium",
"m1.large",
"m1.xlarge",
"m2.xlarge",
"m2.2xlarge",
"m2.4xlarge",
"m3.xlarge",
"m3.2xlarge",
"c1.medium",
"c1.xlarge",
"cc1.4xlarge",
"cc2.8xlarge",
"cg1.4xlarge"
],
"ConstraintDescription":"must be a valid EC2 instance type."
},
"XqueueInstanceType":{
"Description":"RabbitMQ server EC2 instance type",
"Type":"String",
"Default":"m1.small",
"AllowedValues":[
"t1.micro",
"m1.small",
"m1.medium",
"m1.large",
"m1.xlarge",
"m2.xlarge",
"m2.2xlarge",
"m2.4xlarge",
"m3.xlarge",
"m3.2xlarge",
"c1.medium",
"c1.xlarge",
"cc1.4xlarge",
"cc2.8xlarge",
"cg1.4xlarge"
],
"ConstraintDescription":"must be a valid EC2 instance type."
},
"SSHLocation":{
"Description":"The IP address range that can be used to SSH to the EC2 instances",
"Type":"String",
......@@ -393,7 +439,8 @@
"Forum02": { "CIDR":"10.0.81.0/24" },
"Mongo01": { "CIDR":"10.0.90.0/24" },
"Mongo02": { "CIDR":"10.0.91.0/24" },
"Mongo03": { "CIDR":"10.0.92.0/24" }
"Mongo03": { "CIDR":"10.0.92.0/24" },
"Admin": { "CIDR":"10.0.200.0/24" }
},
"MapRegionsToAvailZones":{
"us-east-1": { "AZone2":"us-east-1d", "AZone0":"us-east-1b", "AZone1":"us-east-1c" },
......@@ -460,6 +507,38 @@
}
}
},
"AdminSubnet":{
"Type":"AWS::EC2::Subnet",
"Properties":{
"VpcId":{
"Ref":"EdxVPC"
},
"CidrBlock":{
"Fn::FindInMap":[
"SubnetConfig",
"Admin",
"CIDR"
]
},
"AvailabilityZone":{
"Fn::FindInMap":[
"MapRegionsToAvailZones",
{ "Ref":"AWS::Region" },
"AZone0"
]
},
"Tags":[
{
"Key":"Application",
"Value":"admin"
},
{
"Key":"Network",
"Value":"Private"
}
]
}
},
"EdxappSubnet01":{
"Type":"AWS::EC2::Subnet",
"Properties":{
......@@ -1974,6 +2053,12 @@
"CidrIp":"0.0.0.0/0"
},
{
"IpProtocol":"tcp",
"FromPort":"11371",
"ToPort":"11371",
"CidrIp":"0.0.0.0/0"
},
{
"IpProtocol":"icmp",
"FromPort":"-1",
"ToPort":"-1",
......@@ -2018,6 +2103,12 @@
"FromPort":"10016",
"ToPort":"10016",
"CidrIp":"0.0.0.0/0"
},
{
"IpProtocol":"tcp",
"FromPort":"11371",
"ToPort":"11371",
"CidrIp":"0.0.0.0/0"
}
]
}
......@@ -2430,7 +2521,7 @@
"Fn::FindInMap":[
"AWSInstanceType2Arch",
{
"Ref":"InstanceType"
"Ref":"EdxappInstanceType"
},
"Arch"
]
......@@ -2477,7 +2568,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"EdxappInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -2544,8 +2635,12 @@
"LaunchConfigurationName":{
"Ref":"EdxappServer"
},
"MinSize":"2",
"MaxSize":"2",
"MinSize":{
"Ref":"EdxappDesiredCapacity"
},
"MaxSize":{
"Ref":"EdxappDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"EdxappDesiredCapacity"
},
......@@ -2781,7 +2876,7 @@
"Fn::FindInMap":[
"AWSInstanceType2Arch",
{
"Ref":"InstanceType"
"Ref":"XqueueInstanceType"
},
"Arch"
]
......@@ -2824,7 +2919,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"XqueueInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -2891,8 +2986,12 @@
"LaunchConfigurationName":{
"Ref":"XqueueServer"
},
"MinSize":"2",
"MaxSize":"2",
"MinSize":{
"Ref":"XqueueDesiredCapacity"
},
"MaxSize":{
"Ref":"XqueueDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"XqueueDesiredCapacity"
},
......@@ -3154,7 +3253,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"RabbitInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -3221,8 +3320,12 @@
"LaunchConfigurationName":{
"Ref":"RabbitMQServer"
},
"MinSize":"2",
"MaxSize":"2",
"MinSize":{
"Ref":"RabbitMQDesiredCapacity"
},
"MaxSize":{
"Ref":"RabbitMQDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"RabbitMQDesiredCapacity"
},
......@@ -3477,7 +3580,7 @@
"Fn::FindInMap":[
"AWSInstanceType2Arch",
{
"Ref":"InstanceType"
"Ref":"XserverInstanceType"
},
"Arch"
]
......@@ -3520,7 +3623,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"XserverInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -3587,8 +3690,12 @@
"LaunchConfigurationName":{
"Ref":"XServer"
},
"MinSize":"2",
"MaxSize":"2",
"MinSize":{
"Ref":"XServerDesiredCapacity"
},
"MaxSize":{
"Ref":"XServerDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"XServerDesiredCapacity"
},
......@@ -3979,7 +4086,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"WorkerInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -4046,8 +4153,12 @@
"LaunchConfigurationName":{
"Ref":"WorkerServer"
},
"MinSize":"2",
"MaxSize":"2",
"MinSize":{
"Ref":"WorkerDesiredCapacity"
},
"MaxSize":{
"Ref":"WorkerDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"WorkerDesiredCapacity"
}
......@@ -4224,7 +4335,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"ForumInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -4291,8 +4402,12 @@
"LaunchConfigurationName":{
"Ref":"ForumServer"
},
"MinSize":"2",
"MaxSize":"2",
"MinSize":{
"Ref":"ForumDesiredCapacity"
},
"MaxSize":{
"Ref":"ForumDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"ForumDesiredCapacity"
},
......@@ -4595,7 +4710,7 @@
"Ref":"KeyName"
},
"InstanceType":{
"Ref":"InstanceType"
"Ref":"MongoInstanceType"
},
"BlockDeviceMappings":[
{
......@@ -4683,8 +4798,12 @@
"LaunchConfigurationName":{
"Ref":"MongoServer"
},
"MinSize":"3",
"MaxSize":"3",
"MinSize":{
"Ref":"MongoDesiredCapacity"
},
"MaxSize":{
"Ref":"MongoDesiredCapacity"
},
"DesiredCapacity":{
"Ref":"MongoDesiredCapacity"
}
......@@ -4864,52 +4983,6 @@
"Value":{
"Ref":"EdxappServerSecurityGroup"
}
},
"DatabaseConfigurationString":{
"Description":"JDBC connection string for database",
"Value":{
"Fn::Join":[
"",
[
"'DATABASES': {\n",
" 'default': {\n",
" 'ENGINE': 'django.db.backends.mysql',\n",
" 'NAME': '",
{
"Ref":"DBName"
},
"',\n",
" 'USER': '",
{
"Ref":"DBUsername"
},
"',\n",
" 'PASSWORD': '",
{
"Ref":"DBPassword"
},
"',\n",
" 'HOST': '",
{
"Fn::GetAtt":[
"EdxDB",
"Endpoint.Address"
]
},
"',\n",
" 'PORT': '",
{
"Fn::GetAtt":[
"EdxDB",
"Endpoint.Port"
]
},
"'\n",
" }\n",
"}\n"
]
]
}
}
}
}
}
playbooks/callback_plugins/sqs.py 0 → 100644
# Copyright 2013 John Jarvis <john@jarv.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import time
import json
try:
import boto.sqs
from boto.exception import NoAuthHandlerFound
except ImportError:
print "Boto is required for the sqs_notify callback plugin"
raise
class CallbackModule(object):
"""
This Ansible callback plugin sends task events
to SQS.
The following vars must be set in the environment:
ANSIBLE_ENABLE_SQS - enables the callback module
SQS_REGION - AWS region to connect to
SQS_MSG_PREFIX - Additional data that will be put
on the queue (optional)
The following events are put on the queue
- FAILURE events
- OK events
- TASK events
- START events
"""
def __init__(self):
self.start_time = time.time()
if 'ANSIBLE_ENABLE_SQS' in os.environ:
self.enable_sqs = True
if not 'SQS_REGION' in os.environ:
print 'ANSIBLE_ENABLE_SQS enabled but SQS_REGION ' \
'not defined in environment'
sys.exit(1)
self.region = os.environ['SQS_REGION']
try:
self.sqs = boto.sqs.connect_to_region(self.region)
except NoAuthHandlerFound:
print 'ANSIBLE_ENABLE_SQS enabled but cannot connect ' \
'to AWS due invalid credentials'
sys.exit(1)
if not 'SQS_NAME' in os.environ:
print 'ANSIBLE_ENABLE_SQS enabled but SQS_NAME not ' \
'defined in environment'
sys.exit(1)
self.name = os.environ['SQS_NAME']
self.queue = self.sqs.create_queue(self.name)
if 'SQS_MSG_PREFIX' in os.environ:
self.prefix = os.environ['SQS_MSG_PREFIX']
else:
self.prefix = ''
self.last_seen_ts = {}
else:
self.enable_sqs = False
def runner_on_failed(self, host, res, ignore_errors=False):
if self.enable_sqs:
if not ignore_errors:
self._send_queue_message(res, 'FAILURE')
def runner_on_ok(self, host, res):
if self.enable_sqs:
# don't send the setup results
if res['invocation']['module_name'] != "setup":
self._send_queue_message(res, 'OK')
def playbook_on_task_start(self, name, is_conditional):
if self.enable_sqs:
self._send_queue_message(name, 'TASK')
def playbook_on_play_start(self, pattern):
if self.enable_sqs:
self._send_queue_message(pattern, 'START')
def playbook_on_stats(self, stats):
if self.enable_sqs:
d = {}
delta = time.time() - self.start_time
d['delta'] = delta
for s in ['changed', 'failures', 'ok', 'processed', 'skipped']:
d[s] = getattr(stats, s)
self._send_queue_message(d, 'STATS')
def _send_queue_message(self, msg, msg_type):
if self.enable_sqs:
from_start = time.time() - self.start_time
payload = {msg_type: msg}
payload['TS'] = from_start
payload['PREFIX'] = self.prefix
# update the last seen timestamp for
# the message type
self.last_seen_ts[msg_type] = time.time()
if msg_type in ['OK', 'FAILURE']:
# report the delta between the OK/FAILURE and
# last TASK
if 'TASK' in self.last_seen_ts:
from_task = \
self.last_seen_ts[msg_type] - self.last_seen_ts['TASK']
payload['delta'] = from_task
for output in ['stderr', 'stdout']:
if output in payload[msg_type]:
# only keep the last 1000 characters
# of stderr and stdout
if len(payload[msg_type][output]) > 1000:
payload[msg_type][output] = "(clipping) ... " \
+ payload[msg_type][output][-1000:]
self.sqs.send_message(self.queue, json.dumps(payload))
playbooks/cloudformation.yml deleted 100644 → 0
---
# This playbook demonstrates how to use the ansible cloudformation module to launch an AWS CloudFormation stack.
#
# This module requires that the boto python library is installed, and that you have your AWS credentials
# in $HOME/.boto
#The thought here is to bring up a bare infrastructure with CloudFormation, but use ansible to configure it.
#I generally do this in 2 different playbook runs as to allow the ec2.py inventory to be updated.
#This module also uses "complex arguments" which were introduced in ansible 1.1 allowing you to specify the
#Cloudformation template parameters
#This example launches a 3 node AutoScale group, with a security group, and an InstanceProfile with root permissions.
#If a stack does not exist, it will be created. If it does exist and the template file has changed, the stack will be updated.
#If the parameters are different, the stack will also be updated.
#CloudFormation stacks can take awhile to provision, if you are curious about its status, use the AWS
#web console or one of the CloudFormation CLI's.
#Example update -- try first launching the stack with 3 as the ClusterSize. After it is launched, change it to 4
#and run the playbook again.
- name: provision stack
hosts: localhost
connection: local
gather_facts: false
# Launch the cloudformation-example.json template. Register the output.
tasks:
- name: edX configuration
cloudformation: >
stack_name="$name" state=present
region=$region disable_rollback=false
template=../cloudformation_templates/edx-server-multi-instance.json
args:
template_parameters:
KeyName: $key
InstanceType: m1.small
GroupTag: $group
register: stack
- name: show stack outputs
debug: msg="My stack outputs are ${stack.stack_outputs}"
playbooks/edx-east/callback_plugins 0 → 120000
../callback_plugins
\ No newline at end of file
playbooks/edx-east/deploy_certs.yml → playbooks/edx-east/certs.yml
......@@ -2,7 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- certs
playbooks/edx-east/deploy_common.yml → playbooks/edx-east/common.yml
......@@ -2,6 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- gh_users
- common
playbooks/edx-east/create_cname.yml 0 → 100644
# Creates a cname for a sandbox ec2 instance
- name: Creates a CNAME
hosts: all
gather_facts: False
tasks:
- name: Add DNS name
route53:
overwrite: yes
command: create
zone: "{{ dns_zone }}"
type: CNAME
ttl: 300
record: "{{ dns_name }}.{{ dns_zone }}"
value: "{{ sandbox }}"
playbooks/edx-east/create_user.yml 0 → 100644
# Creates a single user on a server
# Example: ansible-playbook -i "jarv.m.sandbox.edx.org," ./create_user.yml -e "user=jarv"
- name: Create a single user
hosts: all
sudo: True
gather_facts: False
pre_tasks:
- fail: msg="You must pass a user into this play"
when: not user
- set_fact:
gh_users:
- "{{ user }}"
roles:
- gh_users
playbooks/edx-east/deploy_nginx_all_roles.yml 0 → 100644
- name: Configure instance(s)
hosts: all
sudo: True
gather_facts: False
vars_files:
- roles/edxapp/defaults/main.yml
- roles/ora/defaults/main.yml
- roles/xqueue/defaults/main.yml
- roles/xserver/defaults/main.yml
roles:
- common
- role: nginx
nginx_sites:
- cms
- lms
- ora
- xqueue
- xserver
playbooks/edx-east/deploy_devpi.yml → playbooks/edx-east/devpi.yml
......@@ -2,7 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- devpi
playbooks/edx-east/deploy_discern.yml → playbooks/edx-east/discern.yml
......@@ -2,7 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- discern
playbooks/edx-east/dummy.yml 0 → 100644
---
# dummy var file
# This file is needed as a fall through
# for vars_files
dummy_var: True
playbooks/edx-east/edx_ansible.yml 0 → 100644
- name: Deploy ansible
hosts: all
sudo: True
gather_facts: True
roles:
- edx_ansible
playbooks/edx-east/edx_continuous_integration.yml
......@@ -7,7 +7,6 @@
migrate_db: "yes"
openid_workaround: True
roles:
- common
- role: nginx
nginx_sites:
- cms
......@@ -16,7 +15,6 @@
- xqueue
- xserver
- edxlocal
- supervisor
- mongo
- edxapp
- role: demo
......
playbooks/edx-east/edx_dev2.yml
......@@ -14,8 +14,6 @@
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- datadog
- role: nginx
nginx_sites:
......@@ -26,7 +24,7 @@
EDXAPP_LMS_NGINX_PORT: 80
EDXAPP_CMS_NGINX_PORT: 80
edxapp_lms_env: 'lms.envs.load_test'
edx_platform_commit: 'sarina/install-datadog'
edx_platform_version: 'sarina/install-datadog'
- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_worker
sudo: True
vars_files:
......@@ -34,8 +32,6 @@
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- datadog
- role: nginx
nginx_sites:
......@@ -45,14 +41,13 @@
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
celery_worker: True
edx_platform_commit: 'sarina/install-datadog'
edx_platform_version: 'sarina/install-datadog'
#- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_xserver
# sudo: True
# vars_files:
# - "{{ secure_dir }}/vars/dev/dev2.yml"
# - "{{ secure_dir }}/vars/users.yml"
# roles:
# - common
# - nginx
# - xserver
#- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_rabbitmq
......@@ -62,7 +57,6 @@
# - "{{ secure_dir }}/vars/dev/dev2.yml"
# - "{{ secure_dir }}/vars/users.yml"
# roles:
# - common
# - rabbitmq
#- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_xqueue
# sudo: True
......@@ -70,6 +64,5 @@
# - "{{ secure_dir }}/vars/dev/dev2.yml"
# - "{{ secure_dir }}/vars/users.yml"
# roles:
# - common
# - nginx
# - xqueue
playbooks/edx-east/edx_feanilsandbox.yml
......@@ -6,8 +6,6 @@
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- datadog
- role: nginx
nginx_sites:
......@@ -16,7 +14,7 @@
- lms-preview
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
edx_platform_commit: 'release'
edx_platform_version: 'release'
- splunkforwarder
- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_role_worker
sudo: True
......@@ -25,8 +23,6 @@
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- datadog
- role: nginx
nginx_sites:
......@@ -36,7 +32,7 @@
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
celery_worker: True
edx_platform_commit: 'release'
edx_platform_version: 'release'
- splunkforwarder
- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_role_xserver
sudo: True
......@@ -44,8 +40,6 @@
- "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- role: nginx
nginx_sites:
- xserver
......@@ -58,8 +52,6 @@
- "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- rabbitmq
- splunkforwarder
- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_role_xqueue
......@@ -68,8 +60,6 @@
- "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- role: nginx
nginx_sites:
- xqueue
......@@ -81,6 +71,5 @@
- "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- role: 'mongo'
mongo_clustered: true
playbooks/edx-east/edx_mirror.yml
......@@ -4,8 +4,6 @@
sudo: True
gather_facts: False
roles:
- common
- supervisor
- role: nginx
nginx_sites:
- devpi
......
playbooks/edx-east/edx_notifier.yml
......@@ -6,8 +6,6 @@
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- supervisor
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
......@@ -22,8 +20,6 @@
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- supervisor
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
......@@ -38,8 +34,6 @@
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- supervisor
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
......@@ -54,8 +48,6 @@
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- supervisor
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
......@@ -71,8 +63,6 @@
gather_facts: True
vars:
roles:
- common
- supervisor
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
......
playbooks/edx-east/edx_provision.yml
......@@ -15,6 +15,7 @@
dns_name: "{{ dns_name }}"
dns_zone: "{{ dns_zone }}"
terminate_instance: true
instance_profile_name: sandbox
- name: Configure instance(s)
hosts: launched
......@@ -26,9 +27,41 @@
path=/var/log/cloud-init.log
timeout=15
search_regex="final-message"
vars_files:
- roles/edxapp/defaults/main.yml
- roles/ora/defaults/main.yml
- roles/xqueue/defaults/main.yml
- roles/xserver/defaults/main.yml
roles:
# rerun common to set the hostname
# rerun common to set the hostname, nginx to set basic auth
- common
- role: nginx
nginx_sites:
- cms
- lms
- ora
- xqueue
- xserver
# gh_users hash must be passed
# in as a -e variable
- gh_users
post_tasks:
- name: get instance id for elb registration
local_action:
module: ec2_lookup
region: us-east-1
tags:
Name: "{{ name_tag }}"
register: ec2_info
when: elb
sudo: False
- name: register instance into an elb if one was provided
local_action:
module: ec2_elb
region: "{{ region }}"
instance_id: "{{ ec2_info.instance_ids[0] }}"
state: present
ec2_elbs:
- "{{ elb }}"
when: elb
sudo: False
playbooks/edx-east/edx_sandbox.yml deleted 100644 → 0
---
# This playbook is to configuration
# the official edX sandbox instance
# sandbox.edx.org
- name: Configure instance(s)
hosts: tag_Name_edx-sandbox
sudo: True
gather_facts: True
vars:
migrate_db: "yes"
mysql5_workaround: True
roles:
- common
- supervisor
- role: nginx
nginx_sites:
- lms
- cms
- lms-preview
- edxlocal
- mongo
- edxapp
- rabbitmq
- oraclejdk
- elasticsearch
- { role: 'edxapp', celery_worker: True }
- role: rbenv
rbenv_user: "{{ forum_user }}"
rbenv_dir: "{{ forum_home }}"
rbenv_ruby_version: "{{ forum_ruby_version }}"
- forum
playbooks/edx-east/edx_vpc.yml
......@@ -2,31 +2,29 @@
- hosts: first_in_tag_role_mongo
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/users.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
roles:
- common
- gh_users
- role: 'mongo'
mongo_create_users: yes
#- hosts: tag_role_mongo:!first_in_tag_role_mongo
# sudo: True
# vars_files:
# - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
# - "{{ secure_dir }}/vars/users.yml"
# - "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
# - "{{ secure_dir }}/vars/common/common.yml"
# roles:
# - common
# - gh_users
# - mongo
- hosts: first_in_tag_role_edxapp
sudo: True
serial: 1
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- datadog
- supervisor
- role: nginx
nginx_sites:
- lms
......@@ -34,21 +32,18 @@
- lms-preview
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
migrate_db: 'yes'
migrate_db: '{{ RUN_EDXAPP_MIGRATION }}'
openid_workaround: 'yes'
edx_platform_commit: 'HEAD'
- splunkforwarder
- hosts: tag_role_edxapp:!first_in_tag_role_edxapp
sudo: True
serial: 1
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- datadog
- supervisor
- role: nginx
nginx_sites:
- lms
......@@ -56,18 +51,15 @@
- lms-preview
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
edx_platform_commit: 'HEAD'
- splunkforwarder
- hosts: tag_role_worker
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- gh_users
- datadog
- supervisor
- role: nginx
nginx_sites:
- lms
......@@ -76,17 +68,14 @@
- role: 'edxapp'
edxapp_lms_env: 'lms.envs.load_test'
celery_worker: True
edx_platform_commit: 'HEAD'
- splunkforwarder
- hosts: tag_role_xserver
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- gh_users
- role: nginx
nginx_sites:
- xserver
......@@ -96,38 +85,32 @@
serial: 1
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- gh_users
- rabbitmq
- splunkforwarder
- hosts: first_in_tag_role_xqueue
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- gh_users
- role: nginx
nginx_sites:
- xqueue
- role: xqueue
migrate_db: 'yes'
migrate_db: '{{ RUN_XQUEUE_MIGRATION }}'
- splunkforwarder
- hosts: tag_role_xqueue:!first_in_tag_role_xqueue
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- gh_users
- role: nginx
nginx_sites:
- xqueue
......@@ -136,12 +119,10 @@
- hosts: tag_role_forum
sudo: True
vars_files:
- "{{ secure_dir }}/vars/dev/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/{{ENVIRONMENT}}/{{CLOUDFORMATION_STACK_NAME}}.yml"
- "{{ secure_dir }}/vars/common/common.yml"
- "{{ secure_dir }}/vars/users.yml"
roles:
- common
- supervisor
- gh_users
- oraclejdk
- elasticsearch
- forum
playbooks/edx-east/deploy_edxapp.yml → playbooks/edx-east/edxapp.yml
......@@ -2,7 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- edxapp
playbooks/edx-east/deploy_forum.yml → playbooks/edx-east/forum.yml
......@@ -2,7 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- forum
playbooks/jenkins_master.yml → playbooks/edx-east/jenkins_master.yml
......@@ -9,4 +9,6 @@
vars:
COMMON_DATA_DIR: "/mnt"
roles:
- common
- gh_users
- jenkins_master
playbooks/jenkins_worker.yml → playbooks/edx-east/jenkins_worker.yml
......@@ -6,6 +6,8 @@
hosts: jenkins_worker
sudo: True
gather_facts: True
vars:
mongo_enable_journal: False
roles:
- common
- edxlocal
......
playbooks/edx-east/legacy_ora.yml 0 → 100644
# ansible-playbook -i ec2.py --limit="tag_group_grader:&tag_environment_stage" legacy_ora.yml -e "COMMON_ENV_TYPE=stage secure_dir=/path/to/secure/dir"
- name: Deploy legacy_ora
hosts: all
sudo: True
gather_facts: True
vars:
ora_app_dir: '/opt/wwc'
ora_user: 'www-data'
serial: 1
roles:
- legacy_ora
playbooks/edx-east/mlapi_prod.yml
......@@ -5,8 +5,6 @@
- "{{ secure_dir }}/vars/users.yml"
- "{{ secure_dir }}/vars/mlapi_prod_users.yml"
roles:
- common
- supervisor
- discern
sudo: True
- hosts:
......
playbooks/edx-east/mlapi_sandbox.yml
......@@ -5,8 +5,6 @@
- "{{ secure_dir }}/vars/users.yml"
- "{{ secure_dir }}/vars/mlapi_sandbox_users.yml"
roles:
- common
- supervisor
- discern
sudo: True
- hosts:
......
playbooks/edx-east/mlapi_stage.yml
......@@ -5,8 +5,6 @@
- "{{ secure_dir }}/vars/users.yml"
- "{{ secure_dir }}/vars/mlapi_stage_users.yml"
roles:
- common
- supervisor
- discern
sudo: True
- hosts:
......
playbooks/edx-east/deploy_ora.yml → playbooks/edx-east/ora.yml
......@@ -2,7 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- ora
playbooks/edx-east/deploy_rabbitmq.yml → playbooks/edx-east/rabbitmq.yml
......@@ -2,5 +2,7 @@
hosts: all
sudo: True
gather_facts: False
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- rabbitmq
playbooks/edx-east/worker.yml 0 → 100644
- name: Deploy worker
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- role: edxapp
celery_worker: True
playbooks/edx-east/deploy_xqueue.yml → playbooks/edx-east/xqueue.yml
......@@ -2,8 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- role: xqueue
tags: ['xqueue']
playbooks/edx-east/deploy_xserver.yml → playbooks/edx-east/xserver.yml
......@@ -2,8 +2,7 @@
hosts: all
sudo: True
gather_facts: True
vars_files:
- ["{{ secure_vars }}", "dummy.yml"]
roles:
- common
- supervisor
- role: xserver
tags: ['xserver']
playbooks/edx-west/README.md 0 → 100644
Readme
------
This directory has the live playbooks that we use here at Stanford to
maintain our instance of OpenEdX at [class.stanford.edu][c]. We check
it in to this public repo since we think that others might benefit from
seeing how we are configured.
[c]: https://class.stanford.edu/
That said, we haven't documented things in here well, so we have no
expectation that others will be able to make enough sense of this to
give us useful contributions back. Generally a PR affecting files in
here will be ignored / rejected.
This README is a useful proximate place to keep commands. But it is
a public repo so we shouldn't store anything confidential in here.
Other install docs:
- Giulio's install doc [here][1].
[1]: https://docs.google.com/document/d/1ZDx51Jxa-zffyeKvHmTp_tIskLW9D9NRg9NytPTbnrA/edit#heading=h.iggugvghbcpf
Ansible Commands - Prod
-----------------------
Generally we do installs as the "ubuntu" user. You want to make
sure that the stanford-deploy-20130415 ssh key is in your ssh agent.
ANSIBLE_EC2_INI=ec2.ini ansible-playbook prod-log.yml -u ubuntu -c ssh -i ./ec2.py
Ansible Commands - Stage
------------------------
Verify that you're doing something reasonable:
ANSIBLE_CONFIG=stage-ansible.cfg ANSIBLE_EC2_INI=ec2.ini ansible-playbook stage-app.yml -u ubuntu -c ssh -i ./ec2.py --list-hosts
Verify that you're doing something reasonable:
ANSIBLE_CONFIG=stage-ansible.cfg ANSIBLE_EC2_INI=ec2.ini ansible-playbook stage-app.yml -u ubuntu -c ssh -i ./ec2.py
playbooks/edx-west/prod-log.yml 0 → 100644
- hosts: tag_Name_log10_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/users.yml"
vars:
secure_dir: '../../../configuration-secure/ansible'
local_dir: '../../../configuration-secure/ansible/local'
roles:
- common
playbooks/edx_notifier.yml deleted 100644 → 0
- name: Configure stage instance(s)
hosts: notifier_stage
sudo: True
vars_files:
- "{{ secure_dir }}/vars/stage/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure loadtest instance(s)
hosts: notifier_loadtest
sudo: True
vars_files:
- "{{ secure_dir }}/vars/loadtest/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure stage edge instance(s)
hosts: notifier_edge_stage
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edge_stage/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure prod instance(s)
hosts: notifier_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/prod/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
- name: Configure edge prod instance(s)
hosts: notifier_edge_prod
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edge_prod/notifier.yml"
- "{{ secure_dir }}/vars/users.yml"
gather_facts: True
vars:
roles:
- common
- role: virtualenv
virtualenv_user: "notifier"
virtualenv_user_home: "/opt/wwc/notifier"
virtualenv_name: "notifier"
- notifier
playbooks/edx_provision_test.yml deleted 100644 → 0
- name: Create sandbox instance
hosts: localhost
connection: local
gather_facts: False
vars:
keypair: continuous-integration
instance_type: m1.small
security_group: sandbox
image: ami-d0f89fb9
region: us-east-1
instance_tags: '{"disposable": "true"}'
roles:
- launch_instance
- name: Configure instance(s)
hosts: launched
sudo: True
gather_facts: True
vars:
migrate_db: "yes"
openid_workaround: True
ansible_ssh_private_key_file: /var/lib/jenkins/continuous-integration.pem
vars_files:
- "{{ secure_dir }}/vars/edxapp_ref_users.yml"
- "{{ secure_dir }}/vars/edxapp_sandbox.yml"
- "{{ secure_dir }}/vars/edx_jenkins_tests.yml"
roles:
- common
- role: nginx
nginx_sites:
- lms
- cms
- lms-preview
- xserver
- xqueue
- edxlocal
- mongo
- edxapp
- xqueue
- xserver
- name: Terminate instances
hosts: localhost
connection: local
tasks:
- name: Terminate instances that were previously launched
local_action:
module: ec2
state: 'absent'
instance_ids: ${ec2.instance_ids}
playbooks/edx_sandbox.yml
# This playbook is to configure
# the official edX sandbox instance
# sandbox.edx.org
#
# On the machine you want to configure run the following
# command from the configuration/playbooks directory:
# ansible-playbook -c local --limit "localhost:127.0.0.1" /path/to/configuration/playbooks/edx_sandbox.yml -i "localhost,"
#
# To use different default ports for lms-preview, cms and to set the lms_base and lms_preview_base,
# for the following configuration:
# studio listening on port 80 - studio.example.com
# lms listening on port 80 - example.com
# lms-preview listening on port 80 - preview.example.com
#
# ansible-playbook -c local --limit "localhost:127.0.0.1" path/to/configuration/playbooks/edx_sandbox.yml -i "localhost," -e "EDXAPP_CMS_NGINX_PORT=80 EDXAPP_LMS_PREVIEW_NGINX_PORT=80 EDXAPP_LMS_BASE=example.com EDXAPP_PREVIEW_LMS_BASE=preview.example.com"
#
---
# Example sandbox configuration
# for single server community
# installs
- name: Configure instance(s)
hosts: localhost
hosts: all
sudo: True
gather_facts: True
vars:
migrate_db: "yes"
openid_workaround: True
EDXAPP_LMS_NGINX_PORT: '80'
edx_platform_version: 'master'
roles:
- common
- supervisor
- role: nginx
nginx_sites:
- lms
- cms
- lms-preview
- xqueue
- lms
- ora
- xqueue
- edxlocal
- mongo
- edxapp
- demo
- { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
- { role: 'edxapp', celery_worker: True }
- oraclejdk
......@@ -41,3 +31,5 @@
- forum
- { role: "xqueue", update_users: True }
- ora
- discern
- edx_ansible
playbooks/edxapp_ref.yml deleted 100644 → 0
- hosts: tag_Group_edxapp_ref
sudo: True
vars_files:
- "{{ secure_dir }}/vars/edxapp_ref_vars.yml"
- "{{ secure_dir }}/vars/edxapp_ref_users.yml"
roles:
- common
- role: nginx
nginx_sites:
- lms
- cms
- lms-preview
- gunicorn
- edxapp
- ruby
- npm
# run this role last
- in_production
playbooks/edxapp_rolling_example.yml deleted 100644 → 0
# ansible-playbook -v --user=ubuntu edxapp_rolling_example.yml -i ./ec2.py --private-key=/path/to/deployment.pem
- hosts: tag_Group_anothermulti
serial: 2
vars_files:
- "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
- "{{ secure_dir }}/vars/users.yml"
pre_tasks:
- name: Gathering ec2 facts
ec2_facts:
- name: Removing instance from the ELB
local_action: ec2_elb
args:
instance_id: "{{ ansible_ec2_instance_id }}"
state: 'absent'
roles:
- common
- role: nginx
nginx_sites:
- lms
- cms
- lms-preview
- edxapp
- ruby
post_tasks:
- name: Adding instance back to the ELB
local_action: ec2_elb
args:
instance_id: "{{ ansible_ec2_instance_id }}"
ec2_elbs: "{{ ec2_elbs }}"
state: 'present'
playbooks/jenkins-inventory.ini deleted 100644 → 0
[jenkins_test]
jenkins-test.sandbox.edx.org
playbooks/library/vpc_lookup 0 → 100644
#!/usr/bin/python
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
DOCUMENTATION = '''
---
module: vpc_lookup
short_description: returns a list of subnet Ids using tags as criteria
description:
- Returns a list of subnet Ids for a given set of tags that identify one or more VPCs
version_added: "1.5"
options:
region:
description:
- The AWS region to use. Must be specified if ec2_url
is not used. If not specified then the value of the
EC2_REGION environment variable, if any, is used.
required: false
default: null
aliases: [ 'aws_region', 'ec2_region' ]
aws_secret_key:
description:
- AWS secret key. If not set then the value of
the AWS_SECRET_KEY environment variable is used.
required: false
default: null
aliases: [ 'ec2_secret_key', 'secret_key' ]
aws_access_key:
description:
- AWS access key. If not set then the value of the
AWS_ACCESS_KEY environment variable is used.
required: false
default: null
aliases: [ 'ec2_access_key', 'access_key' ]
tags:
desription:
- tags to lookup
required: false
default: null
type: dict
aliases: []
requirements: [ "boto" ]
author: John Jarvis
'''
EXAMPLES = '''
# Note: None of these examples set aws_access_key, aws_secret_key, or region.
# It is assumed that their matching environment variables are set.
# Return all instances that match the tag "Name: foo"
- local_action:
module: vpc_lookup
tags:
Name: foo
'''
import sys
AWS_REGIONS = ['ap-northeast-1',
'ap-southeast-1',
'ap-southeast-2',
'eu-west-1',
'sa-east-1',
'us-east-1',
'us-west-1',
'us-west-2']
try:
from boto.vpc import VPCConnection
from boto.vpc import connect_to_region
except ImportError:
print "failed=True msg='boto required for this module'"
sys.exit(1)
def main():
module=AnsibleModule(
argument_spec=dict(
region=dict(choices=AWS_REGIONS),
aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'],
no_log=True),
aws_access_key=dict(aliases=['ec2_access_key', 'access_key']),
tags=dict(default=None, type='dict'),
)
)
tags = module.params.get('tags')
aws_secret_key = module.params.get('aws_secret_key')
aws_access_key = module.params.get('aws_access_key')
region = module.params.get('region')
# If we have a region specified, connect to its endpoint.
if region:
try:
vpc = connect_to_region(region, aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key)
except boto.exception.NoAuthHandlerFound, e:
module.fail_json(msg=str(e))
else:
module.fail_json(msg="region must be specified")
vpc_conn = VPCConnection()
subnet_ids = []
for subnet in vpc_conn.get_all_subnets(filters={'tag:' + tag: value
for tag, value in tags.iteritems()}):
subnet_ids.append(subnet.id)
vpc_ids = []
for vpc in vpc.get_all_vpcs(filters={'tag:' + tag: value
for tag, value in tags.iteritems()}):
vpc_ids.append(vpc.id)
module.exit_json(changed=False, subnet_ids=subnet_ids, vpc_ids=vpc_ids)
# this is magic, see lib/ansible/module_common.py
#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
main()
playbooks/roles/analytics-server/defaults/main.yml
......@@ -87,4 +87,4 @@ as_redhat_pkgs:
# Installed via pip to get the IAM role feature.
#
as_pip_pkgs:
- git+https://github.com/s3tools/s3cmd.git#egg=s3cmd
\ No newline at end of file
- git+https://github.com/s3tools/s3cmd.git#egg=s3cmd
playbooks/roles/analytics-server/meta/main.yml
---
dependencies:
- {
role: automated,
......@@ -8,4 +7,4 @@ dependencies:
automated_sudoers_template: 'roles/analytics-server/templates/etc/sudoers.d/99-automator-analytics-server.j2'
}
\ No newline at end of file
playbooks/roles/analytics-server/tasks/main.yml
......@@ -133,4 +133,4 @@
src=etc/init/analytics.conf.j2 dest=/etc/init/analytics.conf
owner=root group=root
- include: deploy.yml
\ No newline at end of file
- include: deploy.yml
playbooks/roles/analytics/defaults/main.yml
......@@ -87,4 +87,4 @@ analytics_redhat_pkgs:
# Installed via pip to get the IAM role feature.
#
analytics_pip_pkgs:
- git+https://github.com/s3tools/s3cmd.git#egg=s3cmd
\ No newline at end of file
- git+https://github.com/s3tools/s3cmd.git#egg=s3cmd
playbooks/roles/analytics/tasks/main.yml
......@@ -133,4 +133,4 @@
src=etc/init/analytics.conf.j2 dest=/etc/init/analytics.conf
owner=root group=root
- include: deploy.yml
\ No newline at end of file
- include: deploy.yml
playbooks/roles/apache/defaults/main.yml
---
apache_port: 80
\ No newline at end of file
apache_port: 80
playbooks/roles/certs/handlers/main.yml
......@@ -15,7 +15,7 @@
#
- name: certs | restart certs
supervisorctl: >
supervisorctl_local: >
name=certs
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/certs/meta/main.yml 0 → 100644
dependencies:
- supervisor
playbooks/roles/certs/tasks/deploy.yml
......@@ -68,7 +68,7 @@
changed_when: supervisor_update.stdout != ""
- name: certs | ensure certs has started
supervisorctl: >
supervisorctl_local: >
name=certs
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/certs/tasks/main.yml
......@@ -57,7 +57,7 @@
- name: certs | create certs gpg dir
file: >
path="{{ certs_gpg_dir }}" state=directory
owner="{{ certs_user }}" group="{{ certs_user }}"
owner="{{ common_web_user }}"
mode=0700
notify: certs | restart certs
......@@ -65,7 +65,7 @@
copy: >
src={{ CERTS_LOCAL_PRIVATE_KEY }}
dest={{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
owner={{ certs_user }} mode=0600
owner={{ common_web_user }} mode=0600
notify: certs | restart certs
register: certs_gpg_key
......@@ -73,7 +73,7 @@
- name: certs | load the gpg key
shell: >
/usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
sudo_user: "{{ certs_user }}"
sudo_user: "{{ common_web_user }}"
when: certs_gpg_key.changed
notify: certs | restart certs
......
playbooks/roles/common/defaults/main.yml
......@@ -22,6 +22,7 @@ COMMON_GIT_MIRROR: 'github.com'
COMMON_HOSTNAME: !!null
common_debian_pkgs:
- ntp
- ack-grep
- lynx-cur
- logrotate
......@@ -44,3 +45,7 @@ common_web_group: www-data
common_log_user: syslog
common_git_ppa: "ppa:git-core/ppa"
# Skip supervisor tasks
# Useful when supervisor is not installed (local dev)
devstack: False
playbooks/roles/common/tasks/main.yml
......@@ -15,11 +15,6 @@
- "{{ COMMON_BIN_DIR }}"
- "{{ COMMON_CFG_DIR }}"
- name: common | Create common log directory
file: >
path={{ COMMON_LOG_DIR }} state=directory owner=syslog
group=syslog mode=0755
# Need to install python-pycurl to use Ansible's apt_repository module
- name: common | Install python-pycurl
apt: pkg=python-pycurl state=present update_cache=yes
......@@ -36,6 +31,11 @@
pkg={{','.join(common_debian_pkgs)}} install_recommends=yes
state=present update_cache=yes
- name: common | Create common log directory
file: >
path={{ COMMON_LOG_DIR }} state=directory owner=syslog
group=syslog mode=0755
- name: common | upload sudo config for key forwarding as root
copy: >
src=ssh_key_forward dest=/etc/sudoers.d/ssh_key_forward
......
playbooks/roles/datadog/tasks/main.yml
......@@ -15,50 +15,43 @@
# - datadog
#
- name: datadog | add apt key
apt_key: id=C7A7DA52 url={{datadog_apt_key}} state=present
- name: datadog | install debian needed pkgs
apt: pkg={{ item }}
with_items: datadog_debian_pkgs
tags:
- datadog
- ubuntu
when: ansible_distribution in common_debian_variants
- name: datadog | install apt repository
shell: echo 'deb http://apt.datadoghq.com/ unstable main' > /etc/apt/sources.list.d/datadog-source.list
- name: datadog | add apt key
apt_key: id=C7A7DA52 url={{datadog_apt_key}} state=present
tags:
- datadog
- ubuntu
when: ansible_distribution in common_debian_variants
- name: datadog | add yum repo
copy:
src=etc/yum.repo.d/datdog.repo
dest=/etc/yum.repo.d/datdog.repo
- name: datadog | install apt repository
apt_repository: repo='deb http://apt.datadoghq.com/ unstable main' update_cache=yes
tags:
- datadog
- redhat
when_string: ansible_distribution in common_redhat_variants
- name: datadog | install datadog agent
apt: pkg="datadog-agent" update_cache=yes
apt: pkg="datadog-agent"
tags:
- datadog
- ubuntu
when: ansible_distribution in common_debian_variants
- name: datadog | bootstrap config
shell: cp /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf creates=/etc/dd-agent/datadog.conf
tags:
- datadog
# quoting intentional, missing space after line=api_key: also
# ansible wasn't handling the double quoted yaml properly
# otherwise.
- name: datadog | update api-key
lineinfile:
lineinfile: >
dest="/etc/dd-agent/datadog.conf"
"regexp=^api_key:.*"
"line=api_key:{{ common_dd_api_key }}"
regexp="^api_key:.*"
line="api_key:{{ datadog_api_key }}"
notify:
- datadog | restart the datadog service
tags:
- datadog
- name: datadog | ensure started and enabled
service: name=datadog-agent state=started enabled=yes
tags:
- datadog
playbooks/roles/demo/tasks/deploy.yml
......@@ -39,3 +39,11 @@
when: demo_checkout.changed
tags: deploy
- name: demo | seed the forums for the demo course
shell: >
{{ edxapp_venv_bin }}/python ./manage.py lms --settings=aws seed_permissions_roles {{ demo_course_id }}
chdir={{ edxapp_code_dir }}
with_items: demo_test_users
when: demo_checkout.changed
tags: deploy
playbooks/roles/devpi/handlers/main.yml
......@@ -12,7 +12,7 @@
#
---
- name: devpi | restart devpi
supervisorctl: >
supervisorctl_local: >
state=restarted
supervisorctl_path={{ devpi_supervisor_ctl }}
config={{ devpi_supervisor_cfg }}
......
playbooks/roles/devpi/tasks/main.yml
......@@ -107,7 +107,7 @@
tags: deploy
- name: devpi | ensure devpi is started
supervisorctl: >
supervisorctl_local: >
state=started
supervisorctl_path={{ devpi_supervisor_ctl }}
config={{ devpi_supervisor_cfg }}
......
playbooks/roles/discern/defaults/main.yml
DISCERN_NGINX_PORT: 18070
DISCERN_BASIC_AUTH: False
DISCERN_MEMCACHE: [ 'localhost:11211' ]
DISCERN_AWS_ACCESS_KEY_ID: ""
DISCERN_AWS_SECRET_ACCESS_KEY: ""
......@@ -32,10 +31,12 @@ discern_ease_pre_requirements_file: "{{ discern_ease_code_dir }}/pre-requirement
discern_ease_post_requirements_file: "{{ discern_ease_code_dir }}/requirements.txt"
discern_nltk_data_dir: "{{ discern_data_dir}}/nltk_data"
discern_nltk_download_url: http://edx-static.s3.amazonaws.com/nltk/nltk-data-20131113.tar.gz
discern_nltk_tmp_file: "{{ discern_data_dir }}/nltk.tmp.tar.tz"
discern_source_repo: https://github.com/edx/discern.git
discern_settings: discern.aws
discern_branch: master
discern_version: master
discern_gunicorn_port: 8070
discern_gunicorn_host: 127.0.0.1
......
playbooks/roles/discern/handlers/main.yml
---
- name: discern | restart discern
supervisorctl: >
supervisorctl_local: >
name=discern
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/discern/meta/main.yml 0 → 100644
---
dependencies:
- supervisor
playbooks/roles/discern/tasks/deploy.yml
......@@ -25,7 +25,7 @@
- deploy
- name: discern | git checkout discern repo into discern_code_dir
git: dest={{ discern_code_dir }} repo={{ discern_source_repo }} version={{ discern_branch }}
git: dest={{ discern_code_dir }} repo={{ discern_source_repo }} version={{ discern_version }}
sudo_user: "{{ discern_user }}"
notify:
- discern | restart discern
......@@ -41,7 +41,7 @@
- deploy
#Numpy has to be a pre-requirement in order for scipy to build
- name : install python pre-requirements for discern and ease
- name : discern | install python pre-requirements for discern and ease
pip: requirements={{item}} virtualenv={{ discern_venv_dir }} state=present
sudo_user: "{{ discern_user }}"
notify:
......@@ -52,7 +52,7 @@
tags:
- deploy
- name : install python requirements for discern and ease
- name : discern | install python requirements for discern and ease
pip: requirements={{item}} virtualenv={{ discern_venv_dir }} state=present
sudo_user: "{{ discern_user }}"
notify:
......@@ -71,16 +71,22 @@
tags:
- deploy
#Needed for the ease package to work
- name: discern | install nltk data using rendered shell script
shell: >
{{ discern_venv_dir }}/bin/python -m nltk.downloader -d {{ discern_nltk_data_dir }} all
- name: discern | download and install nltk
shell: |
set -e
curl -o {{ discern_nltk_tmp_file }} {{ discern_nltk_download_url }}
tar zxf {{ discern_nltk_tmp_file }}
rm -f {{ discern_nltk_tmp_file }}
touch {{ discern_nltk_download_url|basename }}-installed
creates={{ discern_data_dir }}/{{ discern_nltk_download_url|basename }}-installed
chdir={{ discern_data_dir }}
sudo_user: "{{ discern_user }}"
notify:
- discern | restart discern
tags:
- deploy
#Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
#support virtualenvs as of this comment
- name: discern | django syncdb migrate and collectstatic for discern
......@@ -121,7 +127,7 @@
tags: deploy
- name: discern | ensure discern, discern_celery has started
supervisorctl: >
supervisorctl_local: >
name={{ item }}
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/edx_ansible/defaults/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Defaults for role edx_ansible, an edx_ansible role to install edx_ansible
#
#
# OS packages
#
edx_ansible_debian_pkgs:
- python-pip
- python-apt
- git-core
- build-essential
- python-dev
- libxml2-dev
- libxslt1-dev
- curl
edx_ansible_app_dir: "{{ COMMON_APP_DIR }}/edx_ansible"
edx_ansible_code_dir: "{{ edx_ansible_app_dir }}/edx_ansible"
edx_ansible_data_dir: "{{ COMMON_DATA_DIR }}/edx_ansible"
edx_ansible_venvs_dir: "{{ edx_ansible_app_dir }}/venvs"
edx_ansible_venv_dir: "{{ edx_ansible_venvs_dir }}/edx_ansible"
edx_ansible_venv_bin: "{{ edx_ansible_venv_dir }}/bin"
edx_ansible_user: "edx-ansible"
edx_ansible_source_repo: https://github.com/edx/configuration.git
edx_ansible_requirements_file: "{{ edx_ansible_code_dir }}/requirements.txt"
# edX configuration repo
configuration_version: master
playbooks/roles/edx_ansible/meta/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Role includes for role edx_ansible
dependencies:
- supervisor
playbooks/roles/edx_ansible/tasks/deploy.yml 0 → 100644
---
- name: edx_ansible | git checkout edx_ansible repo into edx_ansible_code_dir
git: dest={{ edx_ansible_code_dir }} repo={{ edx_ansible_source_repo }} version={{ configuration_version }}
sudo_user: "{{ edx_ansible_user }}"
tags: deploy
- name : edx_ansible | install edx_ansible venv requirements
pip: requirements="{{ edx_ansible_requirements_file }}" virtualenv="{{ edx_ansible_venv_dir }}" state=present
sudo_user: "{{ edx_ansible_user }}"
tags: deploy
playbooks/roles/edx_ansible/tasks/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
# Tasks for role edx_ansible
#
# Overview:
#
# This is an edx_ansible role that installs edx_ansible :)
# The purpose is to install edx_ansible on a server so
# that it can be updated locally.
#
# This role will also drop some helper scripts that
# for running edx_ansible tasks
#
# Example play:
#
#
#
- name: edx_ansible | create application user
user: >
name="{{ edx_ansible_user }}"
home="{{ edx_ansible_app_dir }}"
createhome=no
shell=/bin/false
- name: edx_ansible | create edx_ansible app and venv dir
file: >
path="{{ item }}"
state=directory
owner="{{ edx_ansible_user }}"
group="{{ common_web_group }}"
with_items:
- "{{ edx_ansible_app_dir }}"
- "{{ edx_ansible_venvs_dir }}"
- name: edx_ansible | install a bunch of system packages on which edx_ansible relies
apt: pkg={{','.join(edx_ansible_debian_pkgs)}} state=present
- include: deploy.yml
- name: edx_ansible | create update script
template: >
dest={{ edx_ansible_app_dir}}/update
src=update.j2 owner={{ edx_ansible_user }} group={{ edx_ansible_user }} mode=755
- name: edxapp | create a symlink for update.sh
file: >
src={{ edx_ansible_app_dir }}/update
dest={{ COMMON_BIN_DIR }}/update
state=link
playbooks/roles/edx_ansible/templates/update.j2 0 → 100644
#!/usr/bin/env bash
# This script runs edx_ansible locally
set -e
usage() {
SAVE_IFS=$IFS
IFS=","
cat<<EO
Usage: $PROG <repo> <version>
-v add verbosity to edx_ansible run
-h this
<repo> - must be one of [${!repos_to_cmd[*]}]
<version> - can be a commit or tag
EO
IFS=$SAVE_IFS
}
declare -A repos_to_cmd
edx_ansible_cmd="{{ edx_ansible_venv_bin}}/ansible-playbook -i localhost, -c local --tags deploy"
repos_to_cmd["edx-platform"]="$edx_ansible_cmd edxapp.yml -e 'edx_platform_version=$2'"
repos_to_cmd["xqueue"]="$edx_ansible_cmd xqueue.yml -e 'xqueue_version=$2'"
repos_to_cmd["forums"]="$edx_ansible_cmd forums.yml -e 'forum_version=$2'"
repos_to_cmd["xserver"]="$edx_ansible_cmd forums.yml -e 'xserver_version=$2'"
repos_to_cmd["ease"]="$edx_ansible_cmd discern.yml -e 'discern_ease_version=$2' && $edx_ansible_cmd ora.yml -e 'ora_ease_version=$2'"
repos_to_cmd["discern"]="$edx_ansible_cmd discern.yml -e 'discern_version=$2'"
repos_to_cmd["edx-ora"]="$edx_ansible_cmd ora.yml -e 'ora_version=$2'"
repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'configuration_version=$2'"
PROG=${0##*/}
while getopts "vh" opt; do
case $opt in
v)
verbose="-vvvv"
shift
;;
h)
usage
exit 0
;;
esac
done
if [[ -z $1 || -z $2 ]]; then
echo
echo "ERROR: You must specify a repo and commit"
usage
exit 1
fi
if [[ -z ${repos_to_cmd[$1]} ]]; then
echo
echo "ERROR: Invalid repo name"
usage
exit 1
fi
cd {{ edx_ansible_code_dir }}/playbooks/edx-east
eval "sudo ${repos_to_cmd["$1"]} $verbose"
playbooks/roles/edxapp/defaults/main.yml
......@@ -45,8 +45,6 @@ EDXAPP_COMMENTS_SERVICE_KEY: 'password'
EDXAPP_EDXAPP_SECRET_KEY: ''
EDXAPP_PEARSON_TEST_PASWORD: ''
EDXAPP_OEE_URL: 'http://localhost:18060/'
EDXAPP_OEE_USER: 'lms'
EDXAPP_OEE_PASSWORD: 'password'
......@@ -59,15 +57,15 @@ EDXAPP_CELERY_PASSWORD: 'celery'
EDXAPP_PLATFORM_NAME: 'edX'
EDXAPP_MITX_FEATURES:
EDXAPP_FEATURES:
AUTH_USE_OPENID_PROVIDER: true
CERTIFICATES_ENABLED: true
ENABLE_DISCUSSION_SERVICE: true
ENABLE_INSTRUCTOR_ANALYTICS: true
ENABLE_PEARSON_HACK_TEST: false
SUBDOMAIN_BRANDING: false
SUBDOMAIN_COURSE_LISTINGS: false
PREVIEW_LMS_BASE: $EDXAPP_PREVIEW_LMS_BASE
ENABLE_S3_GRADE_DOWNLOADS: true
EDXAPP_BOOK_URL: ''
# This needs to be set to localhost
......@@ -88,24 +86,32 @@ EDXAPP_LMS_NGINX_PORT: 18000
EDXAPP_LMS_PREVIEW_NGINX_PORT: 18020
EDXAPP_CMS_NGINX_PORT: 18010
EDXAPP_LMS_BASIC_AUTH: False
EDXAPP_CMS_BASIC_AUTH: False
EDXAPP_LMS_PREVIEW_BASIC_AUTH: False
EDXAPP_LANG: 'en_US.UTF-8'
EDXAPP_TIME_ZONE: 'America/New_York'
EDXAPP_TECH_SUPPORT_EMAIL: ''
EDXAPP_CONTACT_EMAIL: ''
EDXAPP_BUGS_EMAIL: ''
EDXAPP_DEFAULT_FROM_EMAIL: ''
EDXAPP_DEFAULT_FEEDBACK_EMAIL: ''
EDXAPP_SERVER_EMAIL: ''
EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL: ''
EDXAPP_TECH_SUPPORT_EMAIL: 'technical@example.com'
EDXAPP_CONTACT_EMAIL: 'info@example.com'
EDXAPP_BUGS_EMAIL: 'bugs@example.com'
EDXAPP_DEFAULT_FROM_EMAIL: 'registration@example.com'
EDXAPP_DEFAULT_FEEDBACK_EMAIL: 'feedback@example.com'
EDXAPP_DEFAULT_SERVER_EMAIL: 'devops@example.com'
EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL: 'no-reply@example.com'
EDXAPP_ENV_EXTRA: {}
EDXAPP_AUTH_EXTRA: {}
EDXAPP_MKTG_URL_LINK_MAP: {}
# Set this sets the url for static files
# Override this var to use a CDN
# Example: xxxxx.cloudfront.net/static/
EDXAPP_STATIC_URL_BASE: "/static/"
# Settings for Grade downloads
EDXAPP_GRADE_STORAGE_TYPE: 'localfs'
EDXAPP_GRADE_BUCKET: 'edx-grades'
EDXAPP_GRADE_ROOT_PATH: '/tmp/edx-s3/grades'
# Configure rake tasks in edx-platform to skip Python/Ruby/Node installation
EDXAPP_NO_PREREQ_INSTALL: 1
#-------- Everything below this line is internal to the role ------------
......@@ -181,7 +187,7 @@ edxapp_all_req_files:
edxapp_environment:
LANG: $EDXAPP_LANG
NO_PREREQ_INSTALL: 1
NO_PREREQ_INSTALL: $EDXAPP_NO_PREREQ_INSTALL
SKIP_WS_MIGRATIONS: 1
RBENV_ROOT: $edxapp_rbenv_root
GEM_HOME: $edxapp_gem_root
......@@ -228,7 +234,7 @@ edxapp_generic_auth_config: &edxapp_generic_auth
host: $EDXAPP_MONGO_HOSTS
password: $EDXAPP_MONGO_PASSWORD
port: $EDXAPP_MONGO_PORT
render_template: 'mitxmako.shortcuts.render_to_string'
render_template: 'edxmako.shortcuts.render_to_string'
# Needed for the CMS to be able to run update_templates
user: $EDXAPP_MONGO_USER
DOC_STORE_CONFIG: *edxapp_generic_default_docstore
......@@ -244,7 +250,6 @@ edxapp_generic_auth_config: &edxapp_generic_auth
PASSWORD: $EDXAPP_MYSQL_PASSWORD
HOST: $EDXAPP_MYSQL_HOST
PORT: $EDXAPP_MYSQL_PORT
PEARSON_TEST_PASSWORD: $EDXAPP_PEARSON_TEST_PASSWORD
OPEN_ENDED_GRADING_INTERFACE:
url: $EDXAPP_OEE_URL
password: $EDXAPP_OEE_PASSWORD
......@@ -259,6 +264,11 @@ edxapp_generic_auth_config: &edxapp_generic_auth
CELERY_BROKER_PASSWORD: $EDXAPP_CELERY_PASSWORD
generic_env_config: &edxapp_generic_env
GRADES_DOWNLOAD:
STORAGE_TYPE: $EDXAPP_GRADE_STORAGE_TYPE
BUCKET: $EDXAPP_GRADE_BUCKET
ROOT_PATH: $EDXAPP_GRADE_ROOT_PATH
STATIC_URL_BASE: $EDXAPP_STATIC_URL_BASE
STATIC_ROOT_BASE: $edxapp_staticfile_dir
LMS_BASE: $EDXAPP_LMS_BASE
CMS_BASE: $EDXAPP_CMS_BASE
......@@ -268,7 +278,7 @@ generic_env_config: &edxapp_generic_env
LOCAL_LOGLEVEL: $EDXAPP_LOG_LEVEL
# default email backed set to local SMTP
EMAIL_BACKEND: $EDXAPP_EMAIL_BACKEND
MITX_FEATURES: $EDXAPP_MITX_FEATURES
FEATURES: $EDXAPP_FEATURES
WIKI_ENABLED: true
SYSLOG_SERVER: $EDXAPP_SYSLOG_SERVER
SITE_NAME: $EDXAPP_SITE_NAME
......@@ -278,6 +288,8 @@ generic_env_config: &edxapp_generic_env
FEEDBACK_SUBMISSION_EMAIL: $EDXAPP_FEEDBACK_SUBMISSION_EMAIL
TIME_ZONE: $EDXAPP_TIME_ZONE
MKTG_URL_LINK_MAP: $EDXAPP_MKTG_URL_LINK_MAP
# repo root for courses
GITHUB_REPO_ROOT: $edxapp_course_data_dir
CACHES:
default: &default_generic_cache
BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
......@@ -336,7 +348,7 @@ lms_auth_config:
host: $EDXAPP_MONGO_HOSTS
db: $EDXAPP_MONGO_DB_NAME
collection: 'modulestore'
render_template: 'mitxmako.shortcuts.render_to_string'
render_template: 'edxmako.shortcuts.render_to_string'
user: $EDXAPP_MONGO_USER
password: $EDXAPP_MONGO_PASSWORD
port: $EDXAPP_MONGO_PORT
......@@ -401,9 +413,9 @@ edxapp_theme_version: 'HEAD'
# make this the public URL instead of writable
edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-platform.git"
# `edx_platform_commit` can be anything that git recognizes as a commit
# `edx_platform_version` can be anything that git recognizes as a commit
# reference, including a tag, a branch name, or a commit hash
edx_platform_commit: 'release'
edx_platform_version: 'release'
local_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/local.txt"
pre_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/pre.txt"
post_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/post.txt"
......@@ -446,6 +458,8 @@ edxapp_debian_pkgs:
- ntp
# for shapely
- libgeos-dev
# i18n
- gettext
# Ruby Specific Vars
edxapp_ruby_version: "1.9.3-p374"
......@@ -457,6 +471,3 @@ edxapp_cms_variant: cms
# Worker Settings
worker_django_settings_module: 'aws'
# Skip supervisor tasks
# Useful when supervisor is not installed (local dev)
devstack: False
playbooks/roles/edxapp/handlers/main.yml
---
- name: edxapp | restart edxapp
supervisorctl: >
supervisorctl_local: >
state=restarted
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
name="edxapp:{{ item }}"
when: not devstack
when: celery_worker is not defined and not devstack
sudo_user: "{{ supervisor_service_user }}"
with_items: service_variants_enabled
tags: deploy
- name: edxapp | restart edxapp_workers
supervisorctl: >
supervisorctl_local: >
name="edxapp_worker:{{ item.service_variant }}_{{ item.queue }}_{{ item.concurrency }}"
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/edxapp/meta/main.yml
---
dependencies:
- supervisor
- role: rbenv
rbenv_user: "{{ edxapp_user }}"
rbenv_dir: "{{ edxapp_app_dir }}"
......
playbooks/roles/edxapp/tasks/deploy.yml
- name: edxapp | setup the edxapp env
notify:
- "edxapp | restart edxapp"
- "edxapp | restart edxapp_workers"
template: >
src=edxapp_env.j2 dest={{ edxapp_app_dir }}/edxapp_env
owner={{ edxapp_user }} group={{ common_web_user }}
mode=0644
tags: deploy
# Do A Checkout
- name: edxapp | checkout edx-platform repo into {{edxapp_code_dir}}
git: dest={{edxapp_code_dir}} repo={{edx_platform_repo}} version={{edx_platform_commit}}
git: dest={{edxapp_code_dir}} repo={{edx_platform_repo}} version={{edx_platform_version}}
register: chkout
sudo_user: "{{ edxapp_user }}"
notify:
......@@ -201,8 +211,7 @@
# https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
# This is necessary for when syncdb is run and the django_openid_auth module is installed,
# not sure if this fix will ever get merged
# We should never do this in production
- name: edxapp | openid workaround - NOT FOR PRODUCTION
- name: edxapp | openid workaround
shell: sed -i -e 's/claimed_id = models.TextField(max_length=2047, unique=True/claimed_id = models.TextField(max_length=2047/' {{ edxapp_venv_dir }}/lib/python2.7/site-packages/django_openid_auth/models.py
when: openid_workaround is defined
sudo_user: "{{ edxapp_user }}"
......@@ -231,18 +240,18 @@
tags: deploy
- name: edxapp | ensure edxapp has started
supervisorctl: >
supervisorctl_local: >
state=started
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
name="edxapp:{{ item }}"
sudo_user: "{{ supervisor_service_user }}"
when: not devstack
when: celery_worker is not defined and not devstack
with_items: service_variants_enabled
tags: deploy
- name: edxapp | ensure edxapp_workers has started
supervisorctl: >
supervisorctl_local: >
name="edxapp_worker:{{ item.service_variant }}_{{ item.queue }}_{{ item.concurrency }}"
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/edxapp/tasks/main.yml
......@@ -68,15 +68,6 @@
mode=0750
with_items: service_variants_enabled
- name: edxapp | setup the edxapp env
notify:
- "edxapp | restart edxapp"
- "edxapp | restart edxapp_workers"
template: >
src=edxapp_env.j2 dest={{ edxapp_app_dir }}/edxapp_env
owner={{ edxapp_user }} group={{ common_web_user }}
mode=0644
- include: deploy.yml
- name: edxapp | create a symlink for venv python
......
playbooks/roles/edxapp/tasks/service_variant_config.yml
......@@ -63,7 +63,7 @@
executable=/bin/bash
chdir={{ edxapp_code_dir }}
sudo_user: "{{ edxapp_user }}"
when: celery_worker is not defined
when: celery_worker is not defined and not devstack and item != "lms-preview"
with_items: service_variants_enabled
notify:
- "edxapp | restart edxapp"
......
playbooks/roles/edxapp/templates/lms-preview.conf.j2
# gunicorn
description "gunicorn server"
author "Calen Pennington <cpennington@mitx.mit.edu>"
start on started edxapp
stop on stopped edxapp
respawn
respawn limit 3 30
env PID=/var/tmp/lms.pid
[program:lms-preview]
{% if ansible_processor|length > 0 %}
env WORKERS={{ ansible_processor|length * worker_core_mult.lms_preview }}
command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b {{ edxapp_lms_preview_gunicorn_host }}:{{ edxapp_lms_preview_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.lms_preview }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
{% else %}
env WORKERS={{ worker_core_mult.lms_preview }}
command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b {{ edxapp_lms_preview_gunicorn_host }}:{{ edxapp_lms_preview_gunicorn_port }} -w {{ worker_core_mult.lms_preview }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
{% endif %}
env PORT={{edxapp_lms_preview_gunicorn_port}}
env ADDRESS={{edxapp_lms_preview_gunicorn_host}}
env LANG=en_US.UTF-8
env DJANGO_SETTINGS_MODULE=lms.envs.aws
env SERVICE_VARIANT="lms-preview"
chdir {{edxapp_code_dir}}
setuid www-data
exec {{edxapp_venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edxapp_code_dir}} lms.wsgi
post-start script
while true
do
if $(curl -s -i localhost:$PORT/heartbeat | egrep -q '200 OK'); then
break;
else
sleep 1;
fi
done
end script
user={{ common_web_user }}
directory={{ edxapp_code_dir }}
environment=PORT={{edxapp_lms_gunicorn_port}},ADDRESS={{edxapp_lms_gunicorn_host}},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }},SERVICE_VARIANT="lms-preview"
stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
killasgroup=true
stopasgroup=true
playbooks/roles/elasticsearch/defaults/main.yml
......@@ -3,4 +3,4 @@
elasticsearch_version: "0.90.2"
elasticsearch_sha: "397227ce37f616734f39f6e93539c9eaa82bec9"
elasticsearch_file: "elasticsearch-{{ elasticsearch_version }}.deb"
elasticsearch_url: "https://download.elasticsearch.org/elasticsearch/elasticsearch/{{ elasticsearch_file }}"
\ No newline at end of file
elasticsearch_url: "https://download.elasticsearch.org/elasticsearch/elasticsearch/{{ elasticsearch_file }}"
playbooks/roles/forum/defaults/main.yml
......@@ -57,7 +57,3 @@ forum_services:
- {service: "mongo", host: "{{ FORUM_MONGO_HOST }}", port: "28017"}
- {service: "elasticsearch", host: "{{ forum_elasticsearch_host }}", port: "9200"}
- {service: "elasticsearch", host: "{{ forum_elasticsearch_host }}", port: "9300"}
# Skip supervisor tasks
# Used in local dev where supervisor isn't installed
devstack: False
playbooks/roles/forum/handlers/main.yml
---
- name: forum | restart the forum service
supervisorctl: >
supervisorctl_local: >
name=forum
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/forum/meta/main.yml
---
dependencies:
- supervisor
- role: rbenv
# TODO: setting the rbenv ownership to
# the common_web_user is a workaround
......
playbooks/roles/forum/tasks/deploy.yml
......@@ -49,7 +49,7 @@
tags: deploy
- name: forum | ensure forum is started
supervisorctl: >
supervisorctl_local: >
name=forum
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
......
playbooks/roles/gh_mirror/meta/main.yml 0 → 100644
---
dependencies:
- supervisor
playbooks/roles/gluster/defaults/main.yml 0 → 100644
gluster_primary_ip: 127.0.0.1
gluster_peers:
gluster_volumes:
- path: /mnt/gfsv0
name: gfsv0
replicas: 2
cache_size: 128MB
security: "*"
mount_location: /mnt/data
playbooks/roles/gluster/tasks/main.yml 0 → 100644
---
# Install and configure simple glusterFS shared storage
- name: gluster | all | Install common packages
apt: name={{ item }} state=present
with_items:
- glusterfs-client
- glusterfs-common
- nfs-common
tags: gluster
- name: gluster | all | Install server packages
apt: name=glusterfs-server state=present
when: >
"{{ ansible_default_ipv4.address }}" "{{ gluster_peers|join(' ') }}"
tags: gluster
- name: gluster | all | enable server
service: name=glusterfs-server state=started enabled=yes
when: >
"{{ ansible_default_ipv4.address }}" in "{{ gluster_peers|join(' ') }}"
tags: gluster
# Ignoring error below so that we can move the data folder and have it be a link
- name: gluster | all | create folders
file: path={{ item.path }} state=directory
with_items: gluster_volumes
when: >
"{{ ansible_default_ipv4.address }}" in "{{ gluster_peers|join(' ') }}"
ignore_errors: yes
tags: gluster
- name: gluster | primary | create peers
command: gluster peer probe {{ item }}
with_items: gluster_peers
when: ansible_default_ipv4.address == gluster_primary_ip
tags: gluster
- name: gluster | primary | create volumes
command: gluster volume create {{ item.name }} replica {{ item.replicas }} transport tcp {% for server in gluster_peers %}{{ server }}:{{ item.path }} {% endfor %}
with_items: gluster_volumes
when: ansible_default_ipv4.address == gluster_primary_ip
ignore_errors: yes # There should be better error checking here
tags: gluster
- name: gluster | primary | start volumes
command: gluster volume start {{ item.name }}
with_items: gluster_volumes
when: ansible_default_ipv4.address == gluster_primary_ip
ignore_errors: yes # There should be better error checking here
tags: gluster
- name: gluster | primary | set security
command: gluster volume set {{ item.name }} auth.allow {{ item.security }}
with_items: gluster_volumes
when: ansible_default_ipv4.address == gluster_primary_ip
tags: gluster
- name: gluster | primary | set performance cache
command: gluster volume set {{ item.name }} performance.cache-size {{ item.cache_size }}
with_items: gluster_volumes
when: ansible_default_ipv4.address == gluster_primary_ip
tags: gluster
- name: gluster | all | mount volume
mount: >
name={{ item.mount_location }}
src={{ gluster_primary_ip }}:{{ item.name }}
fstype=glusterfs
state=mounted
opts=defaults,_netdev
with_items: gluster_volumes
tags: gluster
# This required due to an annoying bug in Ubuntu and gluster where it tries to mount the system
# before the network stack is up and can't lookup 127.0.0.1
- name: gluster | all | sleep mount
lineinfile: >
dest=/etc/rc.local
line='sleep 5; /bin/mount -a'
regexp='sleep 5; /bin/mount -a'
insertbefore='exit 0'
tags: gluster
playbooks/roles/haproxy/defaults/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Defaults for role haproxy
#
#
# vars are namespace with the module name.
#
haproxy_role_name: haproxy
#
# OS packages
#
haproxy_debian_pkgs: []
haproxy_redhat_pkgs: []
pkgs:
haproxy:
state: installed
haproxy_template_dir: .
haproxy_extra_global_config: ''
haproxy_default_config: |
log global
mode http
option httplog
option dontlognull
option redispatch
retries 3
maxconn 2000
contimeout 5000
clitimeout 50000
srvtimeout 50000
# Sample rabbitmq load balance config
# but this should likely get overidden with your
# desired applications
haproxy_applications:
- |
listen rabbitmq 127.0.0.1:5672
mode tcp
balance roundrobin
option tcplog
option tcpka
server rabbit01 172.23.128.10:5672 check inter 5000 rise 2 fall 3
server rabbit02 172.23.129.10:5672 backup check inter 5000 rise 2 fall 3
server rabbit03 172.23.130.10:5672 backup check inter 5000 rise 2 fall 3
playbooks/roles/haproxy/handlers/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
#
# Handlers for role haproxy
#
# Overview:
#
#
- name: haproxy | restart haproxy
service: name=haproxy state=restarted
- name: haproxy | reload haproxy
service: name=haproxy state=reloaded
- name: haproxy | restart rsyslog
service: name=rsyslog state=restarted
playbooks/roles/haproxy/meta/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Role includes for role haproxy
#
# Example:
#
# dependencies:
# - {
# role: my_role
# my_role_var0: "foo"
# my_role_var1: "bar"
# }
playbooks/roles/haproxy/tasks/main.yml 0 → 100644
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
#
# Tasks for role haproxy
#
# Overview:
# Installs and configures haproxy for load balancing.
# HAProxy doesn't currently support included configuration
# so it allows for a configuration template to be overriden
# with a variable
- name: haproxy | Install haproxy
apt: pkg=haproxy state={{ pkgs.haproxy.state }}
notify: haproxy | restart haproxy
- name: haproxy | Server configuration file
template: >
src={{ haproxy_template_dir }}/haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
owner=root group=root mode=0644
notify: haproxy | reload haproxy
- name: haproxy | Enabled in default
lineinfile: dest=/etc/default/haproxy regexp=^ENABLED=.$ line=ENABLED=1
notify: haproxy | restart haproxy
- name: haproxy | install logrotate
template: src=haproxy.logrotate.j2 dest=/etc/logrotate.d/haproxy mode=0644
- name: haproxy | install rsyslog conf
template: src=haproxy.rsyslog.j2 dest=/etc/rsyslog.d/haproxy.conf mode=0644
notify: haproxy | restart rsyslog
- name: haproxy | make sure haproxy has started
service: name=haproxy state=started
playbooks/roles/haproxy/templates/haproxy.cfg.j2 0 → 100644
# this config needs haproxy-1.1.28 or haproxy-1.2.1
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
#log loghost local0 info
maxconn 4096
#chroot /usr/share/haproxy
user haproxy
group haproxy
daemon
#debug
#quiet
{{ haproxy_extra_global_config }}
defaults
{{ haproxy_default_config }}
{%- for app in haproxy_applications -%}
{{ app }}
{%- endfor -%}
playbooks/roles/haproxy/templates/haproxy.logrotate.j2 0 → 100644
{{ COMMON_LOG_DIR }}/haproxy/*.log {
weekly
missingok
rotate 7
compress
delaycompress
notifempty
create 640 root adm
sharedscripts
postrotate
/etc/init.d/haproxy reload > /dev/null
endscript
}
playbooks/roles/haproxy/templates/haproxy.rsyslog.j2 0 → 100644
if ($programname == 'haproxy' and $syslogseverity-text == 'info') then -{{ COMMON_LOG_DIR }}/haproxy/haproxy-info.log
& ~
if ($programname == 'haproxy' and $syslogseverity-text == 'notice') then -{{ COMMON_LOG_DIR }}/haproxy/haproxy-notice.log
& ~
playbooks/roles/jenkins_master/defaults/main.yml
......@@ -68,3 +68,6 @@ jenkins_debian_pkgs:
- maven
- daemon
- python-pycurl
# Extra packages need for a specific jenkins instance.
JENKINS_EXTRA_PKGS: []
playbooks/roles/jenkins_master/tasks/main.yml
......@@ -7,6 +7,13 @@
tags:
- jenkins
- name: jenkins_master | install jenkins extra system packages
apt:
pkg={{','.join(JENKINS_EXTRA_PKGS)}}
state=present update_cache=yes
tags:
- jenkins
- name: jenkins_master | create jenkins group
group: name={{ jenkins_group }} state=present
......
playbooks/roles/jenkins_worker/tasks/main.yml
......@@ -10,5 +10,4 @@
- include: system.yml
- include: python.yml
- include: browsers.yml
- include: jscover.yml
playbooks/roles/launch_ec2/tasks/main.yml
......@@ -18,15 +18,25 @@
- name: terminating single instance
local_action:
module: ec2
module: ec2_local
state: 'absent'
region: "{{ region }}"
instance_ids: ${tag_lookup.instance_ids}
when: terminate_instance == true and tag_lookup.instance_ids|length == 1
- name: deregister instance from an an elb if it was in one
local_action:
module: ec2_elb
region: "{{ region }}"
instance_id: "{{ tag_lookup.instance_ids[0] }}"
ec2_elbs:
- "{{ elb }}"
state: absent
when: terminate_instance == true and elb and tag_lookup.instance_ids|length == 1
- name: launch_ec2 | Launch ec2 instance
local_action:
module: ec2
module: ec2_local
keypair: "{{ keypair }}"
group: "{{ security_group }}"
instance_type: "{{ instance_type }}"
......@@ -35,6 +45,8 @@
region: "{{ region }}"
instance_tags: "{{instance_tags}}"
root_ebs_size: "{{ root_ebs_size }}"
zone: "{{ zone }}"
instance_profile_name: "{{ instance_profile_name }}"
register: ec2
- name: launch_ec2 | Add DNS name
......@@ -90,5 +102,3 @@
delay=60
timeout=320
with_items: "{{ ec2.instances }}"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment