Merge pull request #2021 from edx/e0d/service-dbs

E0d/service dbs

playbooks/edx-east/edx_service_rds.yml

+---
+
+- name: Build service RDS instances
+  hosts: all
+  connection: local
+  # Needed for timestamps
+  gather_facts: True
+  roles:
+    - edx_service_rds  
\ No newline at end of file

playbooks/roles/ansible-role/templates/defaults/main.yml.j2

@@ -5,7 +5,7 @@
 # 
 
 #
-# vars are namespace with the module name.
+# vars are namespaced with the module name.
 #
 {{ role_name }}_role_name: {{ role_name }}
 

playbooks/roles/edx_service_rds/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role edx_service_rds
+# 
+
+#
+# vars are namespaced with the module name.
+#
+edx_service_rds_role_name: edx_service_rds
+
+E_D_C: "prod-sample-app"
+EDX_SERVICE_RDS_INSTANCE_SIZE: 10
+EDX_SERVICE_RDS_INSTANCE_TYPE: "db.m1.small"
+EDX_SERVICE_RDS_ROOT_USER: "root"
+# no unicode, c cedilla , passwords
+EDX_SERVICE_RDS_ROOT_PASSWORD: "plus_ca_change"
+EDX_SERVICE_RDS_ENGINE: "MySQL"
+EDX_SERVICE_RDS_ENGINE_VERSION: "5.6.22"
+EDX_SERVICE_RDS_PARAM_GROUP_ENGINE: "mysql5.6"
+# will vary depending upon engine, examples assume
+# MySQL 56
+EDX_SERVICE_RDS_PARAM_GROUP_PARAMS:
+  character_set_client: "utf8"
+  character_set_connection: "utf8"
+  character_set_database: "utf8"
+  character_set_filesystem: "utf8"
+  character_set_results: "utf8"
+  character_set_server: "utf8"
+  collation_connection: "utf8_unicode_ci"
+  collation_server: "utf8_unicode_ci"
+EDX_SERVICE_RDS_MULTI_AZ: No
+EDX_SERVICE_RDS_MAINT_WINDOW: "Mon:00:00-Mon:01:15"
+EDX_SERVICE_RDS_BACKUP_DAYS: 30
+EDX_SERVICE_RDS_BACKUP_WINDOW: "02:00-03:00"
+
+EDX_SERVICE_RDS_SUBNET_1_AZ: "us-east-1c"
+EDX_SERVICE_RDS_SUBNET_1_CIDR: "10.253.50.0/24"
+EDX_SERVICE_RDS_SUBNET_2_AZ: "us-east-1d"
+EDX_SERVICE_RDS_SUBNET_2_CIDR: "10.253.51.0/24"
+
+# The defaults are permissive, override
+EDX_SERVICE_RDS_SECURITY_GROUP:
+  name: "{{ e_d_c }}-rds-sg"
+  description: "RDS ingress and egress."
+  rules:
+    - proto: "tcp"
+      from_port: "3306"
+      to_port: "3306"
+      cidr_ip: "0.0.0.0/0"
+  rules_egress:
+    - proto: "tcp"
+      from_port: "3306"
+      to_port: "3306"
+      cidr_ip: "0.0.0.0/0"
+
+# The defaults are permissive, override
+EDX_SERVICE_RDS_VPC_DB_ACL:
+  name: "{{ e_d_c }}-db"
+  rules:
+    - number: "100"
+      type: "ingress"
+      protocol: "tcp"
+      from_port: 3306
+      to_port: 3306
+      cidr_block: "0.0.0.0/0"
+      rule_action: "allow"
+    - number: "100"
+      type: "egress"
+      protocol: "all"
+      from_port: 0
+      to_port: 65535
+      cidr_block: "0.0.0.0/0"
+      rule_action: "allow"
+
+EDX_SERVICE_RDS_VPC_DB_ROUTE_TABLE:
+  - cidr: "10.253.0.0/16"
+    gateway: 'local'
+      
+# typically override the all caps vars, but may
+# be convenient to override the entire structure
+# if you spanning more than two subnets
+edx_service_rds_vpc_db_subnets:
+  - name: "{{ E_D_C }}-db-{{ EDX_SERVICE_RDS_SUBNET_1_AZ }}"
+    cidr: "{{ EDX_SERVICE_RDS_SUBNET_1_CIDR }}"
+    az: "{{ EDX_SERVICE_RDS_SUBNET_1_AZ }}"
+  - name: "{{ E_D_C }}-db-{{ EDX_SERVICE_RDS_SUBNET_2_AZ }}"
+    cidr: "{{ EDX_SERVICE_RDS_SUBNET_2_CIDR }}"
+    az: "{{ EDX_SERVICE_RDS_SUBNET_2_AZ }}"
+      
+edx_service_rds_state: "present"
+
+edx_service_rds_db:
+  state: "{{ edx_service_rds_state }}"
+  name: "{{ E_D_C }}-primary"
+  size: "{{ EDX_SERVICE_RDS_INSTANCE_SIZE }}"
+  instance_type: "{{ EDX_SERVICE_RDS_INSTANCE_TYPE }}"
+  root_user: "{{ EDX_SERVICE_RDS_ROOT_USER }}"
+  root_password: "{{ EDX_SERVICE_RDS_ROOT_PASSWORD }}"
+  engine: "{{ EDX_SERVICE_RDS_ENGINE }}"
+  engine_version: "{{ EDX_SERVICE_RDS_ENGINE_VERSION }}"
+  multi_az: "{{ EDX_SERVICE_RDS_MULTI_AZ }}"
+  maint_window: "{{ EDX_SERVICE_RDS_MAINT_WINDOW }}"
+  backup_days: "{{ EDX_SERVICE_RDS_BACKUP_DAYS }}"
+  backup_window: "{{ EDX_SERVICE_RDS_BACKUP_WINDOW }}"
+  param_group:
+    name: "{{ E_D_C}}"
+    engine: "{{ EDX_SERVICE_RDS_PARAM_GROUP_ENGINE }}"
+    params: "{{ EDX_SERVICE_RDS_PARAM_GROUP_PARAMS }}"
+      
+
+#
+# OS packages
+#
+
+edx_service_rds_debian_pkgs: []
+
+edx_service_rds_redhat_pkgs: []

playbooks/roles/edx_service_rds/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role edx_service_rds
+# 
+# Overview:
+#
+# Creates a VPC RDS instance and related network infrastructure, e.g.,
+# subnets, subnet groups, acls, as well as an instance specific
+# parameter group.
+#
+# NB: When using a boto profile other than the default, you will need
+# to export AWS_PROFILE because some tasks do not properly process
+# the profile argument.
+#
+# NB: You should currently not use this play for deleting databases as
+# the final snapshot functionality doesn't work properly in the ansible
+# module.  First it default to not taking a final snapshot and
+# when you specify one, it throw a key error.
+#
+# Dependencies:
+#
+# Assumes a working VPC, ideally created via the edx_vpc role as this
+# role will produce configuration output that this role requires
+# like the VPC, route table and subnet IDs.
+# 
+# Example play:
+#
+# export AWS_PROFILE=sandbox
+# ansible-playbook -c local -i 'localhost,' edx_service_rds.yml -e@~/vpc-test.yml -e@~/e0dTest-edx.yml -e 'cluster=test'
+#
+# TODO:
+# - handle db deletes and updates
+# - handle DNS updates, consider that a different profile may be required for this.
+#
+
+- name: create database route table
+  ec2_rt:
+    profile: "{{ profile }}"
+    vpc_id: "{{ vpc_id }}"
+    region: "{{ aws_region }}"
+    state: "{{ edx_service_rds_state }}"
+    name: "{{ e_d_c }}-db"
+    routes: "{{ EDX_SERVICE_RDS_VPC_DB_ROUTE_TABLE }}"
+  register: created_db_rt
+
+- name: create db network acl
+  ec2_acl:
+    profile: "{{ profile }}"
+    name: "{{ EDX_SERVICE_RDS_VPC_DB_ACL.name }}"
+    vpc_id: "{{ vpc_id }}"
+    state: "{{ edx_service_rds_state }}"
+    region: "{{ aws_region }}"
+    rules: "{{ EDX_SERVICE_RDS_VPC_DB_ACL.rules }}"
+  register: created_db_acl
+
+- name: create db subnets
+  ec2_subnet:
+    profile: "{{ profile }}"
+    vpc_id: "{{ vpc_id }}"
+    region: "{{ aws_region }}"
+    state: "{{ edx_service_rds_state }}"
+    name: "{{ item.name }}"
+    cidr: "{{ item.cidr }}"
+    az: "{{ item.az }}"
+    route_table_id: "{{ created_db_rt.id }}"
+    network_acl_id: "{{ created_db_acl.id }}"
+  with_items: edx_service_rds_vpc_db_subnets
+  register: created_db_subnets
+
+- name: Apply function to subnet data
+  util_map:
+    function: 'zip_to_list'
+    input: "{{ created_db_subnets.results }}"
+    args:
+      - "subnet_id"
+  register: subnet_data
+
+- name:
+  rds_subnet_group:
+    state: "{{ edx_service_rds_state }}"
+    profile: "{{ profile }}"
+    region: "{{ aws_region }}"
+    name: "{{ e_d_c }}"
+    description: "{{ e_d_c }}"
+    subnets: "{{ subnet_data.function_output }}"
+
+- name: create RDS security group
+  ec2_group:
+    profile: "{{ profile }}"
+    vpc_id: "{{ vpc_id }}"
+    state: "{{ edx_service_rds_state }}"
+    region: "{{ aws_region }}"
+    name: "{{ EDX_SERVICE_RDS_SECURITY_GROUP.name }}"
+    rules: "{{ EDX_SERVICE_RDS_SECURITY_GROUP.rules }}"
+    description: "{{ EDX_SERVICE_RDS_SECURITY_GROUP.description }}"
+    rules_egress: "{{ EDX_SERVICE_RDS_SECURITY_GROUP.rules_egress }}"
+  register: created_rds_security_group
+
+- name: create instance parameter group
+  rds_param_group:
+    state: "{{ edx_service_rds_state }}"
+    region: "{{ aws_region }}"
+    name: "{{ edx_service_rds_db.param_group.name }}"
+    description: "{{ edx_service_rds_db.param_group.name }}"
+    engine: "{{ edx_service_rds_db.param_group.engine }}"
+    params: "{{ edx_service_rds_db.param_group.params }}"
+  register: created_param_group
+
+  #  
+  # Create the database
+  #
+- name: Create service database
+  rds:
+    command: "create"
+    region: "{{ aws_region }}"
+    instance_name: "{{ edx_service_rds_db.name }}"
+    db_engine: "{{ edx_service_rds_db.engine }}"
+    engine_version: "{{ edx_service_rds_db.engine_version }}"
+    size: "{{ edx_service_rds_db.size }}"
+    instance_type: "{{ edx_service_rds_db.instance_type }}"
+    username: "{{ edx_service_rds_db.root_user }}"
+    password: "{{ edx_service_rds_db.root_password }}"
+    subnet: "{{ e_d_c }}"
+    vpc_security_groups: "{{ created_rds_security_group.group_id }}"
+    multi_zone: "{{ edx_service_rds_db.multi_az }}"
+    maint_window: "{{ edx_service_rds_db.maint_window }}"
+    backup_window: "{{ edx_service_rds_db.backup_window }}"
+    backup_retention: "{{ edx_service_rds_db.backup_days }}"
+    parameter_group: "{{ edx_service_rds_db.param_group.name }}"
+    tags:
+      Environment: "{{ env }}"
+      Application: "{{ deployment }}"
+  when: edx_service_rds_db.state == 'present'
+  register: created_db
+
+  # 
+  # Delete the database, need to debug module for this to
+  # full work.
+  #
+- name: Delete service database
+  rds:
+    command: "delete"
+    region: "{{ aws_region }}"
+    instance_name: "{{ edx_service_rds_db.name }}"
+    # bug inthe module related to final snapshots
+    #snapshot: "{{ edx_service_rds_db.name }}-final-{{ ansible_date_time.epoch }}"
+    snapshot: "red-blue"
+  when: edx_service_rds_db.state == 'absent'
+
+  #
+  # Output the basis for a db config file that
+  # includes the yaml connection defintion
+  #
+- name: output a step db config file
+  local_action:
+    module: template
+    src: "db_config.yml.j2"
+    dest: "~/{{ e_d_c }}-db.yml"

playbooks/roles/edx_vpc/tasks/main.yml

@@ -147,30 +147,6 @@
     routes: "{{ vpc_private_route_table }}"
   register: created_private_rt
 
-- name: create db network acl
-  ec2_acl:
-    profile: "{{ vpc_aws_profile }}"
-    name: "{{ vpc_db_acl.name }}"
-    vpc_id: "{{ created_vpc.vpc_id }}"
-    state: "present"
-    region: "{{ vpc_aws_region }}"
-    rules: "{{ vpc_db_acl.rules }}"
-  register: created_db_acl
-
-- name: create db subnets
-  ec2_subnet:
-    profile: "{{ vpc_aws_profile }}"
-    vpc_id: "{{ created_vpc.vpc_id }}"
-    region: "{{ vpc_aws_region }}"
-    state: "present"
-    name: "{{ item.name }}"
-    cidr: "{{ item.cidr }}"
-    az: "{{ item.az }}"
-    route_table_id: "{{ created_private_rt.id }}"
-    network_acl_id: "{{ created_db_acl.id }}"
-  with_items: vpc_db_subnets
-  register: created_db_subnets
-
 - name: output a vpc_config for using to build services
   local_action:
     module: template

playbooks/roles/edx_vpc/templates/vpc_config.yml.j2

@@ -41,11 +41,6 @@ elb_subnets:
   - "{{ subnet.subnet_id }}"
 {% endfor %}
 
-db_subnets:
-{% for subnet in created_db_subnets.results %}
-  - "{{ subnet.subnet_id }}"
-{% endfor %}
-
 #
 # Do not use vars in policies :(
 # Should be specific to the service right?