Commit 0da13cd6 by Edward Zarecor

initial role

changes for ssh

basic play

trying to pull the yaml config template up

basic var refactor

initial working version

templates

WIP toward working deployment of notes

haystack

var refactoring and new tasks

Generalization and code review comments

Maria defaults

Maria defaults, refactor for consistency.

missed rename

var renames

removed stub

refactoring

adding tag task until generalized

adding tagging
parent 6616775c
- name: Deploy edX Notes API
hosts: all
sudo: True
gather_facts: True
vars:
ENABLE_DATADOG: False
ENABLE_SPLUNKFORWARDER: False
ENABLE_NEWRELIC: True
roles:
- role: nginx
nginx_sites:
- edx-notes-api
- aws
- edx-notes-api
- role: datadog
when: COMMON_ENABLE_DATADOG
- role: splunkforwarder
when: COMMON_ENABLE_SPLUNKFORWARDER
- role: newrelic
when: COMMON_ENABLE_NEWRELIC
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Defaults for role edx-notes-api
#
EDX_NOTES_API_VERSION: "master"
EDX_NOTES_API_GUNICORN_WORKERS_EXTRA: ""
EDX_NOTES_API_WORKERS_EXTRA_CONF: ""
EDX_NOTES_API_LANG: "en-us"
EDX_NOTES_API_WORKERS: !!null
EDX_NOTES_API_DATASTORE_NAME: "edx-notes-api"
EDX_NOTES_API_MYSQL_DB_USER: "notes001"
EDX_NOTES_API_MYSQL_DB_NAME: "{{ EDX_NOTES_API_DATASTORE_NAME }}"
EDX_NOTES_API_MYSQL_DB_PASS: "secret"
EDX_NOTES_API_MYSQL_HOST: "localhost"
EDX_NOTES_API_ELASTICSEARCH_URL: "http://127.0.0.1:9200"
EDX_NOTES_API_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-notes"
# Change these values!!
EDX_NOTES_API_SECRET_KEY: "i^,9%i=e=y/Nlpb=Mkx!j&,HD9d/17F][9P,FLdwM2+G6|]BEU"
EDX_NOTES_API_CLIENT_ID: "edx-notes-id"
EDX_NOTES_API_CLIENT_SECRET: "edx-notes-secret"
EDX_NOTES_API_DATABASES:
# rw user
default:
ENGINE: "django.db.backends.mysql"
NAME: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}"
USER: "{{ EDX_NOTES_API_MYSQL_DB_USER }}"
PASSWORD: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}"
HOST: "{{ EDX_NOTES_API_MYSQL_HOST }}"
PORT: 3306
EDX_NOTES_API_ALLOWED_HOSTS:
- "localhost"
#
# This data structure will be written out to yaml configuration file
# in /edx/etc
#
edx_notes_api_service_config:
ALLOWED_HOSTS: "{{ EDX_NOTES_API_ALLOWED_HOSTS }}"
# replace with your secret key
SECRET_KEY: '{{ EDX_NOTES_API_SECRET_KEY }}'
# replace with your oauth id and secret
CLIENT_ID: "{{ EDX_NOTES_API_CLIENT_ID }}"
CLIENT_SECRET: "{{ EDX_NOTES_API_CLIENT_SECRET }}"
ELASTICSEARCH_URL: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}"
ELASTICSEARCH_INDEX: "edx-notes"
# Number of rows to return by default in result.
RESULTS_DEFAULT_SIZE: 25
# Max number of rows to return in result.
RESULTS_MAX_SIZE: 250
DATABASE_OPTIONS:
connect_timeout: 10
DATABASES: "{{ EDX_NOTES_API_DATABASES }}"
HAYSTACK_CONNECTIONS:
default:
ENGINE: 'notesserver.highlight.ElasticsearchSearchEngine'
URL: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}"
INDEX_NAME: '{{ EDX_NOTES_API_DATASTORE_NAME }}'
DISABLE_TOKEN_CHECK: True
#
# vars are namespace with the module name.
#
edx_notes_api_role_name: "edx-notes-api"
edx_notes_api_service_name: "{{ edx_notes_api_role_name }}"
# underscore name
edx_notes_api_role_name_safe: "{{ edx_notes_api_role_name.replace('-', '_') }}"
edx_notes_api_source_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-notes-api.git"
edx_notes_api_user: "{{ edx_notes_api_role_name }}"
edx_notes_api_app_dir: "{{ COMMON_APP_DIR }}/{{ edx_notes_api_service_name }}"
edx_notes_api_home: "{{ COMMON_APP_DIR }}/{{ edx_notes_api_service_name }}"
edx_notes_api_venv_base: "{{ edx_notes_api_home }}/venvs"
edx_notes_api_venv_dir: "{{ edx_notes_api_venv_base }}/{{ edx_notes_api_service_name }}"
edx_notes_api_venv_bin: "{{ edx_notes_api_venv_dir }}/bin"
edx_notes_api_code_dir: "{{ edx_notes_api_app_dir }}/edx-notes-api"
edx_notes_api_python_path: "{{ edx_notes_api_code_dir }}/edx-notes-api"
edx_notes_api_conf_dir: "{{ edx_notes_api_home }}"
edx_notes_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ edx_notes_api_service_name }}"
edx_notes_api_gunicorn_host: "127.0.0.1"
edx_notes_api_gunicorn_port: "8120"
edx_notes_api_gunicorn_timeout: "300"
edx_notes_api_wsgi: "notesserver.wsgi:application"
edx_notes_api_nginx_port: "18120"
edx_notes_api_manage: "{{ edx_notes_api_code_dir }}/manage.py"
edx_notes_api_requirements_base: "{{ edx_notes_api_code_dir }}/requirements"
# Application python requirements
edx_notes_api_requirements:
- base.txt
- optional.txt
#
# OS packages
#
edx_notes_api_debian_pkgs:
- 'libmysqlclient-dev'
- 'python-mysqldb'
edx_notes_api_redhat_pkgs: []
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Role includes for role edx-notes-api
#
# Example:
#
# dependencies:
# - {
# role: my_role
# my_role_var0: "foo"
# my_role_var1: "bar"
# }
dependencies:
- role: edx_service
edx_role_name: "{{ edx_notes_api_role_name }}"
edx_service_name: "{{ edx_notes_api_service_name }}"
edx_service_config: "{{ edx_notes_api_service_config }}"
- supervisor
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
# Tasks for role edx-notes-api
#
# Overview:
#
# Role for installing the edx-notes-api Django application, https://github.com/edx/edx-notes-api.
#
# Dependencies:
#
# For a complete picture of dependencies, see:
#
# configuration/playbooks/role/edx-notes-api/meta/main.yml
# configuration/playbooks/edx-east/notes.yml
#
# Example play:
#
# - name: Deploy edX Notes API
# hosts: all
# sudo: True
# gather_facts: True
# vars:
# ENABLE_DATADOG: False
# ENABLE_SPLUNKFORWARDER: False
# ENABLE_NEWRELIC: True
# roles:
# - role: nginx
# nginx_sites:
# - edx-notes-api
# - aws
# - edx-notes-api
# - role: datadog
# when: COMMON_ENABLE_DATADOG
# - role: splunkforwarder
# when: COMMON_ENABLE_SPLUNKFORWARDER
# - role: newrelic
# when: COMMON_ENABLE_NEWRELIC
- name: checkout code
git: >
dest={{ edx_notes_api_code_dir }} repo={{ edx_notes_api_source_repo }} version={{ EDX_NOTES_API_VERSION }}
accept_hostkey=yes key_file="{{ edx_notes_api_home }}/.ssh/id_rsa"
register: edx_notes_api_code_checkout
sudo_user: "{{ edx_notes_api_role_name }}"
- name: install application requirements
pip: >
requirements="{{ edx_notes_api_requirements_base }}/{{ item }}"
virtualenv="{{ edx_notes_api_venv_dir }}" state=present
extra_args="--exists-action w"
sudo_user: "{{ edx_notes_api_user }}"
with_items: edx_notes_api_requirements
- name: migrate
shell: >
chdir={{ edx_notes_api_code_dir }}
DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
{{ edx_notes_api_venv_bin }}/python {{ edx_notes_api_manage }} migrate --noinput --settings="notesserver.settings.yaml_config"
sudo_user: "{{ edx_notes_api_user }}"
environment:
EDXNOTES_CONFIG_ROOT: "{{ COMMON_CFG_DIR }}"
#when: migrate_db is defined and migrate_db|lower == "yes"
- name: write out gunicorn.py
template: >
src=edx/app/edx-notes-api/{{ edx_notes_api_service_name }}-gunicorn.py.j2
dest={{ edx_notes_api_app_dir }}/{{ edx_notes_api_service_name }}-gunicorn.py
mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
tags:
- deploy
- name: write supervisord config
template: >
src=edx/app/supervisor/conf.d.available/edx_notes_api.conf.j2
dest="{{ supervisor_available_dir }}/{{ edx_notes_api_service_name }}.conf"
owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
tags:
- deploy
- name: enable supervisor script
file: >
src={{ supervisor_available_dir }}/{{ edx_notes_api_service_name }}.conf
dest={{ supervisor_cfg_dir }}/{{ edx_notes_api_service_name }}.conf
state=link
force=yes
when: not disable_edx_services
tags:
- deploy
- name: update supervisor configuration
shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
when: not disable_edx_services
tags:
- deploy
- name: restart supervisor
supervisorctl_local: >
name={{ edx_notes_api_service_name }}
supervisorctl_path={{ supervisor_ctl }}
config={{ supervisor_cfg }}
state=restarted
when: not disable_edx_services
tags:
- deploy
- name: create symlinks from the venv bin dir
file: >
src="{{ edx_notes_api_venv_bin }}/{{ item }}"
dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ edx_notes_api_role_name }}"
state=link
with_items:
- python
- pip
- django-admin.py
- name: create manage.py symlink
file: >
src="{{ edx_notes_api_manage }}"
dest="{{ COMMON_BIN_DIR }}/manage.{{ edx_notes_api_role_name }}"
state=link
- include: tag_ec2.yml tags=deploy
when: COMMON_TAG_EC2_INSTANCE
---
- name: get instance information
action: ec2_facts
- name: tag instance
ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
args:
tags:
"version:{{ edx_notes_api_service_name }}" : "{{ edx_notes_api_source_repo }} {{ edx_notes_api_code_checkout.after |truncate(7,True,'')}}"
when: edx_notes_api_code_checkout.after is defined
"""
gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
{{ ansible_managed }}
"""
import multiprocessing
preload_app = True
timeout = {{ edx_notes_api_gunicorn_timeout }}
bind = "{{ edx_notes_api_gunicorn_host }}:{{ edx_notes_api_gunicorn_port }}"
pythonpath = "{{ edx_notes_api_code_dir }}"
{% if EDX_NOTES_API_WORKERS %}
workers = {{ EDX_NOTES_API_WORKERS }}
{% else %}
workers = (multiprocessing.cpu_count()-1) * 2 + 2
{% endif %}
{{ EDX_NOTES_API_WORKERS_EXTRA_CONF }}
[program:{{ edx_notes_api_service_name }}]
{% if COMMON_ENABLE_NEWRELIC_APP %}
{% set executable = edx_notes_api_venv_bin + '/newrelic-admin run-program ' + edx_notes_api_venv_bin + '/gunicorn' %}
{% else %}
{% set executable = edx_notes_api_venv_bin + '/gunicorn' %}
{% endif %}
command={{ executable }} -c {{ edx_notes_api_app_dir }}/edx-notes-api-gunicorn.py {{ EDX_NOTES_API_GUNICORN_WORKERS_EXTRA }} {{ edx_notes_api_wsgi }}
user={{ common_web_user }}
directory={{ edx_notes_api_code_dir }}
environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDX_NOTES_API_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PID=/var/tmp/edx_notes_api.pid,PORT={{ edx_notes_api_gunicorn_port }},ADDRESS={{ edx_notes_api_gunicorn_host }},LANG={{ EDX_NOTES_API_LANG }},DJANGO_SETTINGS_MODULE=notesserver.settings.yaml_config,SERVICE_VARIANT="{{ edx_notes_api_service_name }}",EDXNOTES_CONFIG_ROOT="{{ COMMON_CFG_DIR }}"
stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
killasgroup=true
stopasgroup=true
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Defaults shared across multiple roles. They are prefixed with the
# COMMON namespace and are mostly used by the common role. They are
# broken out into a separate role so the can be used with running common
#
COMMON_MYSQL_READ_ONLY_USER: 'read_only'
COMMON_MYSQL_READ_ONLY_PASS: 'password'
COMMON_MYSQL_ADMIN_USER: 'admin'
COMMON_MYSQL_ADMIN_PASS: 'password'
COMMON_MYSQL_MIGRATE_USER: 'migrate'
COMMON_MYSQL_MIGRATE_PASS: 'password'
COMMON_MONGO_READ_ONLY_USER: 'read_only'
COMMON_MONGO_READ_ONLY_PASS: !!null
COMMON_ENABLE_DATADOG: False
COMMON_ENABLE_NGINXTRA: False
COMMON_ENABLE_SPLUNKFORWARDER: False
COMMON_ENABLE_NEWRELIC: False
# enables app reporting, you must enable newrelic
# as well
COMMON_ENABLE_NEWRELIC_APP: False
COMMON_ENABLE_MINOS: False
COMMON_TAG_EC2_INSTANCE: False
common_debian_pkgs:
- ntp
- ack-grep
- lynx-cur
- logrotate
- mosh
- rsyslog
- screen
- tmux
- tree
- git
- unzip
- python2.7
- python-pip
- python2.7-dev
# Not installed by default on vagrant ubuntu
# boxes
- curl
common_pip_pkgs:
- pip==1.5.6
- setuptools==3.6
- virtualenv==1.11.6
- virtualenvwrapper
common_web_user: www-data
common_web_group: www-data
common_log_user: syslog
common_git_ppa: "ppa:git-core/ppa"
# Skip supervisor tasks
# When set to true this flag will allow you to install everything but keep
# supervisor from starting any of the services.
# Service files will be placed in supervisor's conf.available.d but not linked
# to supervisors 'conf.d' directory.
disable_edx_services: False
# Some apps run differently in dev mode(forums)
# so different start scripts are generated in dev mode.
devstack: False
# Some cluster apps need special settings when in vagrant
# due to eth0 always being the same IP address
vagrant_cluster: False
common_debian_variants:
- Ubuntu
- Debian
common_redhat_variants:
- CentOS
- Red Hat Enterprise Linux
- Amazon
\ No newline at end of file
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
#
# Handlers for role edx-shared-vars
#
# Overview:
#
#
- name: notify me
debug: msg="stub handler"
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
##
# Role includes for role edx-shared-vars
#
# Example:
#
# dependencies:
# - {
# role: my_role
# my_role_var0: "foo"
# my_role_var1: "bar"
# }
---
#
# edX Configuration
#
# github: https://github.com/edx/configuration
# wiki: https://github.com/edx/configuration/wiki
# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT
#
#
#
# Tasks for role edx-shared-vars
#
# Overview:
#
#
# Dependencies:
#
#
# Example play:
#
#
- name: stub ansible task
debug: msg="This is a stub task created by the ansible-role role"
notify: notify me
......@@ -27,13 +27,18 @@
# - { role: edx_service, edx_service_name: "hotg" }
#
# Generating an ssh key so service users can do a git
# clone over ssh for public repositories without any
# additional configuration
- name: create application user
user: >
name="{{ edx_service_name }}"
home="{{ COMMON_APP_DIR }}/{{ edx_service_name }}"
createhome=no
createhome=yes
shell=/bin/false
generate_ssh_key=yes
# Assumes that the home directory has been created above.
- name: create edx_service app and venv dir
file: >
path="{{ item }}"
......@@ -41,7 +46,6 @@
owner="{{ edx_service_name }}"
group="{{ common_web_group }}"
with_items:
- "{{ COMMON_APP_DIR }}/{{ edx_service_name }}"
- "{{ COMMON_APP_DIR }}/{{ edx_service_name }}/venvs"
- name: create edx_service data and staticfiles dir
......@@ -64,6 +68,15 @@
with_items:
- "{{ COMMON_LOG_DIR }}/{{ edx_service_name }}"
- name: write out app config file
template: >
src=config.yml.j2
dest={{ COMMON_CFG_DIR }}/{{ edx_service_name }}.yml
mode=0644
tags:
- deploy
when: edx_service_config is defined
# Replace dashes with underscores to support roles that use
# dashes (the role vars will contain underscores)
- name: install a bunch of system packages on which edx_service relies
......
---
# {{ ansible_managed }}
{{ edx_service_config | to_nice_yaml }}
......@@ -110,6 +110,20 @@
priv='{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT'
when: ANALYTICS_API_CONFIG is defined
- name: setup the edx-notes-api db user
mysql_user: >
name={{ EDX_NOTES_API_MYSQL_USER }}
password={{ EDX_NOTES_API_MYSQL_PASS }}
priv='{{ EDX_NOTES_API_MYSQL_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE'
when: EDX_NOTES_API_MYSQL_USER is defined
- name: create a database for edx-notes-api
mysql_db: >
db=edx-notes-api
state=present
encoding=utf8
when: EDX_NOTES_API_MYSQL_USER is defined
- name: setup the migration db user
mysql_user: >
name={{ COMMON_MYSQL_MIGRATE_USER }}
......
......@@ -78,6 +78,11 @@ MARIADB_USERS:
priv: "*.*:CREATE USER"
host: "{{ MARIADB_HOST_PRIV }}"
- name: "{{ EDX_NOTES_API_MYSQL_DB_USER|default('notes001') }}"
pass: "{{ EDX_NOTES_API_MYSQL_DB_PASS|default('secret') }}"
priv: "*.*:ALL"
host: "{{ MARIADB_HOST_PRIV }}"
MARIADB_ANALYTICS_USERS:
- name: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['USER']|default('api001') }}"
pass: "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['PASSWORD']|default('password') }}"
......
......@@ -91,6 +91,8 @@ nginx_insights_gunicorn_hosts:
- 127.0.0.1
nginx_gitreload_gunicorn_hosts:
- 127.0.0.1
nginx_edx_notes_api_gunicorn_hosts:
- 127.0.0.1
nginx_cfg:
# - link - turn on
......
upstream {{ edx_notes_api_role_name_safe }}_app_server {
{% for host in nginx_edx_notes_api_gunicorn_hosts %}
server {{ host }}:{{ edx_notes_api_gunicorn_port }} fail_timeout=0;
{% endfor %}
}
server {
listen {{ edx_notes_api_nginx_port }} default_server;
location / {
try_files $uri @proxy_to_app;
}
{% include "robots.j2" %}
location @proxy_to_app {
proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
proxy_set_header X-Forwarded-For $http_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_pass http://{{ edx_notes_api_role_name_safe }}_app_server;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment