Added role to spawn new roles for Django-based IDAs

ECOM-2180

playbooks/create_django_ida.yml

+---
+# Creates a new ansible role
+# Usage:
+#   ansible-playbook ./create_django_ida.yml -i "localhost," -c local -e role_name=my_awesome_role
+#
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - ansible-role-django-ida

playbooks/roles/ansible-role-django-ida/tasks/main.yml

+---
+
+- name: check if the role exists
+  stat: path=roles/{{ role_name }}
+  register: role_exists
+
+- name: prompt for overwrite
+  pause: prompt="Role {{ role_name }} exists. Overwrite? Touch any key to continue or <CTRL>-c, then a, to abort."
+  when: role_exists.stat.exists
+
+- name: create role directories
+  file: path=roles/{{ role_name }}/{{ item }} state=directory
+  with_items:
+    - tasks
+    - meta
+    - defaults
+    - templates/edx/app/supervisor/conf.d.available
+    - templates/edx/app/{{ role_name }}
+
+- name: make an ansible role
+  template: src={{ item }}/main.yml.j2 dest=roles/{{ role_name }}/{{ item }}/main.yml
+  with_items:
+    - tasks
+    - meta
+    - defaults
+
+- name: update templates
+  template: src=templates/{{ item.src }} dest=roles/{{ role_name }}/templates/{{ item.dest }}
+  with_items:
+    - { src: 'edx/app/supervisor/conf.d.available/ROLE_NAME.conf.j2', dest: 'edx/app/supervisor/conf.d.available/{{ role_name }}.conf.j2'}
+    - { src: 'edx/app/ROLE_NAME/ROLE_NAME.sh.j2', dest: 'edx/app/{{ role_name }}/{{ role_name }}.sh.j2'}
+    - { src: 'edx/app/ROLE_NAME/ROLE_NAME_env.j2', dest: 'edx/app/{{ role_name }}/{{ role_name }}_env.j2'}
+    - { src: 'edx/app/ROLE_NAME/ROLE_NAME_gunicorn.py.j2', dest: 'edx/app/{{ role_name }}/{{ role_name }}_gunicorn.py.j2'}

playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2

+---
+{% include 'roles/ansible-role/templates/header.j2' %}
+#
+# Defaults for role {{ role_name }}
+# 
+{{ role_name|upper }}_GIT_IDENTITY: !!null
+
+# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
+# and a key being provided via NEWRELIC_LICENSE_KEY
+{{ role_name|upper }}_NEWRELIC_APPNAME: "{{ '{{ COMMON_ENVIRONMENT }}' }}-{{ '{{ COMMON_DEPLOYMENT }}' }}-{{ '{{' }} {{ role_name }}_service_name }}"
+{{ role_name|upper }}_PIP_EXTRA_ARGS: "-i {{ '{{ COMMON_PYPI_MIRROR_URL }}' }}"
+{{ role_name|upper }}_NGINX_PORT: "XXX" # TODO Change this!!!
+
+{{ role_name|upper }}_DEFAULT_DB_NAME: '{{ role_name }}'
+
+{{ role_name|upper }}_DATABASES:
+  # rw user
+  default:
+    ENGINE: 'django.db.backends.mysql'
+    NAME: '{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}'
+    USER: '{{ role_name }}001'
+    PASSWORD: 'password'
+    HOST: 'localhost'
+    PORT: '3306'
+    ATOMIC_REQUESTS: true
+    CONN_MAX_AGE: 60
+
+{{ role_name|upper }}_VERSION: "master"
+
+{{ role_name|upper }}_SECRET_KEY: 'Your secret key here'
+{{ role_name|upper }}_TIME_ZONE: 'UTC'
+{{ role_name|upper }}_LANGUAGE_CODE: 'en-us'
+
+# Used to automatically configure OAuth2 Client
+{{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_KEY : 'some-secret'
+{{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_SECRET : 'some-secret'
+{{ role_name|upper }}_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+
+{{ role_name|upper }}_PLATFORM_NAME: 'Your Platform Name Here'
+
+{{ role_name|upper }}_SERVICE_CONFIG:
+  SECRET_KEY: '{{ '{{' }} {{ role_name|upper }}_SECRET_KEY }}'
+  TIME_ZONE: '{{ '{{' }} {{ role_name|upper }}_TIME_ZONE }}'
+  LANGUAGE_CODE: '{{ '{{' }} {{ role_name|upper }}_LANGUAGE_CODE }}'
+
+  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ '{{' }} {{ role_name|upper }}_LMS_URL_ROOT }}/oauth2'
+  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+
+  STATIC_ROOT: "{{ '{{' }} COMMON_DATA_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}/staticfiles"
+  # db config
+  DATABASE_OPTIONS:
+    connect_timeout: 10
+  DATABASES: '{{ '{{' }} {{ role_name|upper }}_DATABASES }}'
+
+  PLATFORM_NAME: '{{ '{{' }} {{ role_name|upper }}_PLATFORM_NAME }}'
+
+
+{{ role_name|upper }}_REPOS:
+  - PROTOCOL: "{{ '{{' }} COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ '{{' }} COMMON_GIT_MIRROR }}"
+    PATH: "{{ '{{' }} COMMON_GIT_PATH }}"
+    REPO: {{ role_name }}.git
+    VERSION: "{{ '{{' }} {{ role_name|upper }}_VERSION }}"
+    DESTINATION: "{{ '{{' }} {{ role_name }}_code_dir }}"
+    SSH_KEY: "{{ '{{' }} {{ role_name|upper }}_GIT_IDENTITY }}"
+
+
+{{ role_name|upper }}_GUNICORN_WORKERS: "2"
+{{ role_name|upper }}_GUNICORN_EXTRA: ""
+{{ role_name|upper }}_GUNICORN_EXTRA_CONF: ""
+{{ role_name|upper }}_GUNICORN_WORKER_CLASS: "gevent"
+
+#
+# vars are namespace with the module name.
+#
+{{ role_name }}_role_name: {{ role_name }}
+{{ role_name }}_environment:
+  DJANGO_SETTINGS_MODULE: "{{ role_name }}.settings.production"
+  {{ role_name|upper }}_CFG: "{{ '{{' }} COMMON_CFG_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}.yml"
+
+{{ role_name }}_service_name: "{{ role_name }}"
+{{ role_name }}_user: "{{ '{{' }} {{ role_name }}_service_name }}"
+{{ role_name }}_home: "{{ '{{' }} COMMON_APP_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}"
+{{ role_name }}_code_dir: "{{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}"
+
+{{ role_name }}_gunicorn_host: "127.0.0.1"
+{{ role_name }}_gunicorn_port: "XXX" # TODO Change this!!!
+{{ role_name }}_gunicorn_timeout: "300"
+
+{{ role_name }}_log_dir: "{{ '{{' }} COMMON_LOG_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}"
+
+{{ role_name }}_requirements_base: "{{ '{{' }} {{ role_name }}_code_dir }}/requirements"
+{{ role_name }}_requirements:
+  - production.txt
+  - optional.txt
+
+
+#
+# OS packages
+#
+
+{{ role_name }}_debian_pkgs:
+  - libmysqlclient-dev
+  - libssl-dev
+
+{{ role_name }}_redhat_pkgs: []

playbooks/roles/ansible-role-django-ida/templates/header.j2

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#

playbooks/roles/ansible-role-django-ida/templates/meta/main.yml.j2

+---
+{% include 'roles/ansible-role/templates/header.j2' %}
+#
+# Role includes for role {{ role_name }}
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+dependencies:
+  - role: edx_service
+    edx_service_name: "{{ '{{' }} {{ role_name }}_service_name }}"
+    edx_service_config: "{{ '{{' }} {{ role_name|upper }}_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ '{{' }} {{ role_name|upper }}_REPOS }}"
+    edx_service_user: "{{ '{{' }} {{ role_name }}_user }}"
+    edx_service_home: "{{ '{{' }} {{ role_name }}_home }}"
+    edx_service_packages:
+      debian: "{{ '{{' }} {{ role_name }}_debian_pkgs }}"
+      redhat: "{{ '{{' }} {{ role_name }}_redhat_pkgs }}"
+  - supervisor

playbooks/roles/ansible-role-django-ida/templates/tasks/main.yml.j2

+---
+{% include 'roles/ansible-role/templates/header.j2' %}
+
+#
+# Tasks for role {{ role_name }}
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: add gunicorn configuration file
+  template: >
+    src=edx/app/{{ role_name }}/{{ role_name }}_gunicorn.py.j2
+    dest={{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_gunicorn.py
+  sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+
+- name: install application requirements
+  pip: >
+    requirements="{{ '{{' }} {{ role_name }}_requirements_base }}/{{ '{{' }} item }}"
+    virtualenv="{{ '{{' }} {{ role_name }}_home }}/venvs/{{ '{{' }} {{ role_name }}_service_name }}"
+    state=present
+  sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  with_items: {{ '{{' }} role_name }}_requirements
+
+- name: migrate
+  shell: >
+    chdir={{ '{{' }} {{ role_name }}_code_dir }}
+    DB_MIGRATION_USER={{ '{{' }} COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ '{{' }} COMMON_MYSQL_MIGRATE_PASS }}
+    {{ '{{' }} {{ role_name }}_home }}/venvs/{{ '{{' }} {{ role_name }}_service_name }}/bin/python ./manage.py migrate --noinput
+  sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  environment: "{{ '{{' }} {{ role_name }}_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: write out the supervisor wrapper
+  template: >
+    src=edx/app/{{ role_name }}/{{ role_name }}.sh.j2
+    dest={{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}.sh
+    mode=0650 owner={{ '{{' }} supervisor_user }} group={{ '{{' }} common_web_user }}
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/{{ role_name }}.conf.j2
+    dest="{{ '{{' }} supervisor_available_dir }}/{{ '{{' }} {{ role_name }}_service_name }}.conf"
+    owner={{ '{{' }} supervisor_user }} group={{ '{{' }} common_web_user }} mode=0644
+
+- name: setup the {{ role_name }} env file
+  template: >
+    src="./{{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}_env.j2"
+    dest="{{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_env"
+    owner={{ '{{' }} {{ role_name }}_user }}
+    group={{ '{{' }} {{ role_name }}_user }}
+    mode=0644
+
+- name: enable supervisor script
+  file: >
+    src={{ '{{' }} supervisor_available_dir }}/{{ '{{' }} {{ role_name }}_service_name }}.conf
+    dest={{ '{{' }} supervisor_cfg_dir }}/{{ '{{' }} {{ role_name }}_service_name }}.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ '{{' }} supervisor_ctl }} -c {{ '{{' }} supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: create symlinks from the-er venv bin dir
+  file: >
+    src="{{ '{{' }} {{ role_name }}_home }}/venvs/{{ '{{' }} {{ role_name }}_service_name }}/bin/{{ '{{' }} item }}"
+    dest="{{ '{{' }} COMMON_BIN_DIR }}/{{ '{{' }} item.split('.')[0] }}.{{ role_name }}"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create symlinks from the repo dir
+  file: >
+    src="{{ '{{' }} {{ role_name }}_code_dir }}/{{ item }}"
+    dest="{{ '{{' }} COMMON_BIN_DIR }}/{{ '{{' }} item.split('.')[0] }}.{{ role_name }}"
+    state=link
+  with_items:
+  - manage.py
+
+- name: restart the applicaton
+  supervisorctl: >
+    state=restarted
+    supervisorctl_path={{ '{{' }} supervisor_ctl }}
+    config={{ '{{' }} supervisor_cfg }}
+    name={{ '{{' }} {{ role_name }}_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ '{{' }} supervisor_service_user }}"

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/ROLE_NAME.sh.j2

+#!/usr/bin/env bash
+
+# {{ '{{' }} ansible_managed }}
+
+{{ '{%' }} set {{ role_name }}_venv_bin = {{ role_name }}_home + "/venvs/" + {{ role_name }}_service_name + "/bin" %}
+{{ '{%' }} if COMMON_ENABLE_NEWRELIC_APP %}
+{{ '{%' }} set executable = {{ role_name }}_venv_bin + '/newrelic-admin run-program ' + {{ role_name }}_venv_bin + '/gunicorn' %}
+{{ '{%' }} else %}
+{{ '{%' }} set executable = {{ role_name }}_venv_bin + '/gunicorn' %}
+{{ '{%' }} endif %}
+
+{{ '{%' }} if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME="{{ '{{' }} {{ role_name|upper }}_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ '{{' }} NEWRELIC_LICENSE_KEY }}"
+{{ '{%' }} endif -%}
+
+source {{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_env
+# TODO fix application
+{{ '{{' }} executable }} -c {{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_gunicorn.py {{ '{{' }} {{ role_name|upper }}_GUNICORN_EXTRA }} {{ role_name }}.wsgi:application

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/ROLE_NAME_env.j2

+# {{ '{{' }} ansible_managed }}
+
+{{ '{%' }} for name,value in {{ role_name }}_environment.items() -%}
+{{ '{%' }}- if value -%}
+export {{ '{{' }} name }}="{{ '{{' }} value }}"
+{{ '{%' }} endif %}
+{{ '{%' }}- endfor %}

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/ROLE_NAME_gunicorn.py.j2

+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+{{ '{{' }} ansible_managed }}
+"""
+
+timeout = {{ '{{' }} {{ role_name }}_gunicorn_timeout }}
+bind = "{{ '{{' }} {{ role_name }}_gunicorn_host }}:{{ '{{' }} {{ role_name }}_gunicorn_port }}"
+pythonpath = "{{ '{{' }} {{ role_name }}_code_dir }}"
+workers = {{ '{{' }} {{ role_name|upper }}_GUNICORN_WORKERS }}
+worker_class = "{{ '{{' }} {{ role_name|upper }}_GUNICORN_WORKER_CLASS }}"
+
+{{ '{{' }} {{ role_name|upper }}_GUNICORN_EXTRA_CONF }}

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/supervisor/conf.d.available/ROLE_NAME.conf.j2

+#
+# {{ '{{' }} ansible_managed }}
+# 
+[program:{{ '{{' }} {{ role_name }}_service_name }}]
+
+command={{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}.sh
+user={{ '{{' }} common_web_user }}
+directory={{ '{{' }} {{ role_name }}_code_dir }}
+stdout_logfile={{ '{{' }} supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ '{{' }} supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true