Add modular-devstack setup to the django-ida role template

.dockerignore

+*
+!playbooks/
+!docker/

playbooks/roles/ansible-role-django-ida/tasks/main.yml

@@ -8,6 +8,12 @@
   pause: prompt="Role {{ role_name }} exists. Overwrite? Touch any key to continue or <CTRL>-c, then a, to abort."
   when: role_exists.stat.exists
 
+- name: create docker directories
+  file: path=../docker/{{ item }}/{{ role_name }} state=directory
+  with_items:
+    - build
+    - plays
+
 - name: create role directories
   file: path=roles/{{ role_name }}/{{ item }} state=directory
   with_items:
@@ -16,6 +22,7 @@
     - defaults
     - templates/edx/app/supervisor/conf.d.available
     - templates/edx/app/{{ role_name }}
+    - templates/edx/app/nginx/sites-available
 
 - name: make an ansible role
   template: src={{ item }}/main.yml.j2 dest=roles/{{ role_name }}/{{ item }}/main.yml
@@ -24,10 +31,20 @@
     - meta
     - defaults
 
-- name: update templates
+- name: update docker templates
+  template: src=docker/{{ item.src }} dest=../docker/{{ item.dest }}
+  with_items:
+    - { src: 'build/ROLE_NAME/Dockerfile.j2', dest: 'build/{{ role_name }}/Dockerfile'}
+    - { src: 'build/ROLE_NAME/ansible_overrides.yml.j2', dest: 'build/{{ role_name }}/ansible_overrides.yml'}
+    - { src: 'build/ROLE_NAME/inventory', dest: 'build/{{ role_name }}/inventory'}
+    - { src: 'plays/ROLE_NAME.yml.j2', dest: 'plays/{{ role_name }}.yml'}
+
+- name: update role templates
   template: src=templates/{{ item.src }} dest=roles/{{ role_name }}/templates/{{ item.dest }}
   with_items:
     - { src: 'edx/app/supervisor/conf.d.available/ROLE_NAME.conf.j2', dest: 'edx/app/supervisor/conf.d.available/{{ role_name }}.conf.j2'}
     - { src: 'edx/app/ROLE_NAME/ROLE_NAME.sh.j2', dest: 'edx/app/{{ role_name }}/{{ role_name }}.sh.j2'}
     - { src: 'edx/app/ROLE_NAME/ROLE_NAME_env.j2', dest: 'edx/app/{{ role_name }}/{{ role_name }}_env.j2'}
     - { src: 'edx/app/ROLE_NAME/ROLE_NAME_gunicorn.py.j2', dest: 'edx/app/{{ role_name }}/{{ role_name }}_gunicorn.py.j2'}
+    - { src: 'edx/app/ROLE_NAME/devstack.sh.j2', dest: 'edx/app/{{ role_name }}/devstack.sh.j2'}
+    - { src: 'edx/app/nginx/sites-available/ROLE_NAME.j2', dest: 'edx/app/nginx/sites-available/{{ role_name }}.j2'}

playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2

@@ -9,27 +9,42 @@
 # and a key being provided via NEWRELIC_LICENSE_KEY
 {{ role_name|upper }}_NEWRELIC_APPNAME: "{{ '{{ COMMON_ENVIRONMENT }}' }}-{{ '{{ COMMON_DEPLOYMENT }}' }}-{{ '{{' }} {{ role_name }}_service_name }}"
 {{ role_name|upper }}_PIP_EXTRA_ARGS: "-i {{ '{{ COMMON_PYPI_MIRROR_URL }}' }}"
-{{ role_name|upper }}_NGINX_PORT: XXX       # TODO Change this!!!
-{{ role_name|upper }}_SSL_NGINX_PORT: XXX   # TODO Change this!!!
+{{ role_name|upper }}_NGINX_PORT: 18{{ port_suffix }}
+{{ role_name|upper }}_SSL_NGINX_PORT: 48{{ port_suffix }}
 
 {{ role_name|upper }}_DEFAULT_DB_NAME: '{{ role_name }}'
+{{ role_name|upper }}_MYSQL: 'db'
+# MySQL usernames are limited to 16 characters
+{{ role_name|upper }}_MYSQL_USER: '{{ role_name[:13] }}001'
+{{ role_name|upper }}_MYSQL_PASSWORD: 'password'
 
 {{ role_name|upper }}_DATABASES:
   # rw user
   default:
     ENGINE: 'django.db.backends.mysql'
     NAME: '{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}'
-    USER: '{{ role_name }}001'
-    PASSWORD: 'password'
-    HOST: 'localhost'
+    USER: '{{ '{{' }} {{ role_name|upper }}_MYSQL_USER }}'
+    PASSWORD: '{{ '{{' }} {{ role_name|upper }}_MYSQL_PASSWORD }}'
+    HOST: '{{ '{{' }} {{ role_name|upper}}_MYSQL }}'
     PORT: '3306'
     ATOMIC_REQUESTS: true
     CONN_MAX_AGE: 60
 
+{{ role_name|upper }}_DB_ADMIN_USER: 'root'
+{{ role_name|upper }}_DB_ADMIN_PASSWORD: ''
+
+{{ role_name|upper }}_MEMCACHE: [ 'memcache' ]
+
+{{ role_name|upper }}_CACHES:
+  default:
+    BACKEND:  'django.core.cache.backends.memcached.MemcachedCache'
+    KEY_PREFIX: 'default'
+    LOCATION: '{{ '{{' }} {{ role_name|upper}}_MEMCACHE }}'
+
 {{ role_name|upper }}_VERSION: "master"
 {{ role_name|upper }}_DJANGO_SETTINGS_MODULE: "{{ role_name }}.settings.production"
-{{ role_name|upper }}_URL_ROOT: 'http://localhost:####'   # TODO Set port number!
-{{ role_name|upper }}_LMS_URL_ROOT: 'http://127.0.0.1:8000'
+{{ role_name|upper }}_URL_ROOT: 'http://{{ role_name }}:18{{ port_suffix }}'
+{{ role_name|upper }}_OAUTH_URL_ROOT: 'http://127.0.0.1:8000'
 
 {{ role_name|upper }}_SECRET_KEY: 'Your secret key here'
 {{ role_name|upper }}_TIME_ZONE: 'UTC'
@@ -50,7 +65,7 @@
   SOCIAL_AUTH_EDX_OIDC_KEY: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_KEY }}'
   SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
   SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ '{{' }} {{ role_name|upper }}_LMS_URL_ROOT }}/oauth2'
+  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ '{{' }} {{ role_name|upper }}_OAUTH_URL_ROOT }}/oauth2'
   SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ '{{' }} {{ role_name|upper }}_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
 
   STATIC_ROOT: "{{ '{{' }} COMMON_DATA_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}/staticfiles"
@@ -66,7 +81,7 @@
   - PROTOCOL: "{{ '{{' }} COMMON_GIT_PROTOCOL }}"
     DOMAIN: "{{ '{{' }} COMMON_GIT_MIRROR }}"
     PATH: "{{ '{{' }} COMMON_GIT_PATH }}"
-    REPO: {{ role_name }}.git
+    REPO: {{ role_name|replace('_', '-') }}.git
     VERSION: "{{ '{{' }} {{ role_name|upper }}_VERSION }}"
     DESTINATION: "{{ '{{' }} {{ role_name }}_code_dir }}"
     SSH_KEY: "{{ '{{' }} {{ role_name|upper }}_GIT_IDENTITY }}"
@@ -77,6 +92,11 @@
 {{ role_name|upper }}_GUNICORN_EXTRA_CONF: ""
 {{ role_name|upper }}_GUNICORN_WORKER_CLASS: "gevent"
 
+{{ role_name|upper }}_HOSTNAME: '~^((stage|prod)-)?{{ role_name|replace('_', '-') }}.*'
+
+nginx_{{ role_name }}_gunicorn_hosts:
+  - 127.0.0.1
+
 #
 # vars are namespace with the module name.
 #
@@ -94,7 +114,7 @@
 {{ role_name }}_code_dir: "{{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}"
 
 {{ role_name }}_gunicorn_host: "127.0.0.1"
-{{ role_name }}_gunicorn_port: XXX    # TODO Change this!!!
+{{ role_name }}_gunicorn_port: 8{{ port_suffix }}
 {{ role_name }}_gunicorn_timeout: 300
 
 {{ role_name }}_log_dir: "{{ '{{' }} COMMON_LOG_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}"

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/Dockerfile.j2

+# To build this Dockerfile:
+#
+# From the root of configuration:
+#
+# docker build -f docker/build/{{ role_name }}/Dockerfile .
+#
+# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
+# with the currently checked-out configuration repo.
+
+FROM edxops/trusty-common
+MAINTAINER edxops
+
+ARG {{ role_name|upper }}_VERSION=master
+ARG REPO_OWNER=edx
+
+ADD . /edx/app/edx_ansible/edx_ansible
+
+USER docker
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
+COPY docker/build/{{ role_name }}/ansible_overrides.yml /
+RUN sudo ansible-playbook {{ role_name }}.yml -c local \
+    -t 'install:base,install:code,install:system-requirements,install:app-requirements,install:configuration,install:vhosts,install:devstack' \
+    --extra-vars="@/ansible_overrides.yml" \
+    --extra-vars="{{ role_name|upper }}_VERSION=${{ role_name|upper }}_VERSION" \
+    --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"
+
+USER root 
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/ansible_overrides.yml.j2

+---
+{{ role_name }}_gunicorn_host: 0.0.0.0
\ No newline at end of file

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/inventory

+[local]
+localhost

playbooks/roles/ansible-role-django-ida/templates/docker/plays/ROLE_NAME.yml.j2

+- name: Deploy {{ role_name|replace('_', ' ')|title }}
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ '{{' }} serial_count }}"
+  roles:
+    - docker
+    - nginx
+    - role: {{ role_name }}
+      nginx_default_sites:
+        - {{ role_name }}
\ No newline at end of file

playbooks/roles/ansible-role-django-ida/templates/tasks/main.yml.j2

@@ -24,16 +24,61 @@
     - install:configuration
 
 - name: install application requirements
-  pip:
-    requirements: "{{ '{{' }} {{ role_name }}_requirements_base }}/{{ '{{' }} item }}"
-    virtualenv: "{{ '{{' }} {{ role_name|lower }}_venv_dir }}"
-    state: present
+  shell: >
+    chdir=/edx/app/{{ role_name }}/{{ role_name }}
+    . {{ '{{' }} {{ role_name }}_venv_dir }}/bin/activate;
+    make requirements
   sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
-  with_items: "{{ '{{' }} {{ role_name }}_requirements }}"
   tags:
     - install
     - install:app-requirements
 
+- name: install development requirements
+  shell: >
+    chdir=/edx/app/{{ role_name }}/{{ role_name }}
+    . {{ '{{' }} {{ role_name }}_venv_dir }}/bin/activate;
+    make local-requirements
+  sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  tags:
+    - install
+    - install:devstack
+
+- name: wait for database
+  wait_for:
+    host: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.HOST }}"
+    port: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.PORT }}"
+    delay: 2
+  tags:
+    - migrate:devstack
+
+- name: create databases
+  mysql_db:
+    login_host: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.HOST }}"
+    login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.USER }}"
+    login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.PASSWORD }}"
+    db: "{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}"
+    state: present
+    encoding: utf8
+  tags:
+    - migrate:devstack
+
+- name: create database users
+  mysql_user:
+    login_host: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.HOST }}"
+    login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.USER }}"
+    login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.PASSWORD }}"
+    name: "{{ '{{' }} item.name }}"
+    host: "%"
+    name: "{{ '{{' }} item.password }}"
+    priv: "{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}.*:ALL"
+  with_items:
+    - name: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.USER }}"
+      password: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.PASSWORD }}"
+    - name: "{{ '{{' }} COMMON_MYSQL_MIGRATE_USER }}"
+      password: "{{ '{{' }} COMMON_MYSQL_MIGRATE_PASS }}"
+  tags:
+    - migrate:devstack
+
 - name: migrate
   shell: >
     chdir={{ '{{' }} {{ role_name }}_code_dir }}
@@ -46,6 +91,7 @@
   tags:
     - migrate
     - migrate:db
+    - migrate:devstack
 
 - name: write out the supervisor wrapper
   template:
@@ -69,6 +115,16 @@
     - install
     - install:configuration
 
+- name: write devstack script
+  template:
+    src: "edx/app/{{ role_name }}/devstack.sh.j2"
+    dest: "{{ '{{' }} {{ role_name }}_home }}/devstack.sh"
+    owner: "{{ '{{' }} supervisor_user }}"
+    group: "{{ '{{' }} common_web_user }}"
+    mode: 0744
+  tags:
+    - install:devstack
+
 - name: setup the {{ role_name }} env file
   template:
     src: "./{{ '{{' }} {{ role_name }}_home }}/{{ '{{' }} {{ role_name }}_service_name }}_env.j2"
@@ -95,8 +151,8 @@
   shell: "{{ '{{' }} supervisor_ctl }} -c {{ '{{' }} supervisor_cfg }} update"
   when: not disable_edx_services
   tags:
-    - install
-    - install:configuration
+    - manage
+    - manage:start
 
 - name: create symlinks from the venv bin dir
   file:
@@ -109,7 +165,7 @@
     - django-admin.py
   tags:
     - install
-    - install:base
+    - install:app-requirements
 
 - name: create symlinks from the repo dir
   file:
@@ -120,7 +176,7 @@
     - manage.py
   tags:
     - install
-    - install:base
+    - install:app-requirements
 
 - name: restart the application
   supervisorctl: 
@@ -133,3 +189,22 @@
   tags:
     - manage
     - manage:start
+
+- name: Copying nginx configs for {{ role_name }}
+  template: >
+    src=edx/app/nginx/sites-available/{{ role_name }}.j2
+    dest={{ '{{' }} nginx_sites_available_dir }}/{{ role_name }}
+    owner=root group={{ '{{' }} common_web_user }} mode=0640
+  notify: reload nginx
+  tags:
+    - install:vhosts
+
+- name: Creating nginx config links for {{ role_name }}
+  file: >
+    src={{ '{{' }} nginx_sites_available_dir }}/{{ role_name  }}
+    dest={{ '{{' }} nginx_sites_enabled_dir }}/{{ role_name }}
+    state=link owner=root group=root
+  notify: reload nginx
+  tags:
+    - install:vhosts
+

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/devstack.sh.j2

+#!/usr/bin/env bash
+
+# {{ '{{' }} ansible_managed }}
+
+cd /edx/app/edx_ansible/edx_ansible/docker/plays
+sudo ansible-playbook {{ role_name }}.yml -c local -t 'install:app-requirements'
+sudo ansible-playbook {{ role_name }}.yml -c local -t 'migrate:devstack' \
+                      --extra-vars="migrate_db=yes"
+sudo ansible-playbook {{ role_name }}.yml -c local -t 'manage:devstack'
+
+COMMAND=$1
+
+case $COMMAND in
+    start)
+        {{ '{%' }} set {{ role_name }}_venv_bin = {{ role_name }}_home + "/venvs/" + {{ role_name }}_service_name + "/bin" %}
+        {{ '{%' }} set executable = {{ role_name }}_venv_bin + '/gunicorn' %}
+
+        {{ '{{' }} executable }} -c {{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_gunicorn.py \
+                                 {{ '{{' }} {{ role_name|upper }}_GUNICORN_EXTRA }} \
+                                 --reload \
+                                 {{ role_name }}.wsgi:application
+
+        ;;
+    open)
+        cd {{ '{{' }} {{ role_name }}_home }}
+        . {{ '{{' }} {{ role_name }}_venv_bin }}/activate
+        /bin/bash
+        ;;
+esac

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/nginx/sites-available/ROLE_NAME.j2

+#
+# {{ '{{' }} ansible_managed }}
+#
+
+
+{{ '{%' }} if nginx_default_sites is defined and "{{ role_name }}" in nginx_default_sites {{ '%}' }}
+  {{ '{%' }} set default_site = "default" {{ '%}' }}
+{{ '{%' }} else {{ '%}' }}
+  {{ '{%' }} set default_site = "" {{ '%}' }}
+{{ '{%' }} endif {{ '%}' }}
+
+upstream {{ role_name }}_app_server {
+{{ '{%' }} for host in nginx_{{ role_name }}_gunicorn_hosts {{ '%}' }}
+    server {{ '{{' }} host }}:{{ '{{' }} {{ role_name }}_gunicorn_port }} fail_timeout=0;
+{{ '{%' }} endfor {{ '%}' }}
+}
+
+server {
+  server_name {{ '{{' }} {{ role_name|upper }}_HOSTNAME }};
+
+  {{ '{%' }} if NGINX_ENABLE_SSL {{ '%}' }}
+
+  listen {{ '{{' }} {{ role_name|upper }}_NGINX_PORT }} {{ '{{' }} default_site }};
+  listen {{ '{{' }} {{ role_name|upper }}_SSL_NGINX_PORT }} ssl;
+
+  ssl_certificate /etc/ssl/certs/{{ '{{' }} NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ '{{' }} NGINX_SSL_KEY|basename }};
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  {{ '{%' }} else {{ '%}' }}
+  listen {{ '{{' }} {{ role_name|upper }}_NGINX_PORT }} {{ '{{' }} default_site }};
+  {{ '{%' }} endif {{ '%}' }}
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ '{{' }} COMMON_DATA_DIR }}/{{ '{{' }} {{ role_name }}_service_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {{ '{%' }} if NGINX_ROBOT_RULES|length > 0 {{ '%}' }}
+  location /robots.txt {
+      root {{ '{{' }} nginx_app_dir }};
+      try_files $uri /robots.txt =404;
+  }
+  {{ '{%' }} endif {{ '%}' }}
+
+  location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://{{ role_name }}_app_server;
+  }
+
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+}