Merge pull request #349 from edx/jarv/common-refactor

Jarv/common refactor

playbooks/edx-east/edx_continuous_integration.yml

@@ -7,33 +7,24 @@
     migrate_db: "yes"
     openid_workaround: True
   roles:
-    - ansible_debug
     - common
     - role: nginx
       nginx_sites:
       - cms
       - lms
-      - lms-preview
       - ora
       - xqueue
       - xserver
-      #- discern
     - edxlocal
+    - supervisor
+    - mongo
     - edxapp
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - { role: 'edxapp', celery_worker: True }
     - oraclejdk
     - elasticsearch
-    - role: rbenv
-      rbenv_user: "{{ forum_user }}"
-      rbenv_user_home: "{{ forum_home }}"
-      rbenv_ruby_version: "{{ forum_ruby_version }}"
     - forum
-    - role: virtualenv
-      virtualenv_user: "{{ xqueue_user }}"
-      virtualenv_user_home: "{{ xqueue_user_home }}"
-      virtualenv_name: "{{ xqueue_user }}"
     - { role: "xqueue", update_users: True }
     - xserver
     - ora
-    #- discern
+    - discern

playbooks/edx-east/edx_dev2.yml

@@ -15,6 +15,7 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - datadog
     - role: nginx
       nginx_sites:
@@ -34,6 +35,7 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - datadog
     - role: nginx
       nginx_sites:

playbooks/edx-east/edx_feanilsandbox.yml

@@ -7,12 +7,13 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - datadog
     - role: nginx
       nginx_sites:
       - lms
       - cms
-      - lms-preview 
+      - lms-preview
     - role: 'edxapp'
       edxapp_lms_env: 'lms.envs.load_test'
       edx_platform_commit: 'release'
@@ -25,12 +26,13 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - datadog
     - role: nginx
       nginx_sites:
       - lms
       - cms
-      - lms-preview 
+      - lms-preview
     - role: 'edxapp'
       edxapp_lms_env: 'lms.envs.load_test'
       celery_worker: True
@@ -43,9 +45,10 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
-      - xserver 
+      - xserver
     - xserver
     - splunkforwarder
 - hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_rabbitmq
@@ -56,6 +59,7 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - rabbitmq
     - splunkforwarder
 - hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_xqueue
@@ -65,6 +69,7 @@
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - xqueue

playbooks/edx-east/edx_mirror.yml

@@ -5,6 +5,7 @@
   gather_facts: False
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - devpi
@@ -17,4 +18,4 @@
       tags: ['r_devpi']
     - role: gh_mirror
       tags: ['r_gh_mirror']
- 
+

playbooks/edx-east/edx_notifier.yml

@@ -7,6 +7,7 @@
   gather_facts: True
   roles:
     - common
+    - supervisor
     - role: virtualenv
       virtualenv_user: "notifier"
       virtualenv_user_home: "/opt/wwc/notifier"
@@ -22,6 +23,7 @@
   gather_facts: True
   roles:
     - common
+    - supervisor
     - role: virtualenv
       virtualenv_user: "notifier"
       virtualenv_user_home: "/opt/wwc/notifier"
@@ -37,6 +39,7 @@
   gather_facts: True
   roles:
     - common
+    - supervisor
     - role: virtualenv
       virtualenv_user: "notifier"
       virtualenv_user_home: "/opt/wwc/notifier"
@@ -52,6 +55,7 @@
   gather_facts: True
   roles:
     - common
+    - supervisor
     - role: virtualenv
       virtualenv_user: "notifier"
       virtualenv_user_home: "/opt/wwc/notifier"
@@ -68,6 +72,7 @@
   vars:
   roles:
     - common
+    - supervisor
     - role: virtualenv
       virtualenv_user: "notifier"
       virtualenv_user_home: "/opt/wwc/notifier"

playbooks/edx-east/edx_sandbox.yml

@@ -13,6 +13,7 @@
     mysql5_workaround: True
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - lms
@@ -27,6 +28,6 @@
     - { role: 'edxapp', celery_worker: True }
     - role: rbenv
       rbenv_user: "{{ forum_user }}"
-      rbenv_user_home: "{{ forum_home }}"
+      rbenv_dir: "{{ forum_home }}"
       rbenv_ruby_version: "{{ forum_ruby_version }}"
     - forum

playbooks/edx-east/mlapi_prod.yml

@@ -6,6 +6,7 @@
     - "{{ secure_dir }}/vars/mlapi_prod_users.yml"
   roles:
     - common
+    - supervisor
     - discern
   sudo: True
 - hosts:
@@ -17,4 +18,4 @@
     - "{{ secure_dir }}/vars/mlapi_prod_users.yml"
   roles:
     - common
-  sudo: True
+  sudo: True
\ No newline at end of file

playbooks/edx-east/mlapi_sandbox.yml

@@ -6,6 +6,7 @@
     - "{{ secure_dir }}/vars/mlapi_sandbox_users.yml"
   roles:
     - common
+    - supervisor
     - discern
   sudo: True
 - hosts:

playbooks/edx-east/mlapi_stage.yml

@@ -6,6 +6,7 @@
     - "{{ secure_dir }}/vars/mlapi_stage_users.yml"
   roles:
     - common
+    - supervisor
     - discern
   sudo: True
 - hosts:

playbooks/edx-west/carnegie-prod-app.yml

@@ -19,6 +19,7 @@
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
+    - supervisor
     - {'role': 'nginx', 'nginx_conf': true}
     - {'role': 'edxapp', 'openid_workaround': true, 'template_subdir': 'carnegie'}
     # run this role last

playbooks/edx-west/carnegie-prod-worker.yml

@@ -15,4 +15,5 @@
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
+    - supervisor
     - { role: 'edxapp', celery_worker: True }

playbooks/edx-west/cme-prod-app.yml

@@ -32,6 +32,7 @@
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_conf: true
       nginx_sites:

playbooks/edx-west/cme-prod-worker.yml

@@ -15,4 +15,5 @@
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
+    - supervisor
     - { role: 'edxapp', celery_worker: True }

playbooks/edx-west/edxapp_rolling_example.yml

-# ansible-playbook -v --user=ubuntu  edxapp_rolling_example.yml -i ./ec2.py  --private-key=/path/to/deployment.pem 
+# ansible-playbook -v --user=ubuntu  edxapp_rolling_example.yml -i ./ec2.py  --private-key=/path/to/deployment.pem
 
 - hosts: tag_Group_anothermulti
   serial: 2
@@ -6,8 +6,8 @@
     - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
   pre_tasks:
-    - name: Gathering ec2 facts 
-      ec2_facts: 
+    - name: Gathering ec2 facts
+      ec2_facts:
     - name: Removing instance from the ELB
       local_action: ec2_elb
       args:
@@ -15,7 +15,8 @@
         state: 'absent'
   roles:
     - common
-    - role: nginx 
+    - supervisor
+    - role: nginx
       nginx_sites:
       - lms
       - cms

playbooks/edx-west/prod-app.yml

@@ -26,6 +26,7 @@
     - "{{ secure_dir }}/vars/shib_prod_vars.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - lms

playbooks/edx-west/prod-jumpbox.yml

@@ -7,3 +7,4 @@
     local_dir:  '../../../configuration-secure/ansible/local'
   roles:
     - common
+    - supervisor

playbooks/edx-west/prod-ora.yml

@@ -18,6 +18,7 @@
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - ora

playbooks/edx-west/prod-worker.yml

@@ -16,6 +16,7 @@
     - "{{ secure_dir }}/vars/shib_prod_vars.yml"
   roles:
     - common
+    - supervisor
     - { role: 'edxapp', celery_worker: True }
 
 #

playbooks/edx-west/prod-xqueue.yml

@@ -15,6 +15,7 @@
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - xqueue

playbooks/edx-west/stage-all.yml

@@ -12,4 +12,5 @@
     - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
+    - supervisor
 

playbooks/edx-west/stage-app.yml

@@ -17,6 +17,7 @@
     - "{{ secure_dir }}/vars/shib_stage_vars.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - lms

playbooks/edx-west/stage-debug.yml

@@ -12,6 +12,7 @@
     #- "{{ secure_dir }}/vars/shib_stage_vars.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - lms

playbooks/edx-west/stage-notifier-only.yml

@@ -8,6 +8,7 @@
     - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
     - "{{ secure_dir }}/vars/notifier_stage_vars.yml"
   roles:
+    - supervisor
     - role: virtualenv
       virtualenv_user: "notifier"
       virtualenv_user_home: "/opt/wwc/notifier"

playbooks/edx-west/stage-ora.yml

@@ -10,6 +10,7 @@
     - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - ora

playbooks/edx-west/stage-rabbit.yml

@@ -10,6 +10,7 @@
     - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
+    - supervisor
     - rabbitmq
 
 #- hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_edxapp

playbooks/edx-west/stage-worker.yml

@@ -15,6 +15,7 @@
     - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
+    - supervisor
     - { role: 'edxapp', celery_worker: True }
 
 # run the notifier on the first util machine only

playbooks/edx-west/stage-xqueue.yml

@@ -10,6 +10,7 @@
     - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
+    - supervisor
     - role: nginx
       nginx_sites:
       - xqueue

playbooks/group_vars/all

 ---
-# This should only have variables
-# that are applicable to all edX roles
-
-storage_base_dir: /mnt
-app_base_dir: /opt/wwc
-log_base_dir: "{{ storage_base_dir }}/logs"
-venv_dir: /opt/edx
-os_name: ubuntu
-
-ENV_NAME: 'default_env'
-ENV_TYPE: 'default_type'
-
 # these pathes are relative to the playbook dir
 # directory for secret settings (keys, etc)
-secure_dir: 'secure_example'
+#
+
+secure_dir: 'path/to/secure_example'
 
 # this indicates the path to site-specific (with precedence)
 # things like nginx template files
-local_dir:  '../../ansible_local'
-# include http/https
-PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
-# do not include http/https
-GIT_MIRROR: 'github.com'
+local_dir:  'path/to/ansible_local'

playbooks/library/supervisorctl

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2012, Matt Wright <matt@nobien.net>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+DOCUMENTATION = '''
+---
+module: supervisorctl
+short_description: Manage the state of a program or group of programs running via Supervisord
+description:
+     - Manage the state of a program or group of programs running via I(Supervisord)
+version_added: "0.7"
+options:
+  name:
+    description:
+      - The name of the I(supervisord) program/process to manage
+    required: true
+    default: null
+  config:
+    description:
+      - configuration file path, passed as -c to supervisorctl
+    required: false
+    default: null
+    version_added: "1.3"
+  server_url:
+    description:
+      - URL on which supervisord server is listening, passed as -s to supervisorctl
+    required: false
+    default: null
+    version_added: "1.3"
+  username:
+    description:
+      - username to use for authentication with server, passed as -u to supervisorctl
+    required: false
+    default: null
+    version_added: "1.3"
+  password:
+    description:
+      - password to use for authentication with server, passed as -p to supervisorctl
+    required: false
+    default: null
+    version_added: "1.3"
+  state:
+    description:
+      - The state of service
+    required: true
+    default: null
+    choices: [ "present", "started", "stopped", "restarted" ]
+  supervisorctl_path:
+    description:
+      - Path to supervisorctl executable to use
+    required: false
+    default: null
+    version_added: "1.4"
+requirements:
+  - supervisorctl
+requirements: [ ]
+author: Matt Wright
+'''
+
+EXAMPLES = '''
+# Manage the state of program to be in 'started' state.
+- supervisorctl: name=my_app state=started
+
+# Restart my_app, reading supervisorctl configuration from a specified file.
+- supervisorctl: name=my_app state=restarted config=/var/opt/my_project/supervisord.conf
+
+# Restart my_app, connecting to supervisord with credentials and server URL.
+- supervisorctl: name=my_app state=restarted username=test password=testpass server_url=http://localhost:9001
+
+'''
+
+def main():
+    arg_spec = dict(
+        name=dict(required=True),
+        config=dict(required=False),
+        server_url=dict(required=False),
+        username=dict(required=False),
+        password=dict(required=False),
+        supervisorctl_path=dict(required=False),
+        state=dict(required=True, choices=['present', 'started', 'restarted', 'stopped'])
+    )
+
+    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=True)
+
+    name = module.params['name']
+    state = module.params['state']
+    config = module.params.get('config')
+    server_url = module.params.get('server_url')
+    username = module.params.get('username')
+    password = module.params.get('password')
+    supervisorctl_path = module.params.get('supervisorctl_path')
+
+    if supervisorctl_path:
+        supervisorctl_path = os.path.expanduser(supervisorctl_path)
+        if os.path.exists(supervisorctl_path) and module.is_executable(supervisorctl_path):
+            supervisorctl_args = [ supervisorctl_path ]
+        else:
+            module.fail_json(msg="Provided path to supervisorctl does not exist or isn't executable: %s" % supervisorctl_path)
+    else:
+        supervisorctl_args = [ module.get_bin_path('supervisorctl', True) ]
+
+    if config:
+        supervisorctl_args.extend(['-c', os.path.expanduser(config)])
+    if server_url:
+        supervisorctl_args.extend(['-s', server_url])
+    if username:
+        supervisorctl_args.extend(['-u', username])
+    if password:
+        supervisorctl_args.extend(['-p', password])
+
+    def run_supervisorctl(cmd, name=None, **kwargs):
+        args = list(supervisorctl_args)  # copy the master args
+        args.append(cmd)
+        if name:
+            args.append(name)
+        return module.run_command(args, **kwargs)
+
+    rc, out, err = run_supervisorctl('status')
+    present = name in out
+
+    if state == 'present':
+        if not present:
+            if module.check_mode:
+                module.exit_json(changed=True)
+            run_supervisorctl('reread', check_rc=True)
+            rc, out, err = run_supervisorctl('add', name)
+
+            if '%s: added process group' % name in out:
+                module.exit_json(changed=True, name=name, state=state)
+            else:
+                module.fail_json(msg=out, name=name, state=state)
+
+        module.exit_json(changed=False, name=name, state=state)
+
+    rc, out, err = run_supervisorctl('status', name)
+    running = 'RUNNING' in out
+
+    if running and state == 'started':
+        module.exit_json(changed=False, name=name, state=state)
+
+    if running and state == 'stopped':
+        if module.check_mode:
+            module.exit_json(changed=True)
+        rc, out, err = run_supervisorctl('stop', name)
+
+        if '%s: stopped' % name in out:
+            module.exit_json(changed=True, name=name, state=state)
+
+        module.fail_json(msg=out)
+
+    elif state == 'restarted':
+        if module.check_mode:
+            module.exit_json(changed=True)
+        rc, out, err = run_supervisorctl('update', name)
+        rc, out, err = run_supervisorctl('restart', name)
+
+        if '%s: started' % name in out:
+            module.exit_json(changed=True, name=name, state=state)
+
+        module.fail_json(msg=out)
+
+    elif not running and state == 'started':
+        if module.check_mode:
+            module.exit_json(changed=True)
+        rc, out, err = run_supervisorctl('start',name)
+
+        if '%s: started' % name in out:
+            module.exit_json(changed=True, name=name, state=state)
+
+        module.fail_json(msg=out)
+
+    module.exit_json(changed=False, name=name, state=state)
+
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+
+main()

playbooks/roles/apache/templates/lms.j2

-WSGIPythonHome {{venv_dir}}
+WSGIPythonHome {{ edxapp_venv_dir }}
 WSGIRestrictEmbedded On
 
 <VirtualHost *:{{apache_port}}>
@@ -15,9 +15,9 @@ WSGIRestrictEmbedded On
 
     SetEnv SERVICE_VARIANT lms
     
-    WSGIScriptAlias / {{edx_platform_code_dir}}/lms/wsgi_apache_lms.py
+    WSGIScriptAlias / {{ edxapp_code_dir }}/lms/wsgi_apache_lms.py
 
-    <Directory {{edx_platform_code_dir}}/lms>
+    <Directory {{ edxapp_code_dir }}/lms>
     <Files wsgi_apache_lms.py>
         Order deny,allow
         Allow from all
@@ -39,7 +39,7 @@ WSGIRestrictEmbedded On
         require valid-user
     </Location>
 
-    WSGIDaemonProcess lms user=www-data group=adm processes=1 python-path={{edx_platform_code_dir}}:{{venv_dir}}/lib/python2.7/site-packages display-name=%{GROUP}
+    WSGIDaemonProcess lms user=www-data group=adm processes=1 python-path={{ edxapp_code_dir }}:{{ edxapp_venv_dir }}/lib/python2.7/site-packages display-name=%{GROUP}
     WSGIProcessGroup lms
     WSGIApplicationGroup %{GLOBAL}
     
@@ -48,4 +48,4 @@ WSGIRestrictEmbedded On
     LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\" %D" apache-edx
     CustomLog ${APACHE_LOG_DIR}/apache-edx-access.log apache-edx
 
-</VirtualHost>
+</VirtualHost>
\ No newline at end of file

playbooks/roles/common/defaults/main.yml

+
+# Override these variables
+# to change the base directory
+# where edX is installed
+
+COMMON_BASE_DIR: /edx
+COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var"
+COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app"
+COMMON_LOG_DIR: "{{ COMMON_DATA_DIR }}/log"
+
+# these directories contain
+# symlinks for convenience
+COMMON_BIN_DIR: "{{ COMMON_BASE_DIR }}/bin"
+COMMON_CFG_DIR: "{{ COMMON_BASE_DIR }}/etc"
+
+COMMON_ENV_NAME: 'default_env'
+COMMON_ENV_TYPE: 'default_type'
+COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
+# do not include http/https
+COMMON_GIT_MIRROR: 'github.com'
+
+
+
 common_debian_pkgs:
   - ack-grep
   - lynx-cur
@@ -8,3 +31,14 @@ common_debian_pkgs:
   - tree
   - git
   - unzip
+  - python2.7
+  - python-pip
+  - python2.7-dev
+
+common_pip_pkgs:
+  - virtualenv
+  - virtualenvwrapper
+
+common_web_user: www-data
+common_web_group: www-data
+common_log_user: syslog

playbooks/roles/common/tasks/create_github_users.yml

----
-
-# Overview:
-# 
-# Creates OS accounts for users based on their github credential.
-# Expects to find a list in scope named GITHUB_USERS with
-# the following structure:
-#
-# GITHUB_USERS:
-#   - user: me_at_github
-#     groups:
-#     - adm
-#   - user: otheruser
-#     groups: 
-#     - users
-#
-
-- name: common | create local user for github user
-  user: 
-    name={{ item.user }} 
-    groups={{ ",".join(item.groups) }}
-    shell=/bin/bash
-  with_items: GITHUB_USERS
-  tags:
-    - users
-    - update
-
-- name: common | create .ssh directory
-  file: 
-    path=/home/{{ item.user }}/.ssh state=directory mode=0700 
-    owner={{ item.user }} group={{ item.user }}
-  with_items: GITHUB_USERS
-  tags:
-    - users
-    - update
-
-- name: common | copy github key[s] to .ssh/authorized_keys
-  get_url: 
-    url=https://github.com/{{ item.user }}.keys 
-    dest=/home/{{ item.user }}/.ssh/authorized_keys mode=0600 
-    owner={{ item.user }} group={{ item.user }}
-  with_items: GITHUB_USERS
-  tags:
-    - users
-    - update

playbooks/roles/common/tasks/create_users.yml

----
-- name: common | Create 'edx' users group
-  group: name=edx state=present
-  tags:
-  - users
-  - admin_users
-  - update
-
-- name: common | Add user 'ubuntu' to 'edx' group
-  # This is a temporary measure for initial configuration; after the last 
-  # play is run and we've got a good set of users, ubuntu should no longer be used
-  user: name=ubuntu append=yes groups="edx"
-  tags:
-  - users
-  - admin_users
-  - update
-
-- name: common | Creating admin users
-  # Admin users, by definition, should be able to sudo w/ password, and read adm-only files
-  user: name={{ item.user }} append=yes groups={{ "adm,edx,"+",".join(item.groups) }} shell=/bin/bash
-  with_items: admin_users
-  when: admin_users is defined
-  tags:
-  - users
-  - admin_users
-  - update
-
-- name: common | Copying ssh keys for admin users
-  authorized_key: user={{item.user}} key="{{ lookup('file', item.path ) }}"
-  with_items: admin_keys
-  when: admin_keys is defined
-  tags:
-  - users
-  - admin_users
-  - update
-
-- name: common | Creating env users
-  user: name={{ item.user }} {% if item.groups %}groups={{ ",".join(item.groups) }}{% endif %} shell=/bin/bash
-  with_items: ENV_USERS
-  when: ENV_USERS is defined
-  tags:
-  - users
-  - update
-
-- name: common | Copying ssh keys for env users
-  authorized_key: user={{ item.user }} key="{{ lookup('file', item.path ) }}"
-  with_items: env_keys
-  when: env_keys is defined
-  tags:
-  - users
-  - update
-
-- name: common | Group adm passwordless sudo
-  copy: content="%adm ALL=(ALL) NOPASSWD:ALL" dest=/etc/sudoers.d/adm-group owner=root group=root mode=0440
-  tags:
-  - users
-  - admin_users
-  - update

playbooks/roles/common/tasks/create_venv.yml

----
-# create the 'edx' virtual environment in /opt so that roles can populate it
-- name: common | Install python and pip
-  apt: pkg={{item}} install_recommends=yes state=present update_cache=yes
-  with_items:
-  - python2.7
-  - python-pip
-  - python2.7-dev
-  tags:
-  - pre_install
-  - install
-
-- name: common | pip install virtualenv
-  pip: >
-    name=virtualenv 
-    state=present
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags:
-  - venv_base
-  - install
-
-- name: common | pip install virtualenvwrapper
-  pip: >
-    name=virtualenvwrapper 
-    state=present
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags:
-  - venv_base
-  - install
-
-- name: common | create edx virtualenv directory
-  file: path={{ venv_dir }} owner=ubuntu group=adm mode=2775 state=directory
-  tags:
-  - venv_base
-  - install
-
-- name: common | create the edx virtualenv directory initial contents
-  command: /usr/local/bin/virtualenv {{ venv_dir }} --distribute creates=$venv_dir/bin/activate
-  tags:
-  - venv_base
-  - install
-
-- name: common |  pip install gunicorn
-  pip: >
-    name=gunicorn 
-    virtualenv="{{venv_dir}}" 
-    state=present
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags:
-  - gunicorn
-  - install

playbooks/roles/common/tasks/edx_logging_base.yml

----
-#- name: common | Install rsyslog configuration for ansible runs
-#  template: dest=/etc/rsyslog.d/90-edx.conf src=ansible_rsyslog.j2 owner=root group=root mode=644
-#  notify: common | restart rsyslogd
-#  tags:
-#  - lms-env
-#  - cms-env
-#  - logging
-#  - update
-
-- name: common | Install rsyslog configuration for edX
-  template: dest=/etc/rsyslog.d/99-edx.conf src=edx_rsyslog.j2 owner=root group=root mode=644
-  notify: common | restart rsyslogd
-  tags:
-  - logging
-  - update
-
-- name: common | Install logrotate configuration for edX
-  template: dest=/etc/logrotate.d/edx-services src=edx_logrotate.j2 owner=root group=root mode=644
-  tags:
-  - logging
-  - update
-
-- name: common | Touch tracking file into existence
-  command: touch -a {{log_base_dir}}/tracking.log creates={{log_base_dir}}/tracking.log
-  tags:
-  - logging
-  - update
-
-- name: common | Set permissions on tracking file
-  file: path={{log_base_dir}}/tracking.log owner=syslog group=adm mode=640
-  tags:
-  - logging
-  - update
-
-- name: common | Install logrotate configuration for tracking file
-  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
-  tags:
-  - logging
-  - update

playbooks/roles/common/tasks/main.yml

 ---
-- include: create_users.yml
-- include: create_github_users.yml
-  when: GITHUB_USERS is defined
-
 - name: common | Add user www-data
-  # This user should be created on the system by default
-  user: name=www-data
-  tags:
-  - pre_install
-  - update
+  # This is the default user for nginx
+  user: >
+    name="{{ common_web_user }}"
+    shell=/bin/false
 
-- name: common | Create the base directory for storage
+- name: common | Create common directories
   file: >
-    path={{ storage_base_dir }}
-      state=directory
-      owner=root
-      group=root
-      mode=0755
-
-- name: common | Create application root
-  # In the future consider making group edx r/t adm
-  file: path={{ app_base_dir }} state=directory owner=root group=adm mode=2775 
-  tags:
-  - pre_install
-  - update
-
-- name: common | Create upload directory
-  file: path={{ app_base_dir }}/uploads mode=2775 state=directory owner=root group=adm
-  tags:
-  - pre_install
-  - update
+    path={{ item }} state=directory owner=root
+    group=root mode=0755
+  with_items:
+    - "{{ COMMON_DATA_DIR }}"
+    - "{{ COMMON_APP_DIR }}"
+    - "{{ COMMON_LOG_DIR }}"
+    - "{{ COMMON_BIN_DIR }}"
+    - "{{ COMMON_CFG_DIR }}"
 
-- name: common | Create data dir
-  file: path={{ app_base_dir }}/data state=directory owner=www-data group=root
-  tags:
-  - pre_install
-  - update
-  
-- name: common | Create staticfiles dir
-  file: path={{ app_base_dir }}/staticfiles state=directory owner=www-data group=adm mode=2775 
-  tags:
-  - pre_install
-  - update
-  
 - name: common | Install role-independent useful system packages
   # do this before log dir setup; rsyslog package guarantees syslog user present
-  apt: pkg={{','.join(common_debian_pkgs)}} install_recommends=yes state=present update_cache=yes
-  tags:
-  - pre_install
-  - update
+  apt: >
+    pkg={{','.join(common_debian_pkgs)}} install_recommends=yes
+    state=present update_cache=yes
 
-- name: common | Create log directory
-  file: path={{log_base_dir}} state=directory mode=2755 group=adm owner=syslog
-  tags:
-  - pre_install
-  - update
-
-- name: common | Create alias from app_base_dir to the log_base_dir
-  file: state=link src={{log_base_dir}} path={{app_base_dir}}/log
-  tags:
-  - pre_install
-  - logging
-  - update
+- name: common | upload sudo config for key forwarding as root
+  copy: >
+    src=ssh_key_forward dest=/etc/sudoers.d/ssh_key_forward
+    validate='visudo -c -f %s' owner=root group=root mode=0440
 
-- name: common | Create convenience link from log_base_dir to system logs
-  file: state=link src=/var/log path=$log_base_dir/system
-  tags:
-  - pre_install
-  - logging
-  - update
+- name: common | pip install virtualenv
+  pip: >
+    name="{{ item }}" state=present
+    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+  with_items: common_pip_pkgs
 
-- name: common | Touch edx log file into place 
-  # This is done for the benefit of the rake commands, which expect it
-  command: touch -a {{log_base_dir}}/edx.log creates={{log_base_dir}}/edx.log
-  tags:
-  - pre_install
-  - logging
-  - install
+- name: common | Install rsyslog configuration for edX
+  template: dest=/etc/rsyslog.d/99-edx.conf src=edx_rsyslog.j2 owner=root group=root mode=644
+  notify: common | restart rsyslogd
 
-- name: common | Set permissions on edx log file
-  # This is done for the benefit of the rake commands, which expect it
-  file: path={{log_base_dir}}/edx.log owner=syslog group=adm mode=640
-  tags:
-  - pre_install
-  - logging
-  - update
 
-- name: common | upload sudo config for key forwarding as root
-  copy: src=ssh_key_forward dest=/etc/sudoers.d/ssh_key_forward validate='visudo -c -f %s' owner=root group=root mode=0440
+- name: common | Install logrotate configuration for edX
+  template: dest=/etc/logrotate.d/edx-services src=edx_logrotate.j2 owner=root group=root mode=644
 
-- include: create_venv.yml
-- include: edx_logging_base.yml

playbooks/roles/common/tasks/software_update.yml

----
-- name: common | edx-update.sh, manual lms/cms update script
-  template: src=edx-update.sh.j2 dest=/usr/local/bin/edx-update.sh owner=ubuntu group=adm mode=0775
-  tags:
-  - release
-  - update

playbooks/roles/common/templates/edx_logrotate.j2

-{{log_base_dir}}/*/edx.log {
+{{ COMMON_LOG_DIR }}/*/edx.log {
   create
   compress
   copytruncate

playbooks/roles/common/templates/edx_rsyslog.j2

@@ -27,12 +27,12 @@ auth,authpriv.*                                         /var/log/auth.log
 $template tracking,"%syslogtag%%msg%\n"
 
 # looks for [service_name=<name>] in the beginning of the log message,
-# if it exists the log will go into {{log_base_dir}}/<name>/edx.log, otherwise
-# it will go into {{log_base_dir}}/edx.log
-$template DynaFile,"{{log_base_dir}}/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
+# if it exists the log will go into {{ COMMON_LOG_DIR }}/<name>/edx.log, otherwise
+# it will go into {{ COMMON_LOG_DIR }}/edx.log
+$template DynaFile,"{{ COMMON_LOG_DIR }}/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
 
 local0.*                        -?DynaFile
-local1.*                        {{log_base_dir}}/tracking.log;tracking
+local1.*                        {{ COMMON_LOG_DIR }}/tracking.log;tracking
 #cron.*                         /var/log/cron.log
 #daemon.*                       -/var/log/daemon.log
 kern.*                          -/var/log/kern.log

playbooks/roles/devpi/defaults/main.yml

 ---
-devpi_venv_dir: "{{ app_base_dir }}/devpi/venvs/devpi"
+devpi_venv_dir: "{{ COMMON_APP_DIR }}/devpi/venvs/devpi"
 devpi_pip_pkgs:
   - devpi-server
   - eventlet

playbooks/roles/discern/defaults/main.yml

 DISCERN_NGINX_PORT: 18070
 DISCERN_BASIC_AUTH: False
+DISCERN_MEMCACHE: [ 'localhost:11211' ]
+DISCERN_AWS_ACCESS_KEY_ID: ""
+DISCERN_AWS_SECRET_ACCESS_KEY: ""
+DISCERN_BROKER_URL: ""
+DISCERN_RESULT_BACKEND: ""
+DISCERN_GOOGLE_ANALYTICS_PROPERTY_ID: ""
+DISCERN_MYSQL_DB_NAME: 'discern'
+DISCERN_MYSQL_USER: 'root'
+DISCERN_MYSQL_PASSWORD: ''
+DISCERN_MYSQL_HOST: 'localhost'
+DISCERN_MYSQL_PORT: '3306'
+
+
+discern_app_dir: "{{ COMMON_APP_DIR }}/discern"
+discern_code_dir: "{{ discern_app_dir }}/discern"
+discern_data_dir: "{{ COMMON_DATA_DIR }}/discern"
+discern_venvs_dir: "{{ discern_app_dir }}/venvs"
+discern_venv_dir: "{{ discern_venvs_dir }}/discern"
+discern_venv_bin: "{{ discern_venv_dir }}/bin"
+discern_pre_requirements_file: "{{ discern_code_dir }}/pre-requirements.txt"
+discern_post_requirements_file: "{{ discern_code_dir }}/requirements.txt"
+discern_user: "discern"
+
+discern_ease_venv_dir: "{{ discern_venv_dir }}"
+discern_ease_code_dir: "{{ discern_app_dir }}/ease"
+discern_ease_source_repo: https://github.com/edx/ease.git
+discern_ease_version: 'HEAD'
+discern_ease_pre_requirements_file: "{{ discern_ease_code_dir }}/pre-requirements.txt"
+discern_ease_post_requirements_file: "{{ discern_ease_code_dir }}/requirements.txt"
+
+discern_nltk_data_dir: "{{ discern_data_dir}}/nltk_data"
 
 discern_source_repo: https://github.com/edx/discern.git
-ease_source_repo: https://github.com/edx/ease.git
-ease_dir: $app_base_dir/ease
-discern_dir: $app_base_dir/discern
 discern_settings: discern.aws
-nltk_data_dir: /usr/share/nltk_data
-ease_branch: master
 discern_branch: dev
 discern_gunicorn_port: 8070
 discern_gunicorn_host: 127.0.0.1
-discern_user: discern
-site_name: discern
+
+discern_env_config:
+  ACCOUNT_EMAIL_VERIFICATION: "mandatory"
+  AWS_SES_REGION_NAME: "us-east-1"
+  DEFAULT_FROM_EMAIL: "registration@example.com"
+  DNS_HOSTNAME: ""
+  ELB_HOSTNAME: ""
+  EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
+  S3_BUCKETNAME: ""
+  USE_S3_TO_STORE_MODElS: false
+
+
+discern_auth_config:
+  AWS_ACCESS_KEY_ID: $DISCERN_AWS_ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY: $DISCERN_SECRET_ACCESS_KEY
+  BROKER_URL: $DISCERN_BROKER_URL
+  CACHES:
+    default:
+      BACKEND: 'django.core.cache.backends.memcached.MemcachedCache'
+      LOCATION: $DISCERN_MEMCACHE
+  CELERY_RESULT_BACKEND: $DISCERN_RESULT_BACKEND
+  DATABASES:
+    default:
+      ENGINE: django.db.backends.mysql
+      HOST: $DISCERN_MYSQL_HOST
+      NAME: $DISCERN_MYSQL_DB_NAME
+      PASSWORD: $DISCERN_MYSQL_PASSWORD
+      PORT: $DISCERN_MYSQL_PORT
+      USER: $DISCERN_MYSQL_USER
+  GOOGLE_ANALYTICS_PROPERTY_ID: $DISCERN_GOOGLE_ANALYTICS_PROPERTY_ID
+
+
+discern_debian_pkgs:
+  - policykit-1
+  - python-virtualenv
+  - gcc
+  - g++
+  - build-essential
+  - python-dev
+  - gfortran
+  - libfreetype6-dev
+  - libpng12-dev
+  - libxml2-dev
+  - libxslt1-dev
+  - libreadline6
+  - libreadline6-dev
+  - redis-server
+  - python-pip
+  - ipython
+  - nginx
+  - libmysqlclient-dev
+  - libblas3gf
+  - libblas-dev
+  - liblapack3gf
+  - liblapack-dev
+  - libatlas-base-dev
+  - curl
+  - yui-compressor
+
+discern_ease_debian_pkgs:
+  - python-pip
+  - gcc
+  - g++
+  - gfortran
+  - libblas3gf
+  - libblas-dev
+  - liblapack3gf
+  - liblapack-dev
+  - libatlas-base-dev
+  - libxml2-dev
+  - libxslt1-dev
+  - aspell
+  - python

playbooks/roles/discern/tasks/deploy.yml

+---
+#Upload config files for django (auth and env)
+- name: discern | create discern application config env.json file
+  template: src=env.json.j2 dest={{ discern_app_dir }}/env.json
+  sudo_user: "{{ discern_user }}"
+  notify:
+    - discern | restart celery
+    - discern | restart discern
+  tags:
+    - deploy
+
+- name: discern | create discern auth file auth.json
+  template: src=auth.json.j2 dest={{ discern_app_dir }}/auth.json
+  sudo_user: "{{ discern_user }}"
+  notify:
+    - discern | restart celery
+    - discern | restart discern
+  tags:
+    - deploy
+
+- name: discern | git checkout discern repo into discern_code_dir
+  git: dest={{ discern_code_dir }} repo={{ discern_source_repo }} version={{ discern_branch }}
+  sudo_user: "{{ discern_user }}"
+  notify:
+    - discern | restart celery
+    - discern | restart discern
+  tags:
+    - deploy
+
+- name: discern | git checkout ease repo into discern_ease_code_dir
+  git: dest={{ discern_ease_code_dir}} repo={{ discern_ease_source_repo }} version={{ discern_ease_version }}
+  sudo_user: "{{ discern_user }}"
+  notify:
+    - discern | restart celery
+    - discern | restart discern
+  tags:
+    - deploy
+
+#Numpy has to be a pre-requirement in order for scipy to build
+- name : install python pre-requirements for discern and ease
+  pip: requirements={{item}} virtualenv={{ discern_venv_dir }} state=present
+  sudo_user: "{{ discern_user }}"
+  with_items:
+    - "{{ discern_pre_requirements_file }}"
+    - "{{ discern_ease_pre_requirements_file }}"
+  tags:
+    - deploy
+
+- name : install python requirements for discern and ease
+  pip: requirements={{item}} virtualenv={{ discern_venv_dir }} state=present
+  sudo_user: "{{ discern_user }}"
+  with_items:
+    - "{{ discern_post_requirements_file }}"
+    - "{{ discern_ease_post_requirements_file }}"
+  tags:
+    - deploy
+
+- name: discern | install ease python package
+  shell: command="{{discern_venv_dir}}/bin/activate; cd {{ discern_ease_code_dir }}; python setup.py install"
+  tags:
+    - deploy
+
+#Needed for the ease package to work
+- name: discern | install nltk data using rendered shell script
+  shell: command="{{ discern_venv_dir }}/bin/python -m nltk.downloader -d {{ discern_nltk_data_dir }} all"
+  sudo_user: "{{ discern_user }}"
+  tags:
+    - deploy
+
+#Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
+#support virtualenvs as of this comment
+- name: discern | django syncdb migrate and collectstatic for discern
+  shell: >
+    command="{{ discern_venv_dir }}/bin/python {{discern_code_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_code_dir}}"
+      chdir={{ discern_code_dir }}
+  sudo_user: "{{ discern_user }}"
+  with_items:
+    - syncdb
+    - migrate
+    - collectstatic
+  tags:
+    - deploy
+#Have this separate from the other three because it doesn't take the noinput flag
+- name: discern | django update_index for discern
+  shell: >
+    command="{{ discern_venv_dir}}/bin/python {{discern_code_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_code_dir}}"
+      chdir={{ discern_code_dir }}
+  sudo_user: "{{ discern_user }}"
+  tags:
+    - deploy

playbooks/roles/discern/tasks/main.yml

 ---
-#Create the templates for upstart services
+- name: discern | create application user
+  user: >
+    name="{{ discern_user }}"
+    home="{{ discern_app_dir }}"
+    createhome=no
+    shell=/bin/false
+
+- name: discern | create discern app and data dir
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ discern_user }}"
+    group="{{ common_web_group }}"
+  with_items:
+    - "{{ discern_app_dir }}"
+    - "{{ discern_data_dir }}"
+    - "{{ discern_venvs_dir }}"
+
+- name: discern | install debian packages that discern needs
+  apt: pkg={{ item }} state=present
+  with_items: discern_debian_pkgs
+
+- name: discern | install debian packages for ease that discern needs
+  apt: pkg={{ item }} state=present
+  with_items: discern_ease_debian_pkgs
+
 - name: discern | render celery service from template
-  template: src=celery.conf.j2 dest=/etc/init/celery.conf owner=root group=edx mode=0664
+  template: >
+    src=celery.conf.j2 dest=/etc/init/celery.conf
+    owner=root group=root
   notify: discern | restart celery
 
 - name: discern | render discern service from template
-  template: src=discern.conf.j2 dest=/etc/init/discern.conf owner=root group=edx mode=0664
+  template: >
+    src=discern.conf.j2 dest=/etc/init/discern.conf
+    owner=root group=root
   notify: discern | restart discern
 
-#Allows us to recover from a bad sudoers file
-- name: discern | Install policykit
-  apt: pkg=policykit-1 install_recommends=yes state=present update_cache=yes
-
-#Discern user is admin
-- name: discern | Create discern user
-  user: name={{ discern_user }} append=yes groups="adm,edx" shell=/bin/bash
-
-- name: discern | upload sudoers template to /tmp/sudoers
-  copy: src=sudoers-discern dest=/tmp/{{site_name}} owner=root group=root mode=0440
-
-#Verify file and move to sudoers.d folder
-- name: discern | move temp file to sudoers.d
-  shell: visudo -q -c -f /tmp/{{site_name}} && cp /tmp/{{site_name}} /etc/sudoers.d/{{site_name}}
-
-#Upload config files for django (auth and env)
-- name: discern | create discern application config env.json file
-  template: src=env.json.j2 dest={{app_base_dir}}/env.json owner={{discern_user}} group=edx mode=0640
-  notify:
-    - discern | restart celery
-    - discern | restart discern
-
-- name: discern | create discern auth file auth.json
-  template: src=auth.json.j2 dest={{app_base_dir}}/auth.json owner={{discern_user}} group=edx mode=0640
-  notify:
-    - discern | restart celery
-    - discern | restart discern
+- name: discern | copy sudoers file for discern
+  copy: >
+    src=sudoers-discern dest=/etc/sudoers.d/discern
+    mode=0440 validate='visudo -cf %s' owner=root group=root
 
 #Needed if using redis to prevent memory issues
 - name: discern | change memory commit settings -- needed for redis
   command: sysctl vm.overcommit_memory=1
 
-- name: discern | set permissions on app_base_dir sgid for edx
-  file: path={{app_base_dir}} owner=root group=edx mode=2775 state=directory
-  file: path={{venv_dir}} owner=root group=edx mode=2775 state=directory
-
-- name: discern | Install git so that we can clone repos
-  apt: pkg=git install_recommends=yes state=present
-
-#Create directories for repos
-- name: discern | create discern and ease directories and set permissions
-  file: path={{item}} owner={{discern_user}} group=edx mode=2775 state=directory
-  with_items:
-   - ${discern_dir}
-   - ${ease_dir}
-
-#Grab both repos or update
-- name: discern | git checkout discern repo into discern_dir
-  git: dest={{discern_dir}} repo={{discern_source_repo}} version={{discern_branch}}
-  notify:
-    - discern | restart celery
-    - discern | restart discern
-
-- name: discern | git checkout ease repo into ease_dir
-  git: dest={{ease_dir}} repo={{ease_source_repo}} version={{ease_branch}}
-  notify:
-    - discern | restart celery
-    - discern | restart discern
-
-#Install system packages
-- name: discern | install discern and ease apt packages
-  command: xargs -a {{item}}/apt-packages.txt apt-get install -y
-  with_items:
-    - ${discern_dir}
-    - ${ease_dir}
-
-#Numpy has to be a pre-requirement in order for scipy to build
-- name : install python pre-requirements for discern and ease
-  pip: requirements="{{item}}/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
-  with_items:
-    - ${discern_dir}
-    - ${ease_dir}
-
-- name : install python requirements for discern and ease
-  pip: requirements="{{item}}/requirements.txt" virtualenv="{{venv_dir}}" state=present
-  with_items:
-    - ${discern_dir}
-    - ${ease_dir}
-
-- name: discern | install ease python package
-  shell: command="{{venv_dir}}/bin/activate; cd {{ease_dir}}; python setup.py install"
-
-#Needed for the ease package to work
-- name: discern | install nltk data using rendered shell script
-  shell: command="{{venv_dir}}/bin/python -m nltk.downloader -d {{nltk_data_dir}} all"
-
-- name: discern | set permissions on nltk data directory
-  file: path={{nltk_data_dir}} owner={{discern_user}} group=edx mode=2775 state=directory
-
-#Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
-#support virtualenvs as of this comment
-- name: discern | django syncdb, migrate, and collectstatic for discern
-  shell: ${venv_dir}/bin/python {{discern_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_dir}}
-  with_items:
-    - syncdb
-    - migrate
-    - collectstatic
+- include: deploy.yml
 
-#Have this separate from the other three because it doesn't take the noinput flag
-- name: discern | django update_index for discern
-  shell: ${venv_dir}/bin/python {{discern_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_dir}}
+- name: discern | create a symlink for venv python
+  file: >
+    src="{{ discern_venv_bin }}/python"
+    dest={{ COMMON_BIN_DIR }}/python.discern
+    state=link

playbooks/roles/discern/templates/auth.json.j2

-{{ auth_config | to_nice_json }}
+{{ discern_auth_config | to_nice_json }}
\ No newline at end of file

playbooks/roles/discern/templates/celery.conf.j2

@@ -11,7 +11,7 @@ respawn limit 3 30
 
 env DJANGO_SETTINGS_MODULE={{discern_settings}}
 
-chdir {{discern_dir}}
+chdir {{ discern_code_dir }}
 setuid {{discern_user}}
 
-exec {{venv_dir}}/bin/python {{discern_dir}}/manage.py celeryd --loglevel=info --settings={{discern_settings}} --pythonpath={{discern_dir}} -B --autoscale={{ ansible_processor_cores * 2 }},1
+exec {{ discern_venv_dir }}/bin/python {{ discern_code_dir }}/manage.py celeryd --loglevel=info --settings={{ discern_settings }} --pythonpath={{ discern_code_dir }} -B --autoscale={{ ansible_processor_cores * 2 }},1

playbooks/roles/discern/templates/discern.conf.j2

@@ -14,9 +14,9 @@ env WORKERS={{ ansible_processor_cores * 2 }}
 env PORT={{ discern_gunicorn_port }}
 env ADDRESS={{ discern_gunicorn_host }}
 env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE={{discern_settings}}
+env DJANGO_SETTINGS_MODULE={{ discern_settings }}
 
-chdir {{discern_dir}}
-setuid {{discern_user}}
+chdir {{ discern_code_dir }}
+setuid {{ discern_user }}
 
-exec {{venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=30 --pythonpath={{discern_dir}} discern.wsgi
+exec {{ discern_venv_bin }}/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=30 --pythonpath={{ discern_code_dir }} discern.wsgi

playbooks/roles/discern/templates/env.json.j2

-{{ env_config | to_nice_json }}
+{{ discern_env_config | to_nice_json }}
\ No newline at end of file

playbooks/roles/edxapp/defaults/main.yml

@@ -54,7 +54,7 @@ EDXAPP_ANALYTICS_API_KEY:  ''
 EDXAPP_ZENDESK_USER: ''
 EDXAPP_ZENDESK_API_KEY: ''
 EDXAPP_CELERY_USER: 'celery'
-EDXAPP_CELERY_PASSWORD: ''
+EDXAPP_CELERY_PASSWORD: 'celery'
 
 EDXAPP_MITX_FEATURES:
   AUTH_USE_OPENID_PROVIDER: true
@@ -75,7 +75,7 @@ EDXAPP_CELERY_BROKER_HOSTNAME: ''
 EDXAPP_LOGGING_ENV:  'sandbox'
 
 EDXAPP_SYSLOG_SERVER: ''
-EDXAPP_RABBIT_HOSTNAME: 'rabbit.{{ENV_NAME}}.vpc.edx.org'
+EDXAPP_RABBIT_HOSTNAME: 'localhost'
 EDXAPP_XML_MAPPINGS: {}
 
 EDXAPP_LMS_NGINX_PORT: 80
@@ -85,12 +85,68 @@ EDXAPP_CMS_NGINX_PORT: 18010
 EDXAPP_LMS_BASIC_AUTH: False
 EDXAPP_CMS_BASIC_AUTH: False
 EDXAPP_LMS_PREVIEW_BASIC_AUTH: False
-
+EDXAPP_LANG: 'en_US.UTF-8'
 
 #-------- Everything below this line is internal to the role ------------
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
 #see http://atechie.net/2009/07/merging-hashes-in-yaml-conf-files/
+
+edxapp_data_dir: "{{ COMMON_DATA_DIR }}/edxapp"
+edxapp_app_dir: "{{ COMMON_APP_DIR }}/edxapp"
+edxapp_log_dir: "{{ COMMON_LOG_DIR }}/edxapp"
+edxapp_venvs_dir: "{{ edxapp_app_dir }}/venvs"
+edxapp_venv_dir: "{{ edxapp_venvs_dir }}/edxapp"
+edxapp_venv_bin: "{{ edxapp_venv_dir }}/bin"
+edxapp_rbenv_dir: "{{ edxapp_app_dir }}"
+edxapp_rbenv_root: "{{ edxapp_rbenv_dir }}/.rbenv"
+edxapp_rbenv_shims: "{{ edxapp_rbenv_root }}/shims"
+edxapp_rbenv_bin: "{{ edxapp_rbenv_root }}/bin"
+edxapp_gem_root: "{{ edxapp_rbenv_dir }}/.gem"
+edxapp_gem_bin: "{{ edxapp_gem_root }}/bin"
+edxapp_user: edxapp
+edxapp_deploy_path: "{{ edxapp_venv_bin }}:{{ edxapp_code_dir }}/bin:{{ edxapp_rbenv_bin }}:{{ edxapp_rbenv_shims }}:{{ edxapp_gem_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+edxapp_staticfile_dir: "{{ edxapp_data_dir }}/staticfiles"
+edxapp_course_data_dir: "{{ edxapp_data_dir }}/data"
+edxapp_upload_dir: "{{ edxapp_data_dir }}/uploads"
+edxapp_theme_dir: "{{ edxapp_data_dir }}/themes"
+edxapp_workers:
+  - queue: low
+    service_variant: cms
+    concurrency: 3
+  - queue: default
+    service_variant: cms
+    concurrency: 4
+  - queue: high
+    service_variant: cms
+    concurrency: 1
+  - queue: low
+    service_variant: lms
+    concurrency: 1
+  - queue: default
+    service_variant: lms
+    concurrency: 3
+  - queue: high
+    service_variant: lms
+    concurrency: 4
+
+
+
+
+
+
+# TODO: old style variable syntax is necessary
+# here until ansible 1.4
+edxapp_deploy_environment:
+  LANG: "en_US.UTF-8"
+  NO_PREREQ_INSTALL: 1
+  SKIP_WS_MIGRATIONS: 1
+  RBENV_ROOT: $edxapp_rbenv_root
+  GEM_HOME: $edxapp_gem_root
+  GEM_PATH: $edxapp_gem_root
+  PATH: $edxapp_deploy_path
+
+
 edxapp_generic_auth_config:  &edxapp_generic_auth
   AWS_ACCESS_KEY_ID:  $EDXAPP_AWS_ACCESS_KEY_ID
   AWS_SECRET_ACCESS_KEY: $EDXAPP_AWS_SECRET_ACCESS_KEY
@@ -126,7 +182,7 @@ edxapp_generic_auth_config:  &edxapp_generic_auth
         collection:  'modulestore'
         db:  $EDXAPP_MONGO_DB_NAME
         default_class:  'xmodule.hidden_module.HiddenDescriptor'
-        fs_root:  '/opt/wwc/data'
+        fs_root:  $edxapp_course_data_dir
         host: $EDXAPP_MONGO_HOSTS
         password:  $EDXAPP_MONGO_PASSWORD
         port: $EDXAPP_MONGO_PORT
@@ -161,6 +217,7 @@ edxapp_generic_auth_config:  &edxapp_generic_auth
   CELERY_BROKER_PASSWORD: $EDXAPP_CELERY_PASSWORD
 
 generic_env_config:  &edxapp_generic_env
+  STATIC_ROOT_BASE: $edxapp_staticfile_dir
   LMS_BASE: $EDXAPP_LMS_BASE
   CMS_BASE: $EDXAPP_CMS_BASE
   BOOK_URL:  $EDXAPP_BOOK_URL
@@ -172,7 +229,7 @@ generic_env_config:  &edxapp_generic_env
   WIKI_ENABLED: true
   SYSLOG_SERVER:  $EDXAPP_SYSLOG_SERVER
   SITE_NAME:  $EDXAPP_SITE_NAME
-  LOG_DIR:  "{{ storage_base_dir }}/logs/edx"
+  LOG_DIR:  "{{ COMMON_DATA_DIR }}/logs/edx"
   MEDIA_URL:  $EDXAPP_MEDIA_URL
   ANALYTICS_SERVER_URL:  $EDXAPP_ANALYTICS_SERVER_URL
   FEEDBACK_SUBMISSION_EMAIL: $EDXAPP_FEEDBACK_SUBMISSION_EMAIL
@@ -218,7 +275,7 @@ lms_auth_config:
           xml:
             ENGINE: 'xmodule.modulestore.xml.XMLModuleStore'
             OPTIONS:
-              data_dir: '/opt/wwc/data'
+              data_dir: $edxapp_course_data_dir
               default_class: 'xmodule.hidden_module.HiddenDescriptor'
           default:
             OPTIONS:
@@ -230,7 +287,7 @@ lms_auth_config:
               user: $EDXAPP_MONGO_USER
               password: $EDXAPP_MONGO_PASSWORD
               port: $EDXAPP_MONGO_PORT
-              fs_root: '/opt/wwc/data'
+              fs_root: $edxapp_course_data_dir
             ENGINE: 'xmodule.modulestore.mongo.MongoModuleStore'
             DOC_STORE_CONFIG: *edxapp_generic_default_docstore
 
@@ -250,7 +307,7 @@ lms_preview_env_config:
 
 
 # install dir for the edx-platform repo
-edx_platform_code_dir: "{{ app_base_dir }}/edx-platform"
+edxapp_code_dir: "{{ edxapp_app_dir }}/edx-platform"
 
 
 # gunicorn ports/hosts, these shouldn't need to be overridden
@@ -273,6 +330,7 @@ service_variants_enabled:
   - cms
 
 edxapp_lms_env: 'lms.envs.aws'
+edxapp_cms_env: 'cms.envs.aws'
 
 
 #Number of gunicorn worker processes to spawn, as a multiplier to number of virtual cores
@@ -285,29 +343,30 @@ worker_core_mult:
 #To turn off theming, specify edxapp_theme_name: ''
 #Stanford, for example, uses edxapp_theme_name: 'stanford'
 edxapp_theme_name: ''
-edxapp_theme_source_repo: 'https://{{ GIT_MIRROR }}/Stanford-Online/edx-theme.git'
+edxapp_theme_source_repo: 'https://{{ COMMON_GIT_MIRROR }}/Stanford-Online/edx-theme.git'
 edxapp_theme_version: 'HEAD'
 
 # make this the public URL instead of writable
-edx_platform_repo: "https://{{ GIT_MIRROR }}/edx/edx-platform.git"
+edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-platform.git"
 # `edx_platform_commit` can be anything that git recognizes as a commit
 # reference, including a tag, a branch name, or a commit hash
 edx_platform_commit: 'release'
-local_requirements_file:  "{{ edx_platform_code_dir }}/requirements/edx/local.txt"
-pre_requirements_file:    "{{ edx_platform_code_dir }}/requirements/edx/pre.txt"
-post_requirements_file:   "{{ edx_platform_code_dir }}/requirements/edx/post.txt"
-base_requirements_file:   "{{ edx_platform_code_dir }}/requirements/edx/base.txt"
-github_requirements_file: "{{ edx_platform_code_dir }}/requirements/edx/github.txt"
-repo_requirements_file:   "{{ edx_platform_code_dir }}/requirements/edx/repo.txt"
+local_requirements_file:  "{{ edxapp_code_dir }}/requirements/edx/local.txt"
+pre_requirements_file:    "{{ edxapp_code_dir }}/requirements/edx/pre.txt"
+post_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/post.txt"
+base_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/base.txt"
+github_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/github.txt"
+repo_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/repo.txt"
 
-sandbox_base_requirements:  "{{ edx_platform_code_dir }}/requirements/edx-sandbox/base.txt"
-sandbox_local_requirements: "{{ edx_platform_code_dir }}/requirements/edx-sandbox/local.txt"
-sandbox_post_requirements:  "{{ edx_platform_code_dir }}/requirements/edx-sandbox/post.txt"
+sandbox_base_requirements:  "{{ edxapp_code_dir }}/requirements/edx-sandbox/base.txt"
+sandbox_local_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/local.txt"
+sandbox_post_requirements:  "{{ edxapp_code_dir }}/requirements/edx-sandbox/post.txt"
 
 #do we want to install the sandbox requirements into the regular virtual env
 install_sandbox_reqs_into_regular_venv: true
 
-lms_debian_pkgs:
+edxapp_debian_pkgs:
+  - npm
   # for compiling the virtualenv
   # (only needed if wheel files aren't available)
   - build-essential
@@ -336,22 +395,11 @@ lms_debian_pkgs:
   - libgeos-dev
 
 # Ruby Specific Vars
-ruby_base: /opt/www
-rbenv_root: "{{ ruby_base }}/.rbenv"
-ruby_version: "1.9.3-p374"
-gem_home: "{{ ruby_base }}/.gem"
+edxapp_ruby_version: "1.9.3-p374"
 
 # Deploy Specific Vars
-lms_variant: lms
-cms_variant: cms
-
-deploy_environment:
-  LANG: "en_US.UTF-8"
-  NO_PREREQ_INSTALL: 1
-  SKIP_WS_MIGRATIONS: 1
-  RBENV_ROOT: "{{ rbenv_root }}"
-  GEM_HOME: "{{ gem_home }}"
-  PATH: "{{ venv_dir }}/bin:{{ edx_platform_code_dir }}/bin:{{ rbenv_root }}/bin:{{ rbenv_root }}/shims:{{ gem_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+edxapp_lms_variant: lms
+edxapp_cms_variant: cms
 
 # Worker Settings
 worker_django_settings_module: 'aws'

playbooks/roles/edxapp/handlers/main.yml

 ---
-- name: start edxapp
+- name: edxapp | start edxapp
   service: name=edxapp state=started
   tags:
   - lms
@@ -7,7 +7,7 @@
   - cms
   - deploy
 
-- name: stop edxapp
+- name: edxapp | stop edxapp
   service: name=edxapp state=stopped
   tags:
   - lms
@@ -15,7 +15,7 @@
   - cms
   - deploy
 
-- name: restart edxapp
+- name: edxapp | restart edxapp
   service: name=edxapp state=restarted
   tags:
   - lms

playbooks/roles/edxapp/meta/main.yml

+---
+dependencies:
+  - role: rbenv
+    rbenv_user: "{{ edxapp_user }}"
+    rbenv_dir: "{{ edxapp_app_dir }}"
+    rbenv_ruby_version: "{{ edxapp_ruby_version }}"

playbooks/roles/edxapp/tasks/cms.yml

-# requires:
-#  - group_vars/all
-#  - common/tasks/main.yml
----
-- name: create cms application config
-  template: src=cms.env.json.j2 dest=$app_base_dir/cms.env.json mode=640 owner=www-data group=adm
-  tags:
-  - cms-env
-  - cms
-  - update
-  - deploy
-
-- name: create cms auth file
-  template: src=cms.auth.json.j2 dest=$app_base_dir/cms.auth.json mode=640 owner=www-data group=adm
-  tags:
-  - cms-env
-  - cms
-  - update
-  - deploy
-
-- name: Create CMS log target directory
-  file: path={{log_base_dir}}/cms state=directory owner=syslog group=syslog mode=2750
-  tags:
-  - cms 
-  - cms-env
-  - logging
-  - update
-  - deploy
-
-# Creates CMS upstart file
-- include: upstart.yml basename=cms
-  when: celery_worker is not defined
-
-- include: upstart.yml basename=edx-worker-cms
-  when: celery_worker is defined

playbooks/roles/edxapp/tasks/lms-preview.yml

-# requires:
-#  - group_vars/all
-#  - common/tasks/main.yml
----
-- name: create lms application config
-  template: src=lms-preview.env.json.j2 dest=$app_base_dir/lms-preview.env.json mode=640 owner=www-data group=adm
-  tags:
-  - lms-preview
-  - lms-preview-env
-  - deploy
-
-- name: create lms auth file
-  template: src=lms-preview.auth.json.j2 dest=$app_base_dir/lms-preview.auth.json mode=640 owner=www-data group=adm
-  tags:
-  - lms-preview
-  - lms-preview-env
-  - deploy
-
-- name: Create lms-preview log target directory
-  file: path={{log_base_dir}}/lms-preview state=directory owner=syslog group=syslog mode=2750
-  tags:
-  - lms-preview
-  - lms-preview-env
-  - logging
-  - update
-  - deploy
-
-# Creates LMS Preview upstart file
-- include: upstart.yml basename=lms-preview

playbooks/roles/edxapp/tasks/lms.yml

----
-- name: create lms application config
-  template: src=lms.env.json.j2 dest=$app_base_dir/lms.env.json mode=640 owner=www-data group=adm
-  tags:
-  - lms
-  - lms-env
-  - update
-  - deploy
-
-- name: create lms auth file
-  template: src=lms.auth.json.j2 dest=$app_base_dir/lms.auth.json mode=640 owner=www-data group=adm
-  tags:
-  - lms
-  - lms-env
-  - update
-  - deploy
-
-- name: Create lms log target directory
-  file: path={{log_base_dir}}/lms state=directory owner=syslog group=syslog mode=2750
-  tags:
-  - lms
-  - lms-env
-  - logging
-  - update
-  - deploy
-
-# Creates LMS upstart file
-- include: upstart.yml basename=lms
-  when: celery_worker is not defined
-
-- include: upstart.yml basename=edx-worker-lms
-  when: celery_worker is defined

playbooks/roles/edxapp/tasks/main.yml

@@ -2,54 +2,61 @@
 #  - group_vars/all
 #  - common/tasks/main.yml
 ---
-- name: Change permissions on datadir
-  file: path={{ app_base_dir }}/data state=directory owner=www-data group=www-data
-  tags:
-  - cms
-  - lms
-  - lms-env
-  - update
-
-- name: Change owner on staticfiles
-  file: path={{ app_base_dir }}/staticfiles state=directory owner=www-data group=adm
-  tags:
-  - cms
-  - lms
-  - lms-env
-  - update
-
-- name: Create theming directory
-  file: path={{ app_base_dir }}/themes state=directory mode=2775 group=adm owner=www-data
-  tags:
-  - cms
-  - lms
-  - cms-env
-  - lms-env
-  - update
-
-- name: install a bunch of system packages on which LMS and CMS rely
-  apt: pkg={{','.join(lms_debian_pkgs)}} state=present
-  tags:
-  - lms
-  - cms
-  - install
-
-- name: creating edxapp upstart script
-  sudo: True
-  template: src=edxapp.conf.j2 dest=/etc/init/edxapp.conf owner=root group=root
-  when: "celery_worker is not defined"
-  tags:
-  - upstart
-  - gunicorn
-  - update
-
-- name: create edx-workers upstart script
-  template: src=edx-workers.conf.j2 dest=/etc/init/edx-workers.conf owner=root group=root
-  when: "celery_worker is defined"
-  tags:
-  - upstart
-  - update
-
-- include: npm.yml
-- include: ruby.yml
+
+
+- name: edxapp | Install logrotate configuration for tracking file
+  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
+
+- name: edxapp | create application user
+  user: >
+    name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"
+    createhome=no shell=/bin/false
+
+- name: edxapp | create edxapp app dir
+  file: >
+    path="{{ item }}" state=directory
+    owner="{{ edxapp_user }}" group="{{ common_web_group }}"
+  with_items:
+    - "{{ edxapp_app_dir }}"
+    - "{{ edxapp_venvs_dir }}"
+
+- name: edxapp | create edxapp log dir
+  file: >
+    path="{{ edxapp_log_dir }}" state=directory
+    owner="{{ common_log_user }}" group="{{ common_log_user }}"
+
+- name: edxapp | create edxapp writable dirs
+  file: >
+    path="{{ item }}" state=directory
+    owner="{{ edxapp_user }}" group="{{ edxapp_user }}"
+  with_items:
+    - "{{ edxapp_staticfile_dir }}"
+    - "{{ edxapp_theme_dir }}"
+
+
+- name: edxapp | create web-writable edxapp data dirs
+  file: >
+    path="{{ item }}" state=directory
+    owner="{{ common_web_user }}" group="{{ edxapp_user }}"
+    mode="0775"
+  with_items:
+    - "{{ edxapp_course_data_dir }}"
+    - "{{ edxapp_upload_dir }}"
+
+- name: edxapp | install system packages on which LMS and CMS rely
+  apt: pkg={{','.join(edxapp_debian_pkgs)}} state=present
+
+- name: edxapp | create log directories for service variants
+  file: >
+    path={{ edxapp_log_dir }}/{{ item }} state=directory
+    owner={{ common_log_user }} group={{ common_log_user }}
+    mode=0750
+  with_items: service_variants_enabled
+
 - include: deploy.yml
+
+- name: edxapp | create a symlink for venv python
+  file: >
+    src="{{ edxapp_venv_bin }}/python"
+    dest={{ COMMON_BIN_DIR }}/python.edxapp
+    state=link

playbooks/roles/edxapp/tasks/npm.yml

-# requires:
-#   - common/tasks/main.yml
-#   - ruby/tasks/main.yml
----
-- name: Install npm
-  apt: pkg=npm state=present install_recommends=no
-  tags:
-  - npm
-  - install

playbooks/roles/edxapp/tasks/ruby.yml

-#
-#cribbed from https://github.com/mmoya/ansible-playbooks/blob/master/rbenv/main.yml
-- name: Create 'www' user (replicating historical environment)
-  user: name=www state=present
-  tags:
-  - ruby
-  - update
-
-- name: Create ruby base
-  file: path=$ruby_base state=directory owner=www group=www
-  tags:
-  - ruby
-  - update
-
-- name: rbenv | install build depends
-  apt: pkg=$item state=present install_recommends=no
-  with_items:
-    - build-essential
-    - git
-    - libcurl4-openssl-dev
-    - libmysqlclient-dev
-    - libreadline-dev
-    - libssl-dev
-    - libxml2-dev
-    - libxslt1-dev
-    - zlib1g-dev
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | update rbenv repo
-  git: repo=https://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | add rbenv to path
-  file: path=/usr/local/bin/rbenv src=${rbenv_root}/bin/rbenv state=link
-  tags:
-  - ruby
-  - update
-
-- name: rbenv | add rbenv initialization to profile
-  template: src=rbenv.sh.j2 dest=/etc/profile.d/rbenv.sh owner=root group=root mode=0755
-  tags:
-  - ruby
-  - update
-
-- name: rbenv | check ruby-build installed
-  command: test -x /usr/local/bin/ruby-build
-  register: rbuild_present
-  ignore_errors: yes
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | create temporary directory
-  command: mktemp -d
-  register: tempdir
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | clone ruby-build repo
-  git: repo=https://github.com/sstephenson/ruby-build.git dest=${tempdir.stdout}/ruby-build
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | install ruby-build
-  command: ./install.sh chdir=${tempdir.stdout}/ruby-build
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | remove temporary directory
-  file: path=${tempdir.stdout} state=absent
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | check ruby $ruby_version installed
-  shell: RBENV_ROOT=${rbenv_root} rbenv versions | grep $ruby_version
-  register: ruby_installed
-  ignore_errors: yes
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | install ruby $ruby_version
-  shell: RBENV_ROOT=${rbenv_root} rbenv install $ruby_version
-  when: ruby_installed|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | set global ruby $ruby_version
-  shell: RBENV_ROOT=${rbenv_root} rbenv global $ruby_version
-  when: ruby_installed|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | rehash
-  shell: RBENV_ROOT=${rbenv_root} rbenv rehash
-  when: ruby_installed|failed
-  tags:
-  - ruby
-  - install
-

playbooks/roles/edxapp/tasks/service_variant_config.yml

+- name: "create {{ item }} application config"
+  template: >
+    src={{ item }}.env.json.j2
+    dest={{ edxapp_app_dir }}/{{ item }}.env.json
+  sudo_user: "{{ edxapp_user }}"
+  with_items: service_variants_enabled
+  tags:
+  - deploy
+
+- name: "create {{ item }} auth file"
+  template: >
+    src={{ item }}.auth.json.j2
+    dest={{ edxapp_app_dir }}/{{ item }}.auth.json
+  sudo_user: "{{ edxapp_user }}"
+  with_items: service_variants_enabled
+  tags:
+  - deploy
+
+# write the supervisor script for {{ service_variant }}
+# for non-celery servers
+
+- name: "writing {{ item }} supervisor script"
+  template: >
+    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  with_items: service_variants_enabled
+  when: celery_worker is not defined
+  notify: supervisor | reload supervisor
+  tags:
+  - deploy
+
+- name: "writing edxapp supervisor script"
+  template: >
+    src=edxapp.conf.j2 dest={{ supervisor_cfg_dir }}/edxapp.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  when: celery_worker is not defined
+  tags:
+  - deploy
+
+
+# write the supervisor script for celery workers
+
+- name: writing celery worker supervisor script
+  template: >
+    src=workers.conf.j2 dest={{ supervisor_cfg_dir }}/workers.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  when: celery_worker is defined
+  tags:
+  - deploy
+
+- name: supervisor | reload supervisor
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} reload"
+
+# Gather assets using rake if possible
+
+- name: edxapp | gather {{ item }} static assets with rake
+  shell: >
+    command=SERVICE_VARIANT={{ item }} rake {{ item }}:gather_assets:aws
+    executable=/bin/bash
+    chdir={{ edxapp_code_dir }}
+  sudo_user: "{{ edxapp_user }}"
+  when: celery_worker is not defined
+  with_items: service_variants_enabled
+  environment: "{{ edxapp_deploy_environment }}"
+  tags:
+  - deploy
+
+
+- name: edxapp | syncdb and migrate
+  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin}}/django-admin.py syncdb --migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
+  when: migrate_db is defined and migrate_db|lower == "yes"
+  tags:
+  - deploy
+
+- name: edxapp | db migrate
+  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin }}/django-admin.py migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
+  when: migrate_only is defined and migrate_only|lower == "yes"
+  tags:
+  - deploy
+
+

playbooks/roles/edxapp/tasks/upstart.yml

-# write the gunicorn upstart script for {{ service_variant }}
-- name: writing {{ basename }} upstart script to /etc/init
-  sudo: True
-  template: src={{ basename }}.conf.j2 dest=/etc/init/{{ basename }}.conf owner=root group=root
-  tags:
-  - upstart
-  - gunicorn
-  - update
-  - deploy

playbooks/roles/edxapp/templates/cms.conf.j2

-# gunicorn
-# Templated and placed by ansible from jinja2 source
-description "cms gunicorn"
-# CMS Upstart Script
-start on started edxapp
-stop on stopped edxapp
- 
-respawn
-respawn limit 3 30
- 
-env PID=/var/tmp/cms.pid
-#env NEW_RELIC_CONFIG_FILE={{app_base_dir}}/newrelic.ini
-#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
+[program:cms]
 {% if ansible_processor|length > 0 %}
-env WORKERS={{ ansible_processor|length * worker_core_mult.cms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
 {% else %}
-env WORKERS={{ worker_core_mult.cms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b {{ edxapp_cms_gunicorn_host }}:{{ edxapp_cms_gunicorn_port }} -w {{ worker_core_mult.cms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} cms.wsgi
 {% endif %}
-env PORT={{edxapp_cms_gunicorn_port}}
-env ADDRESS={{edxapp_cms_gunicorn_host}}
-env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE=cms.envs.aws
-env SERVICE_VARIANT="cms"
- 
-chdir {{edx_platform_code_dir}}
-setuid www-data
 
-exec {{venv_dir}}/bin/gunicorn_django -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}} --settings=cms.envs.aws
+user={{ common_web_user }}
+directory={{ edxapp_code_dir }}
+environment=PORT={{edxapp_cms_gunicorn_port}},ADDRESS={{edxapp_cms_gunicorn_host}},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ edxapp_cms_env }},SERVICE_VARIANT="cms"
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log

playbooks/roles/edxapp/templates/edx-worker-cms.conf.j2

-# gunicorn
-# Templated and placed by ansible from jinja2 source
-# CMS Celery Worker Upstart Script
-description "cms celery worker"
-
-stop on stopping edx-workers
-
-respawn
-
-instance edx.${SERVICE_VARIANT}.core.${QUEUE}
-
-#env NEW_RELIC_CONFIG_FILE=/opt/wwc/newrelic.ini
-#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
-env CONCURRENCY=${CONCURRENCY}
-env LOGLEVEL=info
-env DJANGO_SETTINGS_MODULE={{worker_django_settings_module}}
-env PYTHONPATH={{edx_platform_code_dir}}
-env SERVICE_VARIANT=${SERVICE_VARIANT}
-
-setuid www-data
-
-chdir {{edx_platform_code_dir}}
-
-exec {{venv_dir}}/bin/python {{edx_platform_code_dir}}/manage.py $SERVICE_VARIANT --settings=$DJANGO_SETTINGS_MODULE celery worker --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-worker-lms.conf.j2

-# gunicorn
-# Templated and placed by ansible from jinja2 source
-# CMS Celery Worker Upstart Script
-description "lms celery worker"
-
-stop on stopping edx-workers
-
-respawn
-
-instance edx.${SERVICE_VARIANT}.core.${QUEUE}
-
-#env NEW_RELIC_CONFIG_FILE=/opt/wwc/newrelic.ini
-#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
-env CONCURRENCY=${CONCURRENCY}
-env LOGLEVEL=info
-env DJANGO_SETTINGS_MODULE={{worker_django_settings_module}}
-env PYTHONPATH={{edx_platform_code_dir}}
-env SERVICE_VARIANT=${SERVICE_VARIANT}
-
-setuid www-data
-
-chdir {{edx_platform_code_dir}}
-
-exec {{venv_dir}}/bin/python {{edx_platform_code_dir}}/manage.py lms --service-variant=$SERVICE_VARIANT --settings=$DJANGO_SETTINGS_MODULE celery worker --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-workers.conf.j2

-# edx workers
-# managed by puppet or ansible
-
-description "start edX app workers"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-pre-start script
-
-   {% if 'cms' in service_variants_enabled %}
-   start edx-worker-cms QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=cms
-   start edx-worker-cms QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=cms
-   start edx-worker-cms QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=cms
-   {% endif %}
-
-   {% if 'lms' in service_variants_enabled %}
-   start edx-worker-lms QUEUE=low CONCURRENCY=1 SERVICE_VARIANT=lms
-   start edx-worker-lms QUEUE=default CONCURRENCY=3 SERVICE_VARIANT=lms
-   start edx-worker-lms QUEUE=high CONCURRENCY=4 SERVICE_VARIANT=lms
-   {% endif %}
-
-end script

playbooks/roles/common/templates/edx_logrotate_tracking_log.j2 → playbooks/roles/edxapp/templates/edx_logrotate_tracking_log.j2

-{{log_base_dir}}/tracking.log {
+{{ COMMON_LOG_DIR }}/tracking.log {
   create
   compress
   delaycompress

playbooks/roles/edxapp/templates/edxapp.conf.j2

-#/etc/init/edxapp.conf
-
-description "Starts and stops multiple edX services, e.g., lms, cms, etc., installed in a stacked configuration."
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-##
-## Each awaited service is responsible for ensuring that it is ready
-## for service when it returns.
-##
-pre-start script
-
-{% if 'lms' in service_variants_enabled %}
-if [ -e /etc/init/lms.conf ]; then
-  start wait-for-state WAIT_FOR=lms WAITER=$UPSTART_JOB
-fi
-{% endif %}
-
-{% if 'lms-preview' in service_variants_enabled %}
-if [ -e /etc/init/lms-preview.conf ]; then
-  start wait-for-state WAIT_FOR=lms-preview WAITER=$UPSTART_JOB
-fi
-{% endif %}
-
-{% if 'cms' in service_variants_enabled %}
-if [ -e /etc/init/cms.conf ]; then
-  start wait-for-state WAIT_FOR=cms WAITER=$UPSTART_JOB
-fi
-{% endif %}
-
-end script
-
-script
-  # Noop process for other edX components to take their
-  # marching orders from.  In the edxapp deployment, 
-  # lms, cms, etc. will listen for this process to start
-  # and stop and follow suit.
-  while true
-    do
-      logger -t $0 "edX App Shell Daemon is running..."
-      sleep 600
-    done
-end script
-
-pre-stop script
-
-{% if 'lms' in service_variants_enabled %}
-if [ -e /etc/init/lms.conf ]; then
-  start wait-for-state WAIT_FOR=lms WAITER=$UPSTART_JOB TARGET_GOAL="stop"
-fi
-{% endif %}
-
-{% if 'lms-preview' in service_variants_enabled %}
-if [ -e /etc/init/lms-preview.conf ]; then
-  start wait-for-state WAIT_FOR=lms-preview WAITER=$UPSTART_JOB TARGET_GOAL="stop"
-fi
-{% endif %}
-
-{% if 'cms' in service_variants_enabled %}
-if [ -e /etc/init/cms.conf ]; then
-  start wait-for-state WAIT_FOR=cms WAITER=$UPSTART_JOB TARGET_GOAL="stop"
-fi
-{% endif %}
-
-end script
-
+[group:edxapp]
+programs={{ ",".join(service_variants_enabled) }}

playbooks/roles/edxapp/templates/lms-preview.conf.j2

@@ -10,8 +10,6 @@ respawn
 respawn limit 3 30
 
 env PID=/var/tmp/lms.pid
-#env NEW_RELIC_CONFIG_FILE={{app_base_dir}}/newrelic.ini
-#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
 {% if ansible_processor|length > 0 %}
 env WORKERS={{ ansible_processor|length * worker_core_mult.lms_preview }}
 {% else %}
@@ -23,10 +21,10 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms-preview"
   
-chdir {{edx_platform_code_dir}}
+chdir {{edxapp_code_dir}}
 setuid www-data
 
-exec {{venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}} lms.wsgi
+exec {{edxapp_venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edxapp_code_dir}} lms.wsgi
 
 post-start script
   while true

playbooks/roles/edxapp/templates/lms.conf.j2

-# gunicorn
-# Templated and placed by ansible from jinja2 source
-start on started edxapp
-stop on stopped edxapp
-
-respawn
-respawn limit 3 30
-
-env PID=/var/tmp/lms.pid
-#env NEW_RELIC_CONFIG_FILE={{app_base_dir}}/newrelic.ini
-#env NEWRELIC={{venv_dir}}/bin/newrelic-admin
+[program:lms]
 {% if ansible_processor|length > 0 %}
-env WORKERS={{ ansible_processor|length * worker_core_mult.lms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ ansible_processor|length * worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
 {% else %}
-env WORKERS={{ worker_core_mult.lms }}
+command={{ edxapp_venv_dir }}/bin/gunicorn --preload -b {{ edxapp_lms_gunicorn_host }}:{{ edxapp_lms_gunicorn_port }} -w {{ worker_core_mult.lms }} --timeout=300 --pythonpath={{ edxapp_code_dir }} lms.wsgi
 {% endif %}
-env PORT={{edxapp_lms_gunicorn_port}}
-env ADDRESS={{edxapp_lms_gunicorn_host}}
-env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }}
-env SERVICE_VARIANT="lms"
-
-chdir {{edx_platform_code_dir}}
-setuid www-data
-
-exec {{venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}} lms.wsgi
 
-post-start script
-  while true
-  do
-    if $(curl -s -i localhost:$PORT/heartbeat | egrep -q '200 OK'); then
-      break;
-    else
-      sleep 1;
-    fi
-  done
-end script
+user={{ common_web_user }}
+directory={{ edxapp_code_dir }}
+environment=PORT={{edxapp_lms_gunicorn_port}},ADDRESS={{edxapp_lms_gunicorn_host}},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }},SERVICE_VARIANT="lms"
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log

playbooks/roles/edxapp/templates/rbenv.sh.j2

-export RBENV_ROOT="{{ rbenv_root }}"
-export GEM_HOME="{{ gem_home }}"
-export PATH="{{ gem_home }}/bin:$PATH"
-eval "$(rbenv init -)"

playbooks/roles/edxapp/templates/workers.conf.j2

+{% for w in edxapp_workers %}
+[program:{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}]
+
+environment=CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE=aws,PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }}
+user={{ common_web_user }}
+directory={{ edxapp_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+
+command={{ edxapp_venv_bin}}/python {{ edxapp_code_dir }}/manage.py {{ w.service_variant }} --settings=aws celery worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.`hostname` --concurrency={{ w.concurrency }}
+
+{% endfor %}
+
+[group:edxapp_worker]
+programs={%- for w in edxapp_workers %}{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}{%- if not loop.last %},{%- endif %}{%- endfor %}

playbooks/roles/edxlocal/tasks/main.yml

@@ -10,7 +10,7 @@
 #  http://downloads.mysql.com/archives/mysql-5.1/mysql-5.1.62.tar.gz
 #
 ---
-- name: edxlocal| install packages needed for single server 
+- name: edxlocal| install packages needed for single server
   apt: pkg={{','.join(edxlocal_debian_pkgs)}} install_recommends=yes state=present
 
 - name: edxlocal | create a database for edxapp
@@ -31,5 +31,12 @@
     state=present
     encoding=utf8
 
+- name: edxlocal | create a database for discern
+  mysql_db: >
+    db=discern
+    state=present
+    encoding=utf8
+
+
 - name: edxlocal | install memcached
   apt: pkg=memcached state=present

playbooks/roles/forum/defaults/main.yml

 ---
+forum_app_dir: "{{ COMMON_APP_DIR }}/forum"
+forum_code_dir: "{{ forum_app_dir }}/cs_comments_service"
+forum_data_dir: "{{ COMMON_DATA_DIR }}/forum"
+forum_rbenv_dir: "{{ forum_app_dir }}"
+forum_rbenv_root: "{{ forum_app_dir }}/.rbenv"
+forum_rbenv_shims: "{{ forum_rbenv_root }}/shims"
+forum_rbenv_bin: "{{ forum_rbenv_root }}/bin"
+forum_supervisor_wrapper: "{{ forum_app_dir }}/forum-supervisor.sh"
+forum_gem_root: "{{ forum_rbenv_dir }}/.gem"
+forum_gem_bin: "{{ forum_gem_root }}/bin"
+forum_path: "{{ forum_code_dir }}/bin:{{ forum_rbenv_bin }}:{{ forum_rbenv_shims }}:{{ forum_gem_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+forum_environment:
+  RBENV_ROOT: "{{ forum_rbenv_root }}"
+  GEM_HOME: "{{ forum_gem_root }}"
+  GEM_PATH: "{{ forum_gem_root }}"
+  PATH: "{{ forum_path }}"
+  MONGOHQ_USER: "{{ forum_mongo_user }}"
+  MONGOHQ_PASS: "{{ forum_mongo_password }}"
+  RACK_ENV: "{{ forum_rack_env }}"
+  SINATRA_ENV: "{{ forum_sinatra_env }}"
+  API_KEY: "{{ forum_api_key }}"
+  SEARCH_SERVER: "{{ forum_elasticsearch_url }}"
+  MONGOHQ_URL: "{{ forum_mongo_url }}"
+  HOME: "{{ forum_app_dir }}"
 
 forum_user: "forum"
-forum_home: "/opt/wwc/forum"
 forum_ruby_version: "1.9.3-p448"
-forum_code_dir: "{{ forum_home }}/cs_comments_service"
 forum_source_repo: "https://github.com/edx/cs_comments_service.git"
 forum_version: "HEAD"
 forum_mongo_user: "cs_comments_service"

playbooks/roles/forum/handlers/main.yml

 ---
-
 - name: forum | restart the forum service
-  service: name=cs_comments_service state=restarted
+  supervisorctl: >
+    name=forum
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted

playbooks/roles/forum/meta/main.yml

+---
+dependencies:
+  - role: rbenv
+    # TODO: setting the rbenv ownership to
+    # the common_web_user is a workaround
+    rbenv_user: "{{ common_web_user }}"
+    rbenv_dir: "{{ forum_app_dir }}"
+    rbenv_ruby_version: "{{ forum_ruby_version }}"

playbooks/roles/forum/tasks/deploy.yml

 ---
 
 - name: forum | stop the forum service
-  service: name=cs_comments_service state=stopped
+  supervisorctl: >
+    name=forum
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=stopped
+  tags:
+  - deploy
+
+- name: forum | create the supervisor wrapper
+  template: >
+    src={{ forum_supervisor_wrapper|basename }}.j2
+    dest={{ forum_supervisor_wrapper }}
+    mode=0755
+  sudo_user: "{{ forum_user }}"
   tags:
   - deploy
 
 - name:  forum | git checkout forum repo into {{ forum_code_dir }}
   git: dest={{ forum_code_dir }} repo={{ forum_source_repo }} version={{ forum_version }}
-  sudo: yes
   sudo_user: "{{ forum_user }}"
-  notify: 
-    - forum | restart the forum service
   tags:
-  - forum
   - deploy
 
+# TODO: This is done as the common_web_user
+# since the process owner needs write access
+# to the rbenv
 - name: forum | install comments service bundle
-  shell: executable=/bin/bash {{ forum_home }}/.rbenv/shims/bundle install chdir={{ forum_code_dir }}
-  sudo: yes
-  sudo_user: "{{ forum_user }}"
+  shell: bundle install chdir={{ forum_code_dir }}
+  sudo_user: "{{ common_web_user }}"
+  environment: "{{ forum_environment }}"
   tags:
-  - forum
   - deploy
 
+- name: forum | create the supervisor config
+  template: >
+    src=forum.conf.j2 dest={{ supervisor_cfg_dir }}/forum.conf
+    owner={{ common_web_user }} group={{ supervisor_user }}
+    mode=0644
+  register: forum_supervisor
+
 - name: forum | restart the forum service
-  service: name=cs_comments_service state=restarted
-  tags:
-  - deploy
+  supervisorctl: >
+    name=forum
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted

playbooks/roles/forum/tasks/main.yml

@@ -8,65 +8,45 @@
 #   * elasticsearch
 #   * oraclejdk
 #   * rbenv
-# 
-# 
+#
+#
 # Example play:
 #   roles:
 #   - common
 #   - oraclejdk
-#   - elasticsearch 
+#   - elasticsearch
 #   - role: rbenv
 #     rbenv_user: "{{ forum_user }}"
-#     rbenv_user_home: "{{ forum_rbenv_dir }}"
+#     rbenv_dir: "{{ forum_rbenv_dir }}"
 #     rbenv_ruby_version: "{{ forum_ruby_version }}"
 #   - forum
 
-- name: forum | setup the forum env
-  template: src=forum_env.j2 dest={{ forum_home }}/forum_env owner={{ forum_user }} group={{ forum_user }}
-  notify: 
-    - forum | restart the forum service
-  tags:
-  - forum
-  - update
-
-- name: forum | ensure .bashrc exists
-  shell: touch {{ forum_home }}/.bashrc
-  sudo: true  
-  sudo_user: "{{ forum_user }}"
-  tags:
-  - forum
-  - update
+- name: forum | create application user
+  user: >
+    name="{{ forum_user }}" home="{{ forum_app_dir }}"
+    createhome=no
+    shell=/bin/false
 
-- name: forum | add source of ruby_env to .bashrc
-  lineinfile:
-    dest="{{ forum_home }}/.bashrc"
-    regexp='. {{ forum_home }}/forum_env'    
-    line='. {{ forum_home }}/forum_env'
-  notify: 
-    - forum | restart the forum service
-  tags:
-  - forum
-  - update
+- name: forum | create forum app dir
+  file: >
+    path="{{ forum_app_dir }}" state=directory
+    owner="{{ forum_user }}" group="{{ common_web_group }}"
 
-
-- name: forum | copy cs_comments_service SysVunit script
-  template: src=cs_comments_service.j2 dest=/etc/init.d/cs_comments_service owner=root group=root mode=750
-  notify: 
+- name: forum | setup the forum env
+  template: >
+    src=forum_env.j2 dest={{ forum_app_dir }}/forum_env
+    owner={{ forum_user }}  group={{ common_web_user }}
+    mode=0644
+  notify:
     - forum | restart the forum service
-  tags:
-  - forum
-  - install
-  when: ansible_distribution == 'Debian'
 
-
-- name: forum | copy cs_comments_service upstart script
-  template: src=cs_comments_service.conf.j2 dest=/etc/init/cs_comments_service.conf owner=root group=root mode=644
-  notify: 
-    - forum | restart the forum service
-  tags:
-  - forum
-  - install
-  when: ansible_distribution == 'Ubuntu'
+- name: forum | create the supervisor config
+  template: >
+    src=forum.conf.j2 dest={{ supervisor_cfg_dir }}/forum.conf
+    owner={{ common_web_user }} group={{ supervisor_user }}
+    mode=0644
+  register: forum_supervisor
 
 - include: deploy.yml
 - include: test.yml
+

playbooks/roles/forum/templates/cs_comments_service.conf.j2

@@ -10,7 +10,7 @@ env PID=/var/tmp/comments_service.pid
 chdir {{ forum_code_dir }}
 
 script
-  . {{forum_home}}/forum_env
-  {{forum_home}}/.rbenv/shims/ruby app.rb
+  . {{forum_app_dir}}/forum_env
+  {{forum_app_dir}}/.rbenv/shims/ruby app.rb
 
-end script
+end script
\ No newline at end of file

playbooks/roles/forum/templates/cs_comments_service.j2

@@ -15,7 +15,7 @@ USER={{ forum_user }}
 NAME="cs_comments_service"
 RETVAL=0
 APP_ROOT={{ forum_code_dir }}
-DAEMON={{ forum_home }}/.rbenv/shims/ruby
+DAEMON={{ forum_app_dir }}/.rbenv/shims/ruby
 PID=/var/tmp/cs_comments_service.pid
 OPTIONS="app.rb"
 

playbooks/roles/forum/templates/forum-supervisor.sh.j2

+#!/bin/bash
+source {{ forum_app_dir }}/forum_env
+cd {{ forum_code_dir }}
+{{ forum_rbenv_shims }}/ruby app.rb

playbooks/roles/forum/templates/forum.conf.j2

+[program:forum]
+command={{ forum_supervisor_wrapper }}
+priority=999
+user={{ common_web_user }}
+startsecs=10
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/forum/templates/forum_env.j2

 # {{ ansible_managed }}
-
-#
-# This file is sourced in .bashrc.
-# The environment variables are used by the cs_comments_service
-# to configure itself at start-time.
-#
-export MONGOHQ_USER="{{ forum_mongo_user }}"
-export MONGOHQ_PASS="{{ forum_mongo_password }}"
-export RACK_ENV="{{ forum_rack_env }}"
-export SINATRA_ENV="{{ forum_sinatra_env }}"
-export API_KEY="{{ forum_api_key }}"
-export SEARCH_SERVER="{{ forum_elasticsearch_url }}"
-export MONGOHQ_URL="{{ forum_mongo_url }}"
+{% for name,value in forum_environment.items() %}
+{%- if value %}
+export {{ name }}="{{ value }}"
+{%- endif %}                 
+{% endfor %}
+eval "$(rbenv init -)"
\ No newline at end of file

playbooks/roles/gh_users/tasks/main.yml

 ---
 
 # gh_users
-# 
+#
 # Creates OS accounts for users based on their github credential.
 # Takes a list gh_users as a parameter which is a list of users
 #
 # roles:
 #   - role: gh_users
 #     gh_users:
-#      - user: github_admin_username
-#        groups: 
-#        - adm
-#      - user: another_github_username
-#        groups: !!null
-         
+#      - joe
+#      - mark
+
 
 - fail: gh_users list must be defined for this parameterized role
   when: not gh_users
 
-- name: gh_users | create local user for github user
+- name: gh_users | create gh group
+  group: name=gh state=present
+
+# TODO: give limited sudo access to this group
+- name: gh_users | grant full sudo access to gh group
+  copy: >
+    content="%adm ALL=(ALL) NOPASSWD:ALL"
+    dest=/etc/sudoers.d/gh owner=root group=root
+    mode=0440 validate='visudo -cf %s'
+
+- name: gh_users | create github users
   user:
-    name={{ item.user }}
-    groups={{ ",".join(item.groups) }}
+    name={{ item }} group=gh
     shell=/bin/bash
   with_items: gh_users
 
 - name: gh_users | create .ssh directory
   file:
-    path=/home/{{ item.user }}/.ssh state=directory mode=0700
-    owner={{ item.user }} group={{ item.user }}
+    path=/home/{{ item }}/.ssh state=directory mode=0700
+    owner={{ item }} group={{ item }}
   with_items: gh_users
 
 - name: gh_users | copy github key[s] to .ssh/authorized_keys

playbooks/roles/jenkins_master/defaults/main.yml

-jenkins_home: "{{ storage_base_dir }}/jenkins"
+jenkins_home: "{{ COMMON_DATA_DIR }}/jenkins"
 jenkins_user: "jenkins"
 jenkins_group: "edx"
 jenkins_server_name: "jenkins.testeng.edx.org"

playbooks/roles/jenkins_master/tasks/main.yml

@@ -31,7 +31,7 @@
   file: path={{ jenkins_home }} recurse=yes state=directory
         owner={{ jenkins_user }} group={{ jenkins_group }}
 
-# Symlink /var/lib/jenkins to {{ storage_base_dir }}/jenkins
+# Symlink /var/lib/jenkins to {{ COMMON_DATA_DIR }}/jenkins
 # since Jenkins will expect its files to be in /var/lib/jenkins
 - name: jenkins_master | Symlink /var/lib/jenkins
   file: src={{ jenkins_home }} dest=/var/lib/jenkins state=link

playbooks/roles/jenkins_worker/defaults/main.yml

 ---
-jenkins_workspace: "{{ storage_base_dir }}/jenkins"
+jenkins_workspace: "{{ COMMON_DATA_DIR }}/jenkins"
 jenkins_phantomjs_url: https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2
 jenkins_phantomjs_archive: phantomjs-1.9.1-linux-x86_64.tar.bz2
 jenkins_phantomjs_folder: phantomjs-1.9.1-linux-x86_64
@@ -48,10 +48,10 @@ jscover_url: "http://superb-dca2.dl.sourceforge.net/project/jscover/JSCover-1.0.
 jscover_version: "1.0.2"
 
 # Mongo config
-mongo_dir: "{{ storage_base_dir }}/mongodb"
-mongo_log_dir: "{{ storage_base_dir }}/logs/mongodb"
+mongo_dir: "{{ COMMON_DATA_DIR }}/mongodb"
+mongo_log_dir: "{{ COMMON_DATA_DIR }}/logs/mongodb"
 
 # URL of S3 bucket containing pre-compiled Python packages
 python_pkg_url: "https://s3.amazonaws.com/jenkins.python_pkgs"
-python_download_dir: "{{ storage_base_dir }}/python_pkgs"
-python_virtualenv: "{{ storage_base_dir}}/venv"
+python_download_dir: "{{ COMMON_DATA_DIR }}/python_pkgs"
+python_virtualenv: "{{ COMMON_DATA_DIR}}/venv"

playbooks/roles/jenkins_worker/tasks/mongo.yml

 ---
 
-# Configure Mongo to use {{ storage_base_dir }} so we don't
+# Configure Mongo to use {{ COMMON_DATA_DIR }} so we don't
 # run out of disk space
 - name: jenkins_worker | Stop mongo service
   service: name=mongodb state=stopped

playbooks/roles/mongo/defaults/main.yml

-mongo_dbpath: /var/lib/mongodb
-mongo_logpath: /var/log/mongodb/mongodb.log
 mongo_logappend: true
 mongo_version: 2.4.7
 mongo_bind_ip: 127.0.0.1
 mongo_extra_conf: ''
 mongo_key_file: '/etc/mongodb_key'
 mongo_repl_set: rs0
-mongo_cluster_members: 
+mongo_cluster_members:
 
+mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo"
+mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo"
+mongo_user: mongodb
 MONGO_USERS:
   - user: cs_comments_service
     password: password
@@ -15,3 +16,6 @@ MONGO_USERS:
   - user: exdapp
     password: password
     database: edxapp
+
+mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
+mongo_dbpath: "{{ mongo_data_dir }}/mongodb"

playbooks/roles/mongo/tasks/main.yml

@@ -2,11 +2,8 @@
 ---
 - name: mongo | install python pymongo for mongo_user ansible module
   pip: >
-    name=pymongo
-    state=present
-    version=2.6.3
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags: mongo
+    name=pymongo state=present
+    version=2.6.3 extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
 
 - name: mongo | add the mongodb signing key
   apt_key: >
@@ -22,21 +19,25 @@
 - name: mongo | install mongo server and recommends
   apt: >
     pkg=mongodb-10gen={{ mongo_version }}
-    state=present
-    install_recommends=yes
+    state=present install_recommends=yes
     update_cache=yes
 
+- name: mongo | create mongo dirs
+  file: >
+    path="{{ item }}" state=directory
+    owner="{{ mongo_user  }}"
+    group="{{ mongo_user }}"
+  with_items:
+    - "{{ mongo_data_dir }}"
+    - "{{ mongo_dbpath }}"
+    - "{{ mongo_log_dir }}"
+
 - name: mongo | stop mongo service
   service: name=mongodb state=stopped
-  tags: mongo
 
-- name: mongo | move mongodb to {{ storage_base_dir }}
-  command: mv /var/lib/mongodb  {{ storage_base_dir}}/.  creates={{ storage_base_dir }}/mongodb
-  tags: mongo
+- name: mongo | move mongodb to {{ mongo_data_dir }}
+  command: mv /var/lib/mongodb  {{ mongo_data_dir}}/.  creates={{ mongo_data_dir }}/mongodb
 
-- name: mongo | create mongodb symlink
-  file: src={{ storage_base_dir }}/mongodb dest=/var/lib/mongodb state=link
-  tags: mongo
 
 - name: mongo | copy mongodb key file
   copy: >
@@ -50,25 +51,20 @@
 - name: mongo | copy configuration template
   template: src=mongodb.conf.j2 dest=/etc/mongodb.conf backup=yes
   notify: restart mongo
-  tags: mongo
 
 - name: mongo | start mongo service
   service: name=mongodb state=started
-  tags: mongo
 
 - name: mongo | wait for mongo server to start
   wait_for: port=27017 delay=2
-  tags: mongo
 
 - name: mongo | Create the file to initialize the mongod replica set
   template: src=repset_init.j2 dest=/tmp/repset_init.js
   when: mongo_clustered is defined
-  tags: mongo
 
 - name: mongo | Initialize the replication set
-  shell: /usr/bin/mongo  /tmp/repset_init.js 
-  when: mongo_clustered is defined
-  tags: mongo
+  shell: /usr/bin/mongo  /tmp/repset_init.js
+  when: mongo_clusterd is defined
 
 # Ignoring errors here because slave instances will fail this command
 # since slaveOk is false in ansible 1.3.
@@ -79,5 +75,3 @@
     password={{ item.password }}
     state=present
   with_items: MONGO_USERS
-  tags: mongo
-  ignore_errors: yes

playbooks/roles/nginx/defaults/main.yml

 # Variables for nginx role
 ---
+
+nginx_app_dir: "{{ COMMON_APP_DIR }}/nginx"
+nginx_data_dir: "{{ COMMON_DATA_DIR }}/nginx"
+nginx_conf_dir: "{{ COMMON_APP_DIR }}/conf.d"
+nginx_log_dir: "{{ COMMON_LOG_DIR }}/nginx"
+nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available"
+nginx_sites_enabled_dir: "{{ nginx_app_dir }}/sites-enabled"
+nginx_user: root
+
 pkgs:
   nginx:
     state: installed
@@ -25,8 +34,8 @@ nginx_cfg:
     edx_release: link
   # path to version files for the basic
   # nginx configuration
-  version_html: $app_base_dir/versions.html
-  version_json: $app_base_dir/versions.json
+  version_html: "{{ nginx_app_dir }}/versions.html"
+  version_json: "{{ nginx_app_dir }}/versions.json"
   # default htpasswd contents set to edx/edx
   # this value can be overiden in vars/secure/<group>.yml
   htpasswd: |

playbooks/roles/nginx/tasks/main.yml

 # requires:
 #   - common/tasks/main.yml
 ---
+
+- name: nginx | create nginx app dirs
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ nginx_user }}"
+    group="{{ common_web_group }}"
+  with_items:
+    - "{{ nginx_app_dir }}"
+    - "{{ nginx_sites_available_dir }}"
+    - "{{ nginx_sites_enabled_dir }}"
+  notify: nginx | restart nginx
+
+- name: nginx | create nginx data dirs
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ common_web_user }}"
+    group="{{ nginx_user }}"
+  with_items:
+    - "{{ nginx_data_dir }}"
+    - "{{ nginx_log_dir }}"
+  notify: nginx | restart nginx
+
 - name: nginx | Install nginx
   apt: pkg=nginx state={{ pkgs.nginx.state }}
   notify: nginx | restart nginx
-  tags:
-  - nginx
-  - install
 
 - name: nginx | Server configuration file
-  copy: src={{secure_dir}}/files/nginx.conf dest=/etc/nginx/nginx.conf owner=root group=root mode=0644
-  when: nginx_conf is defined
+  template: >
+    src=nginx.conf.j2 dest=/etc/nginx/nginx.conf
+    owner=root group={{ common_web_user }} mode=0644
   notify: nginx | reload nginx
-  tags:
-  - nginx
-  - install
 
 - name: nginx | Creating common nginx configuration
-  template: src=edx-release.j2 dest=/etc/nginx/sites-available/edx-release owner=root group=root mode=0600
+  template: >
+    src=edx-release.j2 dest={{ nginx_sites_available_dir }}/edx-release
+    owner=root group=root mode=0600
   notify: nginx | reload nginx
-  tags:
-  - nginx
 
 - name: nginx | Creating link for common nginx configuration
-  file: src=/etc/nginx/sites-available/edx-release dest=/etc/nginx/sites-enabled/edx-release state=link owner=root group=root
+  file: >
+    src={{ nginx_sites_available_dir }}/edx-release
+    dest={{ nginx_sites_enabled_dir }}/edx-release
+    state=link owner=root group=root
   notify: nginx | reload nginx
-  tags:
-  - nginx
 
 - name: nginx | Copying nginx configs for {{ nginx_sites }}
-  template: src={{ item }}.j2 dest=/etc/nginx/sites-available/{{ item }} owner=root group=root mode=0600
+  template: >
+    src={{ item }}.j2 dest={{ nginx_sites_available_dir }}/{{ item }}
+    owner=root group={{ common_web_user }} mode=0640
   notify: nginx | reload nginx
   with_items: nginx_sites
-  tags:
-  - nginx
 
 - name: nginx | Creating nginx config links for {{ nginx_sites }}
-  file: src=/etc/nginx/sites-available/{{ item  }} dest=/etc/nginx/sites-enabled/{{ item }} state=link owner=root group=root
+  file: >
+    src={{ nginx_sites_available_dir }}/{{ item  }}
+    dest={{ nginx_sites_enabled_dir }}/{{ item }}
+    state=link owner=root group=root
   notify: nginx | reload nginx
   with_items: nginx_sites
-  tags:
-  - nginx
 
 - name: nginx | Write out default htpasswd file
-  copy: content={{ nginx_cfg.htpasswd }} dest=/etc/nginx/nginx.htpasswd owner=www-data group=www-data mode=0600
-  tags:
-  - nginx
-  - update
+  copy: >
+    content={{ nginx_cfg.htpasswd }} dest={{ nginx_app_dir }}/nginx.htpasswd
+    owner=www-data group=www-data mode=0600
 
 - name: nginx | Create nginx log file location (just in case)
-  file: path={{log_base_dir}}/nginx state=directory owner=syslog group=syslog mode=2770 recurse=yes
-  tags:
-  - nginx
-  - logging
-  - update
+  file: >
+    path={{ nginx_log_dir}} state=directory
+    owner={{ common_web_user }} group={{ common_web_user }}
 
 # removing default link
 - name: nginx | Removing default nginx config and restart (enabled)
-  file: path=/etc/nginx/sites-enabled/default state=absent
+  file: path={{ nginx_sites_enabled_dir }}/default state=absent
   notify: nginx | reload nginx
-  tags:
-  - nginx
-  - update
 
 # Note that nginx logs to /var/log until it reads its configuration, so /etc/logrotate.d/nginx is still good
 
 - name: nginx | Set up nginx access log rotation
-  template: dest=/etc/logrotate.d/nginx-access src=edx_logrotate_nginx_access.j2 owner=root group=root mode=644
-  tags:
-  - logging
-  - update
+  template: >
+    dest=/etc/logrotate.d/nginx-access src=edx_logrotate_nginx_access.j2
+    owner=root group=root mode=644
 
 - name: nginx | Set up nginx access log rotation
-  template: dest=/etc/logrotate.d/nginx-error src=edx_logrotate_nginx_error.j2 owner=root group=root mode=644
-  tags:
-  - logging
-  - update
-
-- name: nginx | Removing default nginx config (available)
-  file: path=/etc/nginx/sites-available/default state=absent
-  notify: nginx | reload nginx
-  tags:
-  - nginx
-  - update
+  template: >
+    dest=/etc/logrotate.d/nginx-error src=edx_logrotate_nginx_error.j2
+    owner=root group=root mode=644
 
 # If tasks that notify restart nginx don't change the state of the remote system
 # their corresponding notifications don't get run.  If nginx has been stopped for

playbooks/roles/nginx/templates/basic-auth.j2

      auth_basic            "Restricted";
-     auth_basic_user_file  /etc/nginx/nginx.htpasswd;
-     root {{ app_base_dir }}/main_static;
+     auth_basic_user_file  {{ nginx_app_dir }}/nginx.htpasswd;
      index index.html
      proxy_set_header X-Forwarded-Proto https;

playbooks/roles/nginx/templates/cms.j2

@@ -11,8 +11,8 @@ server {
 
   server_name studio.*;
 
-  access_log {{log_base_dir}}/nginx/access.log;
-  error_log {{log_base_dir}}/nginx/error.log error;
+  access_log {{ nginx_log_dir }}/access.log;
+  error_log {{ nginx_log_dir }}/error.log error;
 
   # CS184 requires uploads of up to 4MB for submitting screenshots. 
   # CMS requires larger value for course assest, values provided 
@@ -50,7 +50,7 @@ server {
 
   # Check security on this
   location ~ /static/(?P<file>.*) {
-    root {{app_base_dir}};
+    root {{ edxapp_data_dir }};
     try_files /staticfiles/$file /course_static/$file =404;
 
     # return a 403 for static files that shouldn't be

playbooks/roles/nginx/templates/discern.j2

@@ -9,7 +9,7 @@ server {
 
     # https://docs.djangoproject.com/en/dev/howto/static-files/#serving-static-files-in-production
     location /static/ { # STATIC_URL
-        alias {{ discern_dir }}/staticfiles/;
+        alias {{ discern_app_dir }}/staticfiles/;
         expires 1m;
 	autoindex on;
     }

playbooks/roles/nginx/templates/edx_logrotate_nginx_access.j2

 # Put in place by ansible
 
-{{log_base_dir}}/nginx/access.log {
+{{ nginx_log_dir }}/access.log {
   create 0640 www-data adm
   compress
   delaycompress

playbooks/roles/nginx/templates/edx_logrotate_nginx_error.j2

 # Put in place by ansible
 
-{{log_base_dir}}/nginx/error.log {
+{{ nginx_log_dir }}/error.log {
   create 0640 www-data adm
   compress
   delaycompress

playbooks/roles/nginx/templates/lms-preview.j2

@@ -48,7 +48,7 @@ server {
 
   # Check security on this
   location ~ /static/(?P<file>.*) {
-    root {{app_base_dir}};
+    root {{ edxapp_data_dir}};
     try_files /staticfiles/$file /course_static/$file =404;
 
     # return a 403 for static files that shouldn't be

playbooks/roles/nginx/templates/lms.j2

@@ -9,8 +9,8 @@ server {
 
   listen {{EDXAPP_LMS_NGINX_PORT}} default;
 
-  access_log {{log_base_dir}}/nginx/access.log;
-  error_log {{log_base_dir}}/nginx/error.log error;
+  access_log {{ nginx_log_dir }}/access.log;
+  error_log {{ nginx_log_dir }}/error.log error;
 
   # CS184 requires uploads of up to 4MB for submitting screenshots.
   # CMS requires larger value for course assest, values provided
@@ -50,7 +50,7 @@ server {
 
   # Check security on this
   location ~ /static/(?P<file>.*) {
-    root {{app_base_dir}};
+    root {{ edxapp_data_dir }};
     try_files /staticfiles/$file /course_static/$file =404;
 
     # return a 403 for static files that shouldn't be

playbooks/roles/nginx/templates/nginx.conf.j2

+user www-data;
+worker_processes 4;
+pid /var/run/nginx.pid;
+
+events {
+        worker_connections 768;
+        # multi_accept on;
+}
+
+http {
+
+        ##
+        # Basic Settings
+        ##
+
+        sendfile on;
+        tcp_nopush on;
+        tcp_nodelay on;
+        keepalive_timeout 65;
+        types_hash_max_size 2048;
+        # increase header buffer for for https://edx-wiki.atlassian.net/browse/LMS-467&gt
+        # see http://orensol.com/2009/01/18/nginx-and-weird-400-bad-request-responses/
+        large_client_header_buffers 4 16k;
+        # server_tokens off;
+
+        # server_names_hash_bucket_size 64;
+        # server_name_in_redirect off;
+
+        include /etc/nginx/mime.types;
+        default_type application/octet-stream;
+
+        ##
+        # Logging Settings
+        ##
+
+        log_format p_combined '$http_x_forwarded_for - $remote_addr - $remote_user [$time_local]  '
+                            '"$request" $status $body_bytes_sent $request_time '
+                            '"$http_referer" "$http_user_agent"';
+        access_log {{ nginx_log_dir }}/access.log p_combined;
+        error_log {{ nginx_log_dir }}/error.log;
+
+        ##
+        # Gzip Settings
+        ##
+
+        gzip on;
+        gzip_disable "msie6";
+
+        gzip_vary on;
+        gzip_proxied any;
+        gzip_comp_level 6;
+        gzip_buffers 16 8k;
+        gzip_http_version 1.1;
+        gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+        ##
+        # Virtual Host Configs
+        ##
+
+        include {{ nginx_sites_enabled_dir }}/*;
+        include {{ nginx_conf_dir }}/*.conf;
+}
+

playbooks/roles/notifier/defaults/main.yml

@@ -38,7 +38,7 @@ NOTIFIER_USER_SERVICE_HTTP_AUTH_USER: "guido"
 NOTIFIER_USER_SERVICE_HTTP_AUTH_PASS: "vanrossum"
 NOTIFIER_CELERY_BROKER_URL: "django://"
 
-NOTIFIER_SUPERVISOR_LOG_DEST: "{{ storage_base_dir }}/logs/supervisor"
+NOTIFIER_SUPERVISOR_LOG_DEST: "{{ COMMON_DATA_DIR }}/logs/supervisor"
 
 NOTIFER_REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
 

playbooks/roles/notifier/tasks/main.yml

@@ -2,16 +2,16 @@
 
 #
 # notifier
-# 
+#
 # Overview:
-# 
-# Provides the edX notifier service, a service for sending 
+#
+# Provides the edX notifier service, a service for sending
 # notifications over messaging protocols.
 #
 # Dependencies:
 #
 #   * common
-# 
+#
 # Example play:
 #   roles:
 #   - common
@@ -19,153 +19,84 @@
 #
 - name: notifier | install notifier specific system packages
   apt: pkg={{','.join(notifier_debian_pkgs)}} state=present
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | check if incommon ca is installed
   command: test -e /usr/share/ca-certificates/incommon/InCommonServerCA.crt
   register: incommon_present
   ignore_errors: yes
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: common | create incommon ca directory
-  file: 
+  file:
     path="/usr/share/ca-certificates/incommon" mode=2775 state=directory
   when: incommon_present|failed
-  tags:
-  - notifier
-  - install
-  - update
-  - ubuntu
 
 - name: common | retrieve incommon server CA
   shell: curl https://www.incommon.org/cert/repository/InCommonServerCA.txt -o /usr/share/ca-certificates/incommon/InCommonServerCA.crt
   when: incommon_present|failed
-  tags:
-  - notifier
-  - install
-  - update
-  - ubuntu
 
 - name: common | add InCommon ca cert
   lineinfile:
     dest=/etc/ca-certificates.conf
-    regexp='incommon/InCommonServerCA.crt'    
+    regexp='incommon/InCommonServerCA.crt'
     line='incommon/InCommonServerCA.crt'
-  tags:
-  - notifier
-  - install
-  - update
-  - ubuntu
 
 - name: common | update ca certs globally
   shell: update-ca-certificates
-  tags:
-  - notifier
-  - install
-  - update
-  - ubuntu
 
 - name: notifier | create notifier user {{ NOTIFIER_USER }}
-  user: 
-    name={{ NOTIFIER_USER }} state=present shell=/bin/bash 
+  user:
+    name={{ NOTIFIER_USER }} state=present shell=/bin/bash
     home={{ NOTIFIER_HOME }} createhome=yes
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | setup the notifier env
-  template: 
-    src=notifier_env.j2 dest={{ NOTIFIER_HOME }}/notifier_env 
+  template:
+    src=notifier_env.j2 dest={{ NOTIFIER_HOME }}/notifier_env
     owner="{{ NOTIFIER_USER }}" group="{{ NOTIFIER_USER }}"
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | drop a bash_profile
   copy: >
-    src=../../common/files/bash_profile 
-    dest={{ NOTIFIER_HOME }}/.bash_profile 
-    owner={{ NOTIFIER_USER }} 
+    src=../../common/files/bash_profile
+    dest={{ NOTIFIER_HOME }}/.bash_profile
+    owner={{ NOTIFIER_USER }}
     group={{ NOTIFIER_USER }}
 
 - name: notifier | ensure .bashrc exists
   shell: touch {{ NOTIFIER_HOME }}/.bashrc
-  sudo: true  
+  sudo: true
   sudo_user: "{{ NOTIFIER_USER }}"
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | add source of notifier_env to .bashrc
   lineinfile:
     dest={{ NOTIFIER_HOME }}/.bashrc
-    regexp='. {{ NOTIFIER_HOME }}/notifier_env'    
+    regexp='. {{ NOTIFIER_HOME }}/notifier_env'
     line='. {{ NOTIFIER_HOME }}/notifier_env'
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | add source venv to .bashrc
   lineinfile:
     dest={{ NOTIFIER_HOME }}/.bashrc
-    regexp='. {{ NOTIFIER_VENV_DIR }}/bin/activate'    
+    regexp='. {{ NOTIFIER_VENV_DIR }}/bin/activate'
     line='. {{ NOTIFIER_VENV_DIR }}/bin/activate'
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | create notifier DB directory
   file:
     path="{{ NOTIFIER_DB_DIR }}" mode=2775 state=directory
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | create notifier/bin directory
-  file: 
+  file:
     path="{{ NOTIFIER_HOME }}/bin" mode=2775 state=directory
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: common | create supervisor log directoy
-  file: 
+  file:
     path={{NOTIFIER_SUPERVISOR_LOG_DEST }} mode=2750 state=directory
-  tags:
-  - notifier
-  - install
-  - update
-  - ubuntu
 
 - name: notifier | supervisord config for celery workers
-  template: 
-    src=etc/supervisor/conf.d/notifier-celery-workers.conf.j2 dest=/etc/supervisor/conf.d/notifier-celery-workers.conf 
+  template:
+    src=etc/supervisor/conf.d/notifier-celery-workers.conf.j2 dest=/etc/supervisor/conf.d/notifier-celery-workers.conf
   notify: notifier | restart notifier-celery-workers
-  tags:
-  - notifier
-  - install
-  - update
 
 - name: notifier | supervisord config for scheduler
-  template: 
-    src=etc/supervisor/conf.d/notifier-scheduler.conf.j2 dest=/etc/supervisor/conf.d/notifier-scheduler.conf 
+  template:
+    src=etc/supervisor/conf.d/notifier-scheduler.conf.j2 dest=/etc/supervisor/conf.d/notifier-scheduler.conf
   notify: notifier | restart notifier-scheduler
-  tags:
-  - notifier
-  - install
-  - update
 
 - include: deploy.yml

playbooks/roles/ora/defaults/main.yml

@@ -3,14 +3,31 @@
 ORA_NGINX_PORT: 18060
 ORA_BASIC_AUTH: False
 
-ora_code_dir: "{{ app_base_dir }}/edx-ora"
+ora_app_dir: "{{ COMMON_APP_DIR }}/ora"
+ora_code_dir: "{{ ora_app_dir }}/ora"
+ora_data_dir: "{{ COMMON_DATA_DIR }}/ora"
+ora_venvs_dir: "{{ ora_app_dir }}/venvs"
+ora_venv_dir: "{{ ora_venvs_dir }}/ora"
+ora_venv_bin: "{{ ora_venv_dir }}/bin"
+ora_user: "ora"
+ora_nltk_data_dir: "{{ ora_data_dir}}/nltk_data"
+
+ora_source_repo: https://github.com/edx/edx-ora.git
+ora_version: 'HEAD'
+ora_pre_requirements_file: "{{ ora_code_dir }}/pre-requirements.txt"
+ora_post_requirements_file: "{{ ora_code_dir }}/requirements.txt"
+
+ora_ease_venv_dir: "{{ ora_venv_dir }}"
+ora_ease_code_dir: "{{ ora_app_dir }}/ease"
+ora_ease_source_repo: https://github.com/edx/ease.git
+ora_ease_version: 'HEAD'
+ora_ease_pre_requirements_file: "{{ ora_ease_code_dir }}/pre-requirements.txt"
+ora_ease_post_requirements_file: "{{ ora_ease_code_dir }}/requirements.txt"
+
+
 # Default nginx listen port
 # These should be overrided if you want
 # to serve all content on port 80
-ora_user: "edx-ora"
-ora_user_home: "/opt/edx-ora"
-ora_venv_dir: "{{ ora_user_home }}/virtualenvs/{{ ora_user }}"
-ease_venv_dir: "{{ ora_venv_dir }}"
 ora_gunicorn_workers: 4
 ora_gunicorn_port: 8060
 ora_gunicorn_host: 127.0.0.1
@@ -67,18 +84,6 @@ ora_auth_config:
   'AWS_ACCESS_KEY_ID' : ''
   'AWS_SECRET_ACCESS_KEY' : ''
 
-ora_source_repo: https://github.com/edx/edx-ora.git
-ora_version: 'HEAD'
-ora_pre_requirements_file: "{{ ora_code_dir }}/pre-requirements.txt"
-ora_post_requirements_file: "{{ ora_code_dir }}/requirements.txt"
-
-ease_code_dir: "{{ app_base_dir }}/ease"
-ease_source_repo: https://github.com/edx/ease.git
-ease_version: 'HEAD'
-ease_pre_requirements_file: "{{ ease_code_dir }}/pre-requirements.txt"
-ease_post_requirements_file: "{{ ease_code_dir }}/requirements.txt"
-nltk_data_dir: /usr/share/nltk_data
-
 ora_debian_pkgs:
   - python-software-properties
   - pkg-config
@@ -107,7 +112,7 @@ ora_debian_pkgs:
   - libatlas-base-dev
   - redis-server
 
-ease_debian_pkgs:
+ora_ease_debian_pkgs:
   - python-pip
   - gcc
   - g++