Merge remote-tracking branch 'origin/master' into fix/jbau/edxapp_instead_of_lms_cms_in_production

Conflicts:
	playbooks/edx-west/edxapp_prod.yml

playbooks/edx-west/edxapp_prod.yml

+# this gets all running prod webservers
 #- hosts: tag_environment_prod:&tag_function_webserver
-- hosts: ~tag_Name_app(test)_prod
-#- hosts: tag_environment_prod:&tag_function_webserver
+# or we can get subsets of them by name
+- hosts: ~tag_Name_app(10|20)_prod
+#- hosts: ~tag_Name_app(11|21)_prod
+## these are cold hosts:
+#- hosts: ~tag_Name_app(12|22)_prod
+## this is the test box
+#- hosts: ~tag_Name_app4_prod
+## you can also do security group, but don't do that
 #- hosts: security_group_edx-prod-EdxappServerSecurityGroup-NSKCQTMZIPQB
   sudo: True
-  vars_files:
-    - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
-    - "{{ secure_dir }}/vars/users.yml"
-    - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
-    - "{{ secure_dir }}/vars/shib_prod_vars.yml"
-
   vars:
     secure_dir: '../../../configuration-secure/ansible'
     # this indicates the path to site-specific (with precedence)
     # things like nginx template files
     local_dir:  '../../../configuration-secure/ansible/local'
-
+  vars_files:
+    - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
+    - "{{ secure_dir }}/vars/shib_prod_vars.yml"
   roles:
     - common
     - nginx

playbooks/edx_sandbox.yml

@@ -27,6 +27,5 @@
   roles:
     - common
     - nginx
-    - gunicorn
     - edxlocal
     - edxapp

playbooks/group_vars/all

 ---
+# This should only have variables
+# that are applicable to all edX roles
+
 app_base_dir: /opt/wwc
 log_base_dir: /mnt/logs
 venv_dir: /opt/edx
-platform_code_dir: $app_base_dir/edx-platform
 os_name: ubuntu
 
 # these pathes are relative to the playbook dir

playbooks/roles/common/templates/edx-update.sh.j2

@@ -46,11 +46,10 @@ source /etc/profile
 source {{venv_dir}}/bin/activate
 export PATH=$PATH:/opt/www/.gem/bin
 
-cd {{platform_code_dir}}
+cd {{edx_platform_code_dir}}
 
 BRANCH="origin/feature/edx-west/stanford-theme"
 
-export GIT_SSH="/tmp/git_ssh.sh"
 run git fetch origin -p
 run git checkout $BRANCH
 

playbooks/roles/discern/tasks/main.yml

@@ -40,10 +40,6 @@
 - name: change memory commit settings -- needed for redis
   command: sysctl vm.overcommit_memory=1
 
-#Upload custom git ssh script
-- name: upload ssh script
-  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=adm mode=750
-
 - name: set permissions on app_base_dir sgid for edx
   file: path={{app_base_dir}} owner=root group=edx mode=2775 state=directory
   file: path={{venv_dir}} owner=root group=edx mode=2775 state=directory
@@ -61,16 +57,12 @@
 #Grab both repos or update
 - name: git checkout discern repo into discern_dir
   git: dest={{discern_dir}} repo={{discern_source_repo}} version={{discern_branch}}
-  environment:
-    GIT_SSH: /tmp/git_ssh.sh
   notify:
     - restart celery
     - restart discern
 
 - name: git checkout ease repo into ease_dir
   git: dest={{ease_dir}} repo={{ease_source_repo}} version={{ease_branch}}
-  environment:
-    GIT_SSH: /tmp/git_ssh.sh
   notify:
     - restart celery
     - restart discern

playbooks/roles/discern/vars/main.yml

-discern_source_repo: git@github.com:edx/discern.git
-ease_source_repo: git@github.com:edx/ease.git
+discern_source_repo: https://github.com/edx/discern.git
+ease_source_repo: https://github.com/edx/ease.git
 ease_dir: $app_base_dir/ease
 discern_dir: $app_base_dir/discern
 discern_settings: discern.aws

playbooks/roles/edxapp/files/git_ssh.sh

-#!/bin/sh
-exec /usr/bin/ssh -o StrictHostKeyChecking=no -i /etc/git-identity "$@"

playbooks/roles/edxapp/handlers/main.yml

@@ -32,14 +32,14 @@
 
 # Ruby plays that need to be run after platform updates.
 - name: gem | gem install bundler
-  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ rbenv_root }}/shims/gem install bundle chdir={{ platform_code_dir }}
+  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ rbenv_root }}/shims/gem install bundle chdir={{ edx_platform_code_dir }}
   tags:
   - ruby
   - deploy
   - install
 
 - name: bundle | bundle install
-  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ gem_home }}/bin/bundle install --binstubs chdir={{ platform_code_dir }}
+  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ gem_home }}/bin/bundle install --binstubs chdir={{ edx_platform_code_dir }}
   tags:
   - ruby
   - deploy
@@ -47,7 +47,7 @@
 
 # Node play that need to be run after platform updates.
 - name: Install edx-platform npm dependencies
-  shell: npm install chdir={{ platform_code_dir }}
+  shell: npm install chdir={{ edx_platform_code_dir }}
   tags:
   - npm
   - update
@@ -69,7 +69,7 @@
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ base_requirements_file }}
+  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ base_requirements_file }}
   tags:
   - lms
   - cms
@@ -90,7 +90,7 @@
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ item }}
+  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ item }}
   with_items:
   - "{{ repo_requirements_file }}"
   - "{{ github_requirements_file }}"
@@ -103,7 +103,7 @@
 
 # This check needs to be run to see if rake can be used but its failure should not stop the run.
 - name: check if rake gather_assets is available
-  shell: executable=/bin/bash chdir={{ platform_code_dir }} rake -T | grep gather_assets
+  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} rake -T | grep gather_assets
   environment: "{{ deploy_environment }}"
   register: grep_gather_assets
   ignore_errors: yes
@@ -115,7 +115,7 @@
   - deploy
 
 - name: check if django can collect lms static data
-  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws
   register: check_lms_collect_static
   sudo: yes
   sudo_user: www-data
@@ -127,7 +127,7 @@
   - deploy
 
 - name: check if django can collect cms static data
-  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
   register: check_cms_collect_static
   sudo: yes
   sudo_user: www-data
@@ -137,7 +137,7 @@
   - deploy
 
 - name: check if django can update cms templates
-  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py help update_templates --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py help update_templates --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
   register: check_cms_update_templates
   sudo: yes
   sudo_user: www-data
@@ -148,7 +148,7 @@
 
 # Gather lms assets using rake if possible
 - name: gather lms static assets with rake
-  shell: executable=/bin/bash chdir={{ platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws
+  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws
   when: grep_gather_assets.rc == 0
   notify:
   - stop edxapp
@@ -164,7 +164,7 @@
 
 # Gather lms assets using django if necessary(When rake doesn't know how)
 - name: gather lms static assets with django
-  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
   when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc == 0
   notify:
   - stop edxapp
@@ -182,7 +182,7 @@
 # Gather cms assets using rake if possible
 - name: gather cms static assets with rake
 #  script: gather_assets.sh
-  shell: executable=/bin/bash chdir={{ platform_code_dir }} SERVICE_VARIANT={{ cms_variant }} rake cms:gather_assets:aws
+  shell: executable=/bin/bash chdir={{ edx_platform_code_dir }} SERVICE_VARIANT={{ cms_variant }} rake cms:gather_assets:aws
   when: grep_gather_assets.rc == 0
   notify:
   - stop edxapp
@@ -195,7 +195,7 @@
   - deploy
 
 - name: gather cms static assets with django
-  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py collectstatic --pythonpath={{ edx_platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
   when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc == 0
   notify:
   - stop edxapp
@@ -208,7 +208,7 @@
   - deploy
 
 - name: update cms templates
-  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py update_templates --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py update_templates --pythonpath={{ edx_platform_code_dir }} --settings=cms.envs.aws
   when: check_cms_update_templates.rc == 0
   notify:
   - stop edxapp

playbooks/roles/edxapp/tasks/deploy.yml

@@ -10,9 +10,7 @@
 
 # Do A Checkout
 - name: git checkout edx-platform repo into $app_base_dir
-  git: dest={{platform_code_dir}} repo={{lms_source_repo}} version={{lms_version}}
-  environment:
-    GIT_SSH: /tmp/git_ssh.sh
+  git: dest={{edx_platform_code_dir}} repo={{lms_source_repo}} version={{lms_version}}
   tags:
   - lms
   - cms
@@ -21,7 +19,7 @@
 
 # Do Post Checkout Tasks.
 - name: create platform code dir
-  file: path={{platform_code_dir}} state=directory owner=www-data group=www-data mode=755
+  file: path={{edx_platform_code_dir}} state=directory owner=www-data group=www-data mode=755
   tags:
   - lms
   - cms
@@ -43,8 +41,8 @@
 # portions of the deploy needs to be incorporated here.
 
 - name: sets permissions on platform code dir and contents
-  file: path={{platform_code_dir}} state=directory owner=www-data group=www-data recurse=yes
-  # Post Checkout tasks will get run as handlers when the {{ platform_code_dir }} is ready.
+  file: path={{edx_platform_code_dir}} state=directory owner=www-data group=www-data recurse=yes
+  # Post Checkout tasks will get run as handlers when the {{ edx_platform_code_dir }} is ready.
   # Look at the handlers/main.yml in this role for a description of the tasks stated below.
   tags:
   - lms

playbooks/roles/edxapp/tasks/main.yml

@@ -28,24 +28,6 @@
   - lms-env
   - update
 
-# Install ssh keys for ubuntu account to be able to check out from edx-platform
-# Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
-# {{ secure_dir }} is relative to the top-level playbooks dir so there is some
-
-- name: install read-only ssh key for edx-platform repo (private)
-  copy: src={{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=ubuntu group=adm mode=600
-  tags:
-  - lms
-  - cms
-  - update
-
-- name: upload ssh script
-  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=adm mode=750
-  tags:
-  - lms
-  - cms
-  - update
-
 # Check out edx-platform repo to $app_base_dir
 - name: install git and its recommends
   apt: pkg=git state=present install_recommends=yes 

playbooks/roles/edxapp/tasks/ruby.yml

@@ -29,7 +29,7 @@
   - install
 
 - name: rbenv | update rbenv repo
-  git: repo=git://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
+  git: repo=https://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
   tags:
   - ruby
   - install
@@ -63,7 +63,7 @@
   - install
 
 - name: rbenv | clone ruby-build repo
-  git: repo=git://github.com/sstephenson/ruby-build.git dest=${tempdir.stdout}/ruby-build
+  git: repo=https://github.com/sstephenson/ruby-build.git dest=${tempdir.stdout}/ruby-build
   when_failed: $rbuild_present
   tags:
   - ruby

playbooks/roles/edxapp/templates/cms.conf.j2

@@ -23,7 +23,7 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=cms.envs.aws
 env SERVICE_VARIANT="cms"
  
-chdir {{platform_code_dir}}
+chdir {{edx_platform_code_dir}}
 setuid www-data
  
-exec {{venv_dir}}/bin/gunicorn_django -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} --settings=cms.envs.aws
+exec {{venv_dir}}/bin/gunicorn_django -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}} --settings=cms.envs.aws

playbooks/roles/edxapp/templates/lms-preview.conf.j2

@@ -22,10 +22,10 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms-preview"
   
-chdir {{platform_code_dir}}
+chdir {{edx_platform_code_dir}}
 setuid www-data
 
-exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} lms.wsgi
+exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}} lms.wsgi
 
 post-start script
   while true

playbooks/roles/edxapp/templates/lms-xml.conf.j2

@@ -22,10 +22,10 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms-xml"
 
-chdir {{platform_code_dir}}
+chdir {{edx_platform_code_dir}}
 setuid www-data
 
-exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}}  lms.wsgi
+exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}}  lms.wsgi
 
 post-start script
   while true

playbooks/roles/edxapp/templates/lms.conf.j2

@@ -22,10 +22,10 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms"
 
-chdir {{platform_code_dir}}
+chdir {{edx_platform_code_dir}}
 setuid www-data
 
-exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} lms.wsgi
+exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{edx_platform_code_dir}} lms.wsgi
 
 post-start script
   while true

playbooks/roles/edxapp/vars/main.yml

@@ -2,6 +2,9 @@
 # when the role is included
 ---
 
+# install dir for the edx-platform repo
+edx_platform_code_dir: "{{ app_base_dir }}/edx-platform"
+
 # Default nginx listen ports
 # These should be overrided if you want
 # to serve all content on port 80
@@ -45,18 +48,18 @@ worker_core_mult:
 #To turn off theming, specify edxapp_theme_name: ''
 #Stanford, for example, uses edxapp_theme_name: 'stanford'
 edxapp_theme_name: ''
-edxapp_theme_source_repo: 'git://github.com/Stanford-Online/edx-theme.git'
+edxapp_theme_source_repo: 'https://github.com/Stanford-Online/edx-theme.git'
 edxapp_theme_version: 'HEAD'
 
 # make this the public URL instead of writable
-lms_source_repo: git://github.com/edx/edx-platform.git
+lms_source_repo: https://github.com/edx/edx-platform.git
 lms_version: 'HEAD'
-local_requirements_file:  "{{ platform_code_dir }}/requirements/edx/local.txt"
-pre_requirements_file:    "{{ platform_code_dir }}/requirements/edx/pre.txt"
-post_requirements_file:   "{{ platform_code_dir }}/requirements/edx/post.txt"
-base_requirements_file:   "{{ platform_code_dir }}/requirements/edx/base.txt"
-github_requirements_file: "{{ platform_code_dir }}/requirements/edx/github.txt"
-repo_requirements_file:   "{{ platform_code_dir }}/requirements/edx/repo.txt"
+local_requirements_file:  "{{ edx_platform_code_dir }}/requirements/edx/local.txt"
+pre_requirements_file:    "{{ edx_platform_code_dir }}/requirements/edx/pre.txt"
+post_requirements_file:   "{{ edx_platform_code_dir }}/requirements/edx/post.txt"
+base_requirements_file:   "{{ edx_platform_code_dir }}/requirements/edx/base.txt"
+github_requirements_file: "{{ edx_platform_code_dir }}/requirements/edx/github.txt"
+repo_requirements_file:   "{{ edx_platform_code_dir }}/requirements/edx/repo.txt"
 
 lms_debian_pkgs:
   - apparmor-utils
@@ -147,4 +150,4 @@ deploy_environment:
   SKIP_WS_MIGRATIONS: 1
   RBENV_ROOT: "{{ rbenv_root }}"
   GEM_HOME: "{{ gem_home }}"
-  PATH: "{{ venv_dir }}/bin:{{ platform_code_dir }}/bin:{{ rbenv_root }}/bin:{{ rbenv_root }}/shims:{{ gem_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"
+  PATH: "{{ venv_dir }}/bin:{{ edx_platform_code_dir }}/bin:{{ rbenv_root }}/bin:{{ rbenv_root }}/shims:{{ gem_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"

playbooks/roles/edxlocal/tasks/main.yml

@@ -13,12 +13,19 @@
 - name: install mysql server and recommends
   apt: pkg=mysql-server-5.5 state=present install_recommends=yes 
 
-- name: create a database
+- name: create a database for edxapp
   mysql_db: >
     db=wwc
     state=present
     encoding=utf8
 
+- name: create a database for xqueue
+  mysql_db: >
+    db=xqueue
+    state=present
+    encoding=utf8
+
+
 - name: install mongo server and recommends  
   apt: pkg=mongodb-server state=present install_recommends=yes
 

playbooks/roles/nginx/templates/xqueue.j2

+upstream app_server {
+    # For a TCP configuration:
+    server 127.0.0.1:{{ xqueue.gunicorn_port }} fail_timeout=0;
+}
+
+server {
+  listen {{ xqueue.nginx_port }} default_server;
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://app_server;
+  }
+}
+

playbooks/roles/xqueue/handlers/main.yml

+---
+- name: invoke all deploy handlers
+  shell: echo "running deploy handlers"
+  notify:
+  - install python pre-requirements
+  - install python post-requirements
+  - syncdb and migrate
+  - stop xqueue
+  - start xqueue
+  tags:
+  - xqueue
+  - deploy
+
+
+
+# Install the python pre requirements into {{ venv_dir }}
+- name : install python pre-requirements
+  pip: requirements="{{xqueue_pre_requirements_file}}" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - xqueue
+  - deploy
+
+# Install the python post requirements into {{ venv_dir }}
+- name : install python post-requirements
+  pip: requirements="{{xqueue_post_requirements_file}}" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - xqueue
+  - deploy
+
+- name: syncdb and migrate
+  shell: sudo -u www-data /opt/edx/bin/django-admin.py syncdb --migrate --noinput --settings=xqueue_aws_settings --pythonpath=/opt/wwc/xqueue
+  when: migrate_db is defined
+  tags:
+  - xqueue
+  - syncdb
+  - deploy
+
+- name: stop xqueue
+  service: name=xqueue state=stopped
+  tags:
+  - xqueue
+  - deploy
+
+- name: start xqueue
+  service: name=xqueue state=started
+  tags:
+  - xqueue
+  - deploy
+

playbooks/roles/xqueue/tasks/deploy.yml

+# Stop xqueue service.
+- name: stop xqueue service
+  service: name=xqueue state=stopped
+  tags:
+  - xqueue
+  - deploy
+
+# Do A Checkout
+- name: git checkout xqueue repo into $app_base_dir
+  git: dest={{xqueue_code_dir}} repo={{xqueue_source_repo}} version={{xqueue_version}}
+  tags:
+  - xqueue
+  - deploy
+
+# Do Post Checkout Tasks.
+- name: create xqueue code dir
+  file: path={{xqueue_code_dir}} state=directory owner=www-data group=www-data mode=755
+  tags:
+  - xqueue
+  - deploy
+
+
+# TODO: Check git.py _run_if_changed() to see if the logic there to skip running certain
+# portions of the deploy needs to be incorporated here.
+
+- name: sets permissions on xqueue code dir and contents
+  file: path={{xqueue_code_dir}} state=directory owner=www-data group=www-data recurse=yes
+  # Post Checkout tasks will get run as handlers when the {{ xqueue_code_dir }} is ready.
+  # Look at the handlers/main.yml in this role for a description of the tasks stated below.
+  tags:
+  - xqueue
+  - deploy
+
+# want deploy handlers to always run when deploy tag is specified, so move into a no op task
+- name: run deploy handlers
+  shell: echo "running deploy handlers"
+  notify:
+  - invoke all deploy handlers
+  tags:
+  - xqueue
+  - deploy

playbooks/roles/xqueue/tasks/main.yml

+# requires:
+#  - group_vars/all
+#  - common/tasks/main.yml
+#  - nginx/tasks/main.yml
+---
+- name: Change permissions on datadir
+  file: path={{ app_base_dir }}/data state=directory owner=www-data group=www-data
+  tags:
+  - xqueue
+
+# Check out xqueue repo to $app_base_dir
+- name: install git and its recommends
+  apt: pkg=git state=present install_recommends=yes 
+  tags:
+  - xqueue
+
+- name: install a bunch of system packages on which LMS and CMS rely
+  apt: pkg={{item}} state=present
+  with_items: xqueue_debian_pkgs
+  tags:
+  - xqueue
+
+- name: create xqueue application config
+  template: src=xqueue_env.json.j2 dest=$app_base_dir/env.json mode=640 owner=www-data group=adm
+  tags:
+  - xqueue
+
+- name: create xqueue auth file
+  template: src=xqueue_auth.json.j2 dest=$app_base_dir/auth.json mode=640 owner=www-data group=adm
+  tags:
+  - xqueue
+
+- name: creating xqueue upstart script
+  sudo: True
+  template: src=xqueue_conf.j2 dest=/etc/init/xqueue_conf owner=root group=root
+  tags:
+  - xqueue
+
+- include: deploy.yml

playbooks/roles/xqueue/templates/xqueue.auth.json.j2

+{{ xqueue_auth_config | to_nice_json }}

playbooks/roles/xqueue/templates/xqueue.conf.j2

+#/etc/init/xqueue_conf
+
+description "xqueue server"
+author "edX <info@edx.org>"
+
+respawn
+respawn limit 3 30
+
+env PID=/var/tmp/xqueue_pid
+env WORKERS={{ ansible_processor_cores * 2 }}
+env PORT={{ xqueue_gunicorn_port }}
+env LANG=en_US.UTF-8
+env DJANGO_SETTINGS_MODULE=xqueue_aws_settings
+env SERVICE_VARIANT="xqueue"
+
+
+chdir {{ xqueue_code_dir }}
+setuid www-data
+
+exec {{ venv_dir }}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{ xqueue_code_dir }} xqueue_wsgi

playbooks/roles/xqueue/templates/xqueue.env.json.j2

+{{ xqueue_env_config | to_nice_json }}

playbooks/roles/xqueue/vars/main.yml

+# variables common to the xqueue role, automatically loaded
+# when the role is included
+---
+xqueue_code_dir: "{{ app_base_dir }}/xqueue"
+
+# Default nginx listen port
+# These should be overrided if you want
+# to serve all content on port 80
+xqueue_code_dir: "{{ app_base_dir }}/xqueue"
+xqueue_nginx_port: 18040
+xqueue_gunicorn_port: 8040
+
+
+xqueue_auth_config: {}
+xqueue_env_config: {}
+
+xqueue_source_repo: https://github.com/edx/xqueue_git
+xqueue_version: 'HEAD'
+xqueue_pre_requirements_file:    "{{ xqueue_code_dir }}/pre-requirements.txt"
+xqueue_post_requirements_file:   "{{ xqueue_code_dir }}/requirements.txt"
+
+
+# These packages are required for the xqueue server,
+# copied from the LMS role for now since there is a lot
+# of overlap
+xqueue_debian_pkgs:
+  - apparmor-utils
+  - aspell
+  - build-essential
+  - curl
+  - dvipng
+  - fabric
+  - facter
+  - g++
+  - gcc
+  - gfortran
+  - ghostscript
+  - git
+  - github-cli
+  - graphviz
+  - graphviz-dev
+  - gunicorn
+  - inoticoming
+  - ipython
+  - libcrypt-ssleay-perl
+  - libcurl4-openssl-dev
+  - libdigest-sha-perl
+  - libfreetype6-dev
+  - libgeos-dev
+  - libgraphviz-dev
+  - libjpeg8-dev
+  - liblapack-dev
+  - liblwp-protocol-https-perl
+  - libmysqlclient-dev
+  - libnet-amazon-ec2-perl
+  - libpng12-dev
+  - libreadline-dev
+  - libreadline6-dev
+  - libssl-dev
+  - libswitch-perl
+  - libwww-perl
+  - libxml++2.6-dev
+  - libxml2-dev
+  - libxml2-utils
+  - libxslt1-dev
+  - maven2
+  - mongodb
+  - mongodb-clients
+  - mysql-client
+  - npm
+  - ntp
+  - openjdk-7-jdk
+  - openjdk-7-jre
+  - pep8
+  - perl
+  - pkg-config
+  - postfix
+  - pylint
+  - python-boto
+  - python-coverage-test-runner
+  - python-django-nose
+  - python-jenkins
+  - python-nose
+  - python-nosexcover
+  - python-numpy
+  - python-pip
+  - python-scipy
+  - rake
+  - reprepro
+  - rsyslog
+  - rubygems
+  - sqlite3
+  - super
+  - vagrant
+  - yui-compressor
+  - zip
+  - zlib1g-dev

playbooks/secure_example/vars/edxapp_sandbox.yml

@@ -16,15 +16,44 @@
 #  Set the following before using this configuration
 #  configuration.
 #   
-#    * AWS_ACCESS_KEY_ID
-#    * AWS_SECRET_ACCESS_KEY
+#    * AWS_ACCESS_KEY_ID (lms, xqueue)
+#    * AWS_SECRET_ACCESS_KEY (lms, xqueue)
 #    * SESSION_COOKIE_DOMAIN
 #    * FEEDBACK_SUBMISSION_EMAIL
 #
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
 #see http://atechie.net/2009/07/merging-hashes-in-yaml-conf-files/
 
-generic_auth_config:  &generic_auth
+
+xqueue:
+  env_config:
+    'XQUEUES':
+      # push queue
+      - 'edX-DemoX': 'http://localhost:18050'
+      # pull queues
+      - 'test-pull': !!null
+      - 'certificates': !!null
+      - 'open-ended': !!null
+    'XQUEUE_WORKERS_PER_QUEUE': 12
+    'LOGGING_ENV' : 'sandbox'
+    'LOG_DIR' : '/mnt/logs'
+    'SYSLOG_SERVER' : 'syslog.a.m.i4x.org'
+    'RABBIT_HOST' : 'localhost'
+    'S3_BUCKET_PREFIX' : 'sandbox-bucket'
+
+  auth_config:
+    'AWS_ACCESS_KEY_ID' : ''
+    'AWS_SECRET_ACCESS_KEY' : ''
+    'REQUESTS_BASIC_AUTH': ['edx', 'edx']
+    'USERS': {'lms': 'password'}
+    'RABBITMQ_USER': 'edx'
+    'RABBITMQ_PASS': 'edx'
+    'DATABASES':
+      'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'xqueue', 'USER': 'root', 'PASSWORD': '', 'HOST': 'localhost', 'PORT': '3306' }
+
+
+
+edxapp_generic_auth_config:  &edxapp_generic_auth
   'CONTENTSTORE':
     'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
   'ANALYTICS_API_KEY':  ''
@@ -77,7 +106,7 @@ generic_auth_config:  &generic_auth
       'username':  'lms'}
     'url':  ''
  
-generic_env_config:  &generic_env
+edxapp_generic_env_config:  &edxapp_generic_env
   'BOOK_URL':  ''
   'CERT_QUEUE':  'certificates'
   'LOCAL_LOGLEVEL':  'INFO'
@@ -130,18 +159,18 @@ generic_env_config:  &generic_env
   'SESSION_COOKIE_DOMAIN': !!null
   'COMMENTS_SERVICE_KEY':  ''
 lms_auth_config:
-  <<: *generic_auth
+  <<: *edxapp_generic_auth
 lms_env_config:
-  <<: *generic_env
+  <<: *edxapp_generic_env
 lms_xml_auth_config:
-  <<: *generic_auth
+  <<: *edxapp_generic_auth
 lms_xml_env_config:
-  <<: *generic_env
+  <<: *edxapp_generic_env
 cms_auth_config:
-  <<: *generic_auth
+  <<: *edxapp_generic_auth
 cms_env_config:
-  <<: *generic_env
+  <<: *edxapp_generic_env
 lms_preview_auth_config:
-  <<: *generic_auth
+  <<: *edxapp_generic_auth
 lms_preview_env_config:
-  <<: *generic_env
+  <<: *edxapp_generic_env