Working role for review.

playbooks/roles/xqwatcher/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role xqwatcher
+# 
+
+#
+# vars are namespace with the module name.
+#
+xqwatcher_role_name: "xqwatcher"
+xqwatcher_service_name: "xqwatcher"
+xqwatcher_user: "xqwatcher"
+xqwatcher_module: "xqueue_watcher"
+
+xqwatcher_app_dir: "{{ COMMON_APP_DIR }}/{{ xqwatcher_service_name }}"
+xqwatcher_home: "{{ COMMON_APP_DIR }}/{{ xqwatcher_service_name }}"
+xqwatcher_venv_base: "{{ xqwatcher_home }}/venvs"
+xqwatcher_venv_dir: "{{ xqwatcher_venv_base }}/{{ xqwatcher_service_name }}"
+xqwatcher_code_dir: "{{ xqwatcher_app_dir }}/src"
+xqwatcher_conf_dir: "{{ xqwatcher_home }}/conf.d"
+xqwatcher_data_dir: "{{ xqwatcher_home }}/data"
+
+xqwatcher_source_repo: "git@github.com:edx/xqueue-watcher.git"
+xqwatcher_version: "master"
+xqwatcher_git_identity: "{{ xqwatcher_app_dir }}/git-identity"
+
+xqwatcher_requirements_file: "{{ xqwatcher_code_dir }}/requirements.txt"
+xqwatcher_log_dir: "{{ COMMON_LOG_DIR }}/{{ xqwatcher_service_name }}"
+
+#
+# supervisor related config
+#
+xqwatcher_supervisor_app_dir: "{{ xqwatcher_app_dir }}/supervisor"
+xqwatcher_supervisor_data_dir: "{{ COMMON_DATA_DIR }}/{{ xqwatcher_service_name }}"
+xqwatcher_supervisor_log_dir: "{{ xqwatcher_log_dir }}"
+xqwatcher_supervisor_venv_dir: "{{ xqwatcher_venv_base }}/supervisor"
+xqwatcher_supervisor_user: "{{ xqwatcher_user }}"
+xqwatcher_supervisor_venv_bin: "{{ xqwatcher_supervisor_venv_dir }}/bin"
+xqwatcher_supervisor_ctl: "{{ xqwatcher_supervisor_venv_bin }}/supervisorctl"
+xqwatcher_supervisor_cfg_dir: "{{ xqwatcher_supervisor_app_dir }}/conf.d"
+xqwatcher_supervisor_available_dir: "{{ xqwatcher_supervisor_app_dir }}/conf.available.d"
+
+#
+# OS packages
+#
+
+xqwatcher_debian_pkgs:
+  - apparmor-utils
+
+xqwatcher_redhat_pkgs: []
\ No newline at end of file

playbooks/roles/xqwatcher/handlers/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role xqwatcher
+# 
+# Overview:
+# 
+#
+- name: notify me
+  debug: msg="stub handler"

playbooks/roles/xqwatcher/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role xqwatcher
+# 
+# the role name are service name differ by _ and -, the latter isn't safe
+# random corners of ansible/jinga/python variable expansion.
+dependencies:
+  - role: edx_service
+    edx_role_name: "{{ xqwatcher_role_name }}"
+    edx_service_name: "{{ xqwatcher_service_name }}" 
+  - role: supervisor
+    supervisor_app_dir: "{{ xqwatcher_supervisor_app_dir }}"
+    supervisor_data_dir: "{{ xqwatcher_supervisor_data_dir }}"
+    supervisor_log_dir: "{{ xqwatcher_supervisor_log_dir }}"
+    supervisor_venv_dir: "{{ xqwatcher_supervisor_venv_dir }}"
+    supervisor_service_user: "{{ xqwatcher_supervisor_user }}"
+    supervisor_available_dir: "{{ xqwatcher_supervisor_available_dir }}"
+    supervisor_service: "supervisor.xqwatcher"
+    supervisor_http_bind_port: '9003'

playbooks/roles/xqwatcher/tasks/code_jail.yml

+---
+#
+# Tasks related to deploying the code jail for the XQWatcher
+#
+- name: Create sandboxed user
+  user: >
+    name="{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}"
+    shell=/bin/false
+    home="/dev/null"
+  with_items: XQWATCHER_COURSES
+
+#
+# Need to disable aa to update the virutalenv
+- name: write out apparmor config
+  template: >
+    src=etc/apparmor.d/code.jail.j2
+    dest="/etc/apparmor.d/code.jail.{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+    mode=0644 owner=root group=root
+  with_items: XQWATCHER_COURSES
+
+- name: write out sudoers config jail user
+  template: >
+    src=etc/sudoers.d/95-jailed-user.j2
+    dest=/etc/sudoers.d/95-{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}
+    mode=0440 owner=root group=root validate='visudo -c -f %s'
+  with_items: XQWATCHER_COURSES
+
+- name: write out sudoers for watcher
+  template: >
+    src=etc/sudoers.d/95-xqwatcher.j2
+    dest=/etc/sudoers.d/95-xqwatcher-{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}
+    mode=0440 owner=root group=root validate='visudo -c -f %s'
+  with_items: XQWATCHER_COURSES
+
+- name: create jail virtualenv
+  shell: >
+    /usr/sbin/aa-complain "/etc/apparmor.d/code.jail.{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}" && /usr/local/bin/virtualenv --no-site-packages {{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}
+  with_items: XQWATCHER_COURSES
+
+- name: give other read permissions to the virtualenv
+  shell: >
+    chown -R {{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }} {{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}
+  with_items: XQWATCHER_COURSES
+
+- name: start apparmor service
+  service: name=apparmor state=started
+
+- name: load code sandbox profile
+  command: apparmor_parser -r "/etc/apparmor.d/code.jail.{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+  with_items: XQWATCHER_COURSES
+
+- name: put code jail into aa-complain
+  command: /usr/sbin/aa-complain "/etc/apparmor.d/code.jail.{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+  when: CODE_JAIL_COMPLAIN
+  with_items: XQWATCHER_COURSES
+
+- name: put code sandbox into aa-enforce
+  command: /usr/sbin/aa-enforce "/etc/apparmor.d/code.jail.{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+  when: not CODE_JAIL_COMPLAIN
\ No newline at end of file

playbooks/roles/xqwatcher/tasks/deploy.yml

+- name: install read-only ssh key
+  copy: >
+    content="{{ XQWATCHER_GIT_IDENTITY }}" dest={{ xqwatcher_git_identity }}
+    owner={{ xqwatcher_user }} group={{ xqwatcher_user }} mode=0600
+
+- name: upload ssh script
+  template: >
+    src=git_ssh.sh.j2 dest=/tmp/git_ssh.sh
+    owner={{ xqwatcher_user }} mode=750
+    
+- include: deploy_watcher.yml tags=deploy-watcher
+
+- include: deploy_courses.yml tags=deploy-courses
+    
+- name: remove read-only ssh key for the content repo
+  file: path={{ xqwatcher_git_identity }} state=absent
+  
\ No newline at end of file

playbooks/roles/xqwatcher/tasks/deploy_courses.yml

+- name: checkout grader code
+  git: >
+    dest={{ xqwatcher_data_dir }}/{{ item.COURSE }} repo={{ item.GIT_REPO }}
+    version={{ item.GIT_REF }}
+    accept_hostkey=yes
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  sudo: true
+  sudo_user: "{{ xqwatcher_user }}"
+  with_items: XQWATCHER_COURSES

playbooks/roles/xqwatcher/tasks/deploy_watcher.yml

+- name: checkout watcher code
+  git: >
+    dest={{ xqwatcher_code_dir }} repo={{ xqwatcher_source_repo }} version={{ xqwatcher_version }}
+    accept_hostkey=yes
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  sudo: true
+  sudo_user: "{{ xqwatcher_user }}"
+
+- name: install application requirements
+  pip: >
+    requirements="{{ xqwatcher_requirements_file }}"
+    virtualenv="{{ xqwatcher_venv_dir }}" state=present
+  sudo: true
+  sudo_user: "{{ xqwatcher_user }}"
+
+- name: write out course config files
+  template: >
+    src=edx/app/xqwatcher/conf.d/course.json.j2
+    dest={{ xqwatcher_conf_dir }}/{{ item.COURSE }}.json
+    mode=0644 owner={{ xqwatcher_user }} group={{ xqwatcher_user }}
+  with_items: XQWATCHER_COURSES
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d/xqwatcher.conf.j2
+    dest="{{ xqwatcher_supervisor_available_dir }}/xqwatcher.conf"
+    group={{ xqwatcher_user }} mode=0650
+
+- name: enable supervisor script
+  file: >
+    src={{ xqwatcher_supervisor_available_dir }}/xqwatcher.conf
+    dest={{ xqwatcher_supervisor_cfg_dir }}/xqwatcher.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ xqwatcher_supervisor_ctl }} -c /edx/app/xqwatcher/supervisor/supervisord.conf update"
+  when: not disable_edx_services
\ No newline at end of file

playbooks/roles/xqwatcher/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role xqwatcher
+# 
+# Overview:
+#
+# This play installs a sandboxed, pull grader that integrates with XQueue.  The
+# code for the XQWatcher lives here: https://github.com/edx/xqueue-watcher
+#
+# Multiple courses can be installed on a single server with distinct sandboxes.
+#
+# Example play:
+#
+# A play to install the XQWatcher would look like this:
+#
+# - name: Deploy xqueue-watcher
+#   hosts: all
+#   sudo: True
+#   gather_facts: True
+#   vars:
+#     COMMON_APP_DIR: "/edx/app"
+#     common_web_group: "www-data"
+#     ENABLE_DATADOG: False
+#     ENABLE_SPLUNKFORWARDER: False
+#     ENABLE_NEWRELIC: False
+#   roles:
+#     - aws
+#     - datadog
+#     - splunkforwarder
+#     - newrelic
+#     - xqwatcher
+#
+# You would use a commone like the following to run the play.
+#
+# ansible-playbook -i ec2.py ./xqwatcher.yml -e@./example-config.yml
+#
+# The contents of the example-config.yml would include the queue
+# meta data and details related to the repository including the
+# grader code.
+#
+# XQWATCHER_COURSES:
+#   - COURSE: "exampleX-101x"
+#     GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
+#     GIT_REF: "master"
+#     QUEUE_NAME: "exampleX-101x"
+#     QUEUE_CONFIG:
+#       SERVER: "https://xqueue.example.com"
+#       CONNECTIONS: 5
+#       AUTH: ["user", "password"]
+#       HANDLERS:
+#         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
+#           CODEJAIL:
+#             name: "exampleX-101x"
+#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#             user: "exampleX-101x"
+#           KWARGS:
+#             grader_root: "../data/exampleX-101x/graders/"
+#   - COURSE: "exampleX-202x"
+#     GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
+#     GIT_REF: "master"
+#     QUEUE_NAME: "exampleX-202x"
+#     QUEUE_CONFIG:
+#       SERVER: "https://xqueue.example.com"
+#       CONNECTIONS: 5
+#       AUTH: ["user", "password"]
+#       HANDLERS:
+#         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
+#           CODEJAIL:
+#             name: "exampleX-202x"
+#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#             user: "exampleX-202x"
+#           KWARGS:
+#             grader_root: "../data/exampleX-202x/graders/"
+
+# XQWATCHER_GIT_IDENTITY: |
+#   -----BEGIN RSA PRIVATE KEY-----
+#   Your key if you need to access any private repositories
+#   -----END RSA PRIVATE KEY-----
+#
+
+- include: code_jail.yml CODE_JAIL_COMPLAIN=false
+  
+- name: create conf dir
+  file: >
+    path="{{ xqwatcher_conf_dir }}"
+    state=directory
+    owner="{{ xqwatcher_user }}"
+    group="{{ xqwatcher_user }}"
+
+
+- name: create supervisor dirs
+  file: >
+    path="{{ xqwatcher_conf_dir }}"
+    state=directory
+    owner="{{ xqwatcher_user }}"
+    group="{{ xqwatcher_user }}"
+  with_items:
+    - "{{ xqwatcher_supervisor_data_dir }}"
+    - "{{ xqwatcher_supervisor_log_dir }}"  
+    
+- name: write out course config files
+  template: >
+    src=edx/app/xqwatcher/conf.d/course.json.j2
+    dest={{ xqwatcher_conf_dir }}/{{ item.COURSE }}.json
+    mode=0644 owner={{ xqwatcher_user }} group={{ xqwatcher_user }}
+  with_items: XQWATCHER_COURSES
+  
+- include: deploy.yml tags=deploy

playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2

+;  {{ ansible_managed }}
+;
+[program:xqwatcher]
+command={{ xqwatcher_venv_dir }}/bin/python -m {{ xqwatcher_module }} -d ../conf.d/
+process_name=%(program_name)s
+numprocs=1
+directory={{ xqwatcher_code_dir }}
+umask=022
+autostart=true
+autorestart=true
+startsecs=10
+startretries=3
+exitcodes=0,2
+stopsignal=TERM
+stopwaitsecs=10
+redirect_stderr=false
+stdout_logfile={{ xqwatcher_supervisor_log_dir }}/%(program_name)-stdout.log
+stdout_logfile_maxbytes=1MB
+stdout_logfile_backups=10
+stdout_capture_maxbytes=1MB
+stderr_logfile={{ xqwatcher_supervisor_log_dir }}/%(program_name)-stderr.log
+stderr_logfile_maxbytes=1MB
+stderr_logfile_backups=10
+stderr_capture_maxbytes=1MB

playbooks/roles/xqwatcher/templates/edx/app/xqwatcher/conf.d/course.json.j2

+{
+ "{{ item.QUEUE_NAME }}":
+    {{ item.QUEUE_CONFIG | to_nice_json }}
+}
\ No newline at end of file

playbooks/roles/xqwatcher/templates/etc/apparmor.d/code.jail.j2

+#include <tunables/global>
+
+{{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python {
+    #include <abstractions/base>
+
+    {{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/** mr,
+    #todo need a way of providing.
+    # edxapp_code_dir /common/lib/sandbox-packages/** r,
+    /tmp/codejail-*/ rix,
+    /tmp/codejail-*/** wrix,
+
+    #
+    # Whitelist particiclar shared objects from the system
+    # python installation
+    #
+    /usr/lib/python2.7/lib-dynload/_json.so mr,
+    /usr/lib/python2.7/lib-dynload/_ctypes.so mr,
+    /usr/lib/python2.7/lib-dynload/_heapq.so mr,
+    /usr/lib/python2.7/lib-dynload/_io.so mr,
+    /usr/lib/python2.7/lib-dynload/_csv.so mr,
+    /usr/lib/python2.7/lib-dynload/datetime.so mr,
+    /usr/lib/python2.7/lib-dynload/_elementtree.so mr,
+    /usr/lib/python2.7/lib-dynload/pyexpat.so mr,
+
+    #
+    # Allow access to selections from /proc
+    #
+    /proc/*/mounts r,
+
+}

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-course-sandbox.j2

+{{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=({{ item.QUEUE.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_venv_base }}/{{ item.QUEUE.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/bin/kill
+{{ item.QUEUE.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-jailed-user.j2

+{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/bin/kill
+{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-xqwatcher.j2

+{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_venv_base }}/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ xqwatcher_user }} ALL=(ALL) NOPASSWD:/bin/kill
+{{ xqwatcher_user }} ALL=(ALL) NOPASSWD:/usr/bin/pkill

playbooks/roles/xqwatcher/templates/git_ssh.sh.j2

+#!/bin/sh
+exec /usr/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ xqwatcher_git_identity }} "$@"