Merge pull request #3255 from edx/max/ops-1536

OPS-1536: Add python3 support to xqwatcher

playbooks/roles/aws/tasks/main.yml

@@ -87,13 +87,22 @@
  # and add a custom motd.  These do not require an
  # ssh restart
  # Only needed for EC2 instances.
-- name: Update the ssh motd on Ubuntu
-  file:
+- name: Check if files exist so the next task doesn't fail
+  stat:
     path: "{{ item }}"
-    mode: "0644"
-  when: (vagrant_home_dir.stat.exists == false) and (ansible_distribution in common_debian_variants)
+  register: motd_files_exist
   with_items:
     - "/etc/update-motd.d/10-help-text"
     - "/usr/share/landscape/landscape-sysinfo.wrapper"
     - "/etc/update-motd.d/51-cloudguest"
     - "/etc/update-motd.d/91-release-upgrade"
+
+- name: Update the ssh motd on Ubuntu
+  file:
+    path: "{{ item.item }}"
+    mode: "0644"
+  when: >
+    vagrant_home_dir.stat.exists == false and
+    ansible_distribution in common_debian_variants and
+    item.stat.exists
+  with_items: motd_files_exist.results

playbooks/roles/common/tasks/main.yml

@@ -34,24 +34,32 @@
     path: /home/vagrant
   register: vagrant_home_dir
 
+# Ensure that we get a current version of Git
+# GitHub requires version 1.7.10 or later
+# https://help.github.com/articles/https-cloning-errors
+- name: Add git apt repository
+  apt_repository:
+    repo: "{{ common_git_ppa }}"
+  when: ansible_distribution in common_debian_variants
+
+
+# Ensure that we get the latest version of python 2.7
 - name: add edx ppa apt key
   apt_key:
     id: "{{ COMMON_EDX_PPA_KEY_ID }}"
     keyserver: "{{ COMMON_EDX_PPA_KEY_SERVER }}"
     state: "present"
-  when: ansible_distribution in common_debian_variants
-
+  when: >
+    ansible_distribution in common_debian_variants and
+    ansible_distribution_release in common_custom_ppa_releases
 
-# Ensure that we get a current version of Git and latest version of python 2.7
-# GitHub requires version 1.7.10 or later
-# https://help.github.com/articles/https-cloning-errors
-- name: Add git apt repository
+- name: Add custom edX PPA
   apt_repository:
-    repo: "{{ item }}"
-  with_items:
-    - "{{ common_git_ppa }}"
-    - "{{ COMMON_EDX_PPA }}"
-  when: ansible_distribution in common_debian_variants
+    repo: "{{ COMMON_EDX_PPA }}"
+  when: >
+    ansible_distribution in common_debian_variants and
+    ansible_distribution_release in common_custom_ppa_releases
+
 
 - name: Install role-independent useful system packages
   # do this before log dir setup; rsyslog package guarantees syslog user present
@@ -63,6 +71,17 @@
   with_items: "{{ common_debian_pkgs }}"
   when: ansible_distribution in common_debian_variants
 
+- name: Install role-independent useful system packages from custom PPA
+  apt:
+    name: "{{ item }}"
+    install_recommends: yes
+    state: present
+    update_cache: yes
+  with_items: "{{ common_custom_debian_pkgs }}"
+  when: >
+    ansible_distribution in common_debian_variants and 
+    ansible_distribution_release in common_custom_ppa_releases
+
 - name: Install role-independent useful system packages
   yum:
     name: "{{ item }}"

playbooks/roles/common_vars/defaults/main.yml

@@ -112,10 +112,13 @@ common_debian_pkgs:
   - rsyslog
   - git
   - unzip
-  - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1"
   - python-pip
   - python2.7-dev
-  
+
+# Packages that should be installed from our custom PPA, i.e. COMMON_EDX_PPA
+common_custom_debian_pkgs:
+  - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1"
+
 common_pip_pkgs:
   - pip==8.1.2
   - setuptools==24.0.3
@@ -147,6 +150,11 @@ common_debian_variants:
   - Ubuntu
   - Debian
 
+# Only attempt to use our custom PPA for these releases
+common_custom_ppa_releases:
+  - precise
+  - trusty
+
 common_redhat_variants:
   - CentOS
   - Red Hat Enterprise Linux

playbooks/roles/supervisor/defaults/main.yml

@@ -46,3 +46,36 @@ supervisor_version: 3.2.3
 supervisor_pip_pkgs:
   - boto=="{{ common_boto_version }}"
   - python-simple-hipchat
+
+supervisor_spec:
+  - service: edxapp
+    python: python.edxapp
+    code: "{{ edxapp_code_dir | default(None) }}"
+    env: "{{ edxapp_app_dir | default(None) }}/edxapp_env"
+  - service: xqueue
+    python: python.xqueue
+    code: "{{ xqueue_code_dir | default(None) }}"
+  - service: ecommerce
+    python: python.ecommerce
+    code: "{{ ecommerce_code_dir | default(None) }}"
+    env: "{{ ecommerce_home | default(None) }}/ecommerce_env"
+  - service: insights
+    python: python.insights
+    code: "{{ insights_code_dir | default(None) }}"
+    env: "{{ insights_home | default(None) }}/insights_env"
+  - service: analytics_api
+    python: python.analytics_api
+    code: "{{ analytics_api_code_dir | default(None) }}"
+    env: "{{ analytics_api_home | default(None) }}/analytics_api_env"
+  - service: programs
+    python: python.programs
+    code: "{{ programs_code_dir | default(None) }}"
+    env: "{{ programs_home | default(None) }}/programs_env"
+  - service: credentials
+    python: python.credentials
+    code: "{{ credentials_code_dir | default(None) }}"
+    env: "{{ credentials_home | default(None) }}/credentials_env"
+  - service: discovery
+    python: python.discovery
+    code: "{{ discovery_code_dir | default(None) }}"
+    env: "{{ discovery_home | default(None) }}/discovery_env"

playbooks/roles/supervisor/tasks/main.yml

@@ -61,7 +61,7 @@
   tags:
     - install
     - install:base
-    
+
 - name: Create supervisor and service user accessible directories
   file:
     path: "{{ item }}"
@@ -121,6 +121,7 @@
     dest: "/etc/init/{{ supervisor_service }}.conf"
     owner: root
     group: root
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < 16
   tags:
     - install
     - install:base
@@ -134,11 +135,25 @@
     dest: "/etc/init/pre_supervisor.conf"
     owner: root
     group: root
-  when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+  when: >
+    supervisor_service == "supervisor" and disable_edx_services and not devstack
+    and ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < 16
   tags:
     - to-remove
     - aws-specfic
 
+# NB: with systemd, pre_supervisor is a pre-task for supervisor, not a separate service
+- name: Create supervisor systemd job
+  template:
+    src: "etc/init/supervisor-systemd.service.j2"
+    dest: "/etc/systemd/system/{{ supervisor_service }}.service"
+    owner: root
+    group: root
+  when: not (ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < 16)
+  tags:
+    - install
+    - install:base
+
 - name: Write the pre_suprevisor python script
   copy:
     src: pre_supervisor_checks.py
@@ -198,11 +213,21 @@
     - install
     - install:configuration
 
+- name: Enable supervisor to start on boot
+  service:
+    name: "{{ supervisor_service }}.service"
+    enabled: yes
+  when: not (ansible_distribution == 'Ubuntu' and ansible_distribution_major_version < 16)
+  tags:
+    - install
+    - install:base
+
 - name: Start supervisor
   service:
     name: "{{ supervisor_service }}"
     state: started
   register: start_supervisor
+  when: not disable_edx_services
   tags:
     - manage
     - manage:start
@@ -226,6 +251,7 @@
   # See https://github.com/ansible/ansible/issues/4853
 - name: Update supervisor configuration
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
   register: supervisor_update
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
   tags:

playbooks/roles/supervisor/templates/etc/init/supervisor-systemd.service.j2

+[Unit]
+Description=supervisord - Supervisor process control system
+Documentation=http://supervisord.org
+After=network.target
+
+
+[Service]
+{% if disable_edx_services and not devstack -%}
+# Run pre_supervisor
+ExecStartPre={{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py \
+  {% if SUPERVISOR_HIPCHAT_API_KEY is defined -%}
+    --hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} \
+  {% endif -%}
+  {%- for item in supervisor_spec -%}
+    {%- if item.code -%}
+      {%- set name = item.service.replace('_', '-') -%}
+      --{{ name }}-python {{ COMMON_BIN_DIR }}/{{ item.python }} --{{ name }}-code-dir {{ item.code }}
+      {%- if item.env is defined %} --{{ name }}-env {{ item.env }}{% endif %} \
+    {% endif %}
+  {%- endfor -%}
+  --available={{ supervisor_available_dir }} --enabled={{ supervisor_cfg_dir }}
+{% endif %}
+
+# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre)
+# This is needed because pre_supervisor needs to write to supervisor/conf.d, which
+# supervisor_service_user does not have permission to do.
+PermissionsStartOnly=true
+User={{ supervisor_service_user }}
+
+Type=forking
+TimeoutStartSec=432000
+
+ExecStart={{ supervisor_venv_dir }}/bin/supervisord --configuration {{ supervisor_cfg }}
+ExecReload={{ supervisor_venv_dir }}/bin/supervisorctl reload
+ExecStop={{ supervisor_venv_dir }}/bin/supervisorctl shutdown
+
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/xqueue/tasks/deploy.yml

@@ -152,3 +152,12 @@
   with_items:
     - xqueue
     - xqueue_consumer
+  tags:
+    - install
+    - install:configuration
+    - install:code
+    - install:app-requirements
+    - migrate
+    - migrate:db
+    - manage
+    - manage:app-users

playbooks/roles/xqwatcher/defaults/main.yml

@@ -14,6 +14,7 @@
 # - COURSE: "exampleX-101x"
 #   GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
 #   GIT_REF: "master"
+#   PYTHON_EXECUTABLE: python2
 #   PYTHON_REQUIREMENTS: []
 #   QUEUE_NAME: "exampleX-101x"
 #   QUEUE_CONFIG:
@@ -24,13 +25,15 @@
 #       - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #         CODEJAIL:
 #           name: "exampleX-101x"
-#           python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#           bin_path: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
 #           user: "exampleX-101x"
+#           lang: python2
 #         KWARGS:
 #           grader_root: "../data/exampleX-101x/graders/"
 # - COURSE: "exampleX-202x"
 #   GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
 #   GIT_REF: "master"
+#   PYTHON_EXECUTABLE: python3
 #   PYTHON_REQUIREMENTS: []
 #   QUEUE_NAME: "exampleX-202x"
 #   QUEUE_CONFIG:
@@ -41,10 +44,13 @@
 #       - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #         CODEJAIL:
 #           name: "exampleX-202x"
-#           python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#           bin_path: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
 #           user: "exampleX-202x"
+#           lang: python2
 #         KWARGS:
 #           grader_root: "../data/exampleX-202x/graders/"
+#
+# NB: only python2 and python3 are supported.
 
 XQWATCHER_CONFIG:
   HTTP_BASIC_AUTH: ["{{ COMMON_HTPASSWD_USER }}","{{ COMMON_HTPASSWD_PASS }}"]
@@ -94,21 +100,6 @@ xqwatcher_module: "xqueue_watcher"
 xqwatcher_venv_base: "{{ xqwatcher_app_dir }}/venvs"
 xqwatcher_venv_dir: "{{ xqwatcher_venv_base }}/{{ xqwatcher_service_name }}"
 
-
-#
-# supervisor related config
-#
-xqwatcher_supervisor_app_dir: "{{ xqwatcher_app_dir }}/supervisor"
-xqwatcher_supervisor_http_port: 9003
-xqwatcher_supervisor_data_dir: "{{ COMMON_DATA_DIR }}/{{ xqwatcher_service_name }}"
-xqwatcher_supervisor_log_dir: "{{ xqwatcher_log_dir }}"
-xqwatcher_supervisor_venv_dir: "{{ xqwatcher_venv_base }}/supervisor"
-xqwatcher_supervisor_user: "{{ xqwatcher_user }}"
-xqwatcher_supervisor_venv_bin: "{{ xqwatcher_supervisor_venv_dir }}/bin"
-xqwatcher_supervisor_ctl: "{{ xqwatcher_supervisor_venv_bin }}/supervisorctl"
-xqwatcher_supervisor_cfg_dir: "{{ xqwatcher_supervisor_app_dir }}/conf.d"
-xqwatcher_supervisor_available_dir: "{{ xqwatcher_supervisor_app_dir }}/conf.available.d"
-
 #
 # OS packages
 #

playbooks/roles/xqwatcher/meta/main.yml

@@ -14,6 +14,7 @@
 # random corners of ansible/jinga/python variable expansion.
 dependencies:
   - common
+  - role: supervisor
   - role: edx_service
     edx_service_name: "{{ xqwatcher_service_name }}"
     edx_service_repos: "{{ XQWATCHER_REPOS }}"
@@ -22,12 +23,3 @@ dependencies:
     edx_service_packages:
         debian: "{{ xqwatcher_debian_pkgs }}"
         redhat: "{{ xqwatcher_redhat_pkgs }}"
-  - role: supervisor
-    supervisor_app_dir: "{{ xqwatcher_supervisor_app_dir }}"
-    supervisor_data_dir: "{{ xqwatcher_supervisor_data_dir }}"
-    supervisor_log_dir: "{{ xqwatcher_supervisor_log_dir }}"
-    supervisor_venv_dir: "{{ xqwatcher_supervisor_venv_dir }}"
-    supervisor_service_user: "{{ xqwatcher_supervisor_user }}"
-    supervisor_available_dir: "{{ xqwatcher_supervisor_available_dir }}"
-    supervisor_service: "supervisor.xqwatcher"
-    supervisor_http_bind_port: "{{ xqwatcher_supervisor_http_port }}"

playbooks/roles/xqwatcher/tasks/code_jail.yml

@@ -48,7 +48,7 @@
     - manage:sandbox
 
 - name: Create jail virtualenv
-  shell: "/usr/local/bin/virtualenv --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+  shell: "/usr/local/bin/virtualenv --python={{ item.PYTHON_EXECUTABLE }} --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
   with_items: "{{ XQWATCHER_COURSES }}"
   tags:
     - install

playbooks/roles/xqwatcher/tasks/deploy_watcher.yml

@@ -28,30 +28,20 @@
 - name: Write supervisord config
   template:
     src: "edx/app/supervisor/conf.d/xqwatcher.conf.j2"
-    dest: "{{ xqwatcher_supervisor_available_dir }}/xqwatcher.conf"
-    group: "{{ xqwatcher_user }}"
-    mode: "0650"
-  tags:
-    - install
-    - install:configuration
-
-- name: Enable supervisor script
-  file:
-    src: "{{ xqwatcher_supervisor_available_dir }}/xqwatcher.conf"
-    dest: "{{ xqwatcher_supervisor_cfg_dir }}/xqwatcher.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
+    dest: "{{ supervisor_available_dir }}/xqwatcher.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: "0644"
   tags:
     - install
     - install:configuration
 
 - name: Update supervisor configuration
-  shell: "{{ xqwatcher_supervisor_ctl }} -c {{ xqwatcher_supervisor_app_dir }}/supervisord.conf update"
+  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   when: not disable_edx_services
   tags:
     - manage
-    - manage:update
+    - manage:start
 
 - name: Restart xqwatcher
   supervisorctl:

playbooks/roles/xqwatcher/tasks/main.yml

@@ -48,6 +48,7 @@
 #   - COURSE: "exampleX-101x"
 #     GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
 #     GIT_REF: "master"
+#     PYTHON_EXECUTABLE: python2
 #     PYTHON_REQUIREMENTS: []
 #     QUEUE_NAME: "exampleX-101x"
 #     QUEUE_CONFIG:
@@ -58,13 +59,15 @@
 #         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #           CODEJAIL:
 #             name: "exampleX-101x"
-#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#             bin_path: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
 #             user: "exampleX-101x"
+#             lang: python2
 #           KWARGS:
 #             grader_root: "../data/exampleX-101x/graders/"
 #   - COURSE: "exampleX-202x"
 #     GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
 #     GIT_REF: "master"
+#     PYTHON_EXECUTABLE: python3
 #     PYTHON_REQUIREMENTS: []
 #     QUEUE_NAME: "exampleX-202x"
 #     QUEUE_CONFIG:
@@ -75,8 +78,9 @@
 #         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #           CODEJAIL:
 #             name: "exampleX-202x"
-#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#             bin_path: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
 #             user: "exampleX-202x"
+#             lang: python2
 #           KWARGS:
 #             grader_root: "../data/exampleX-202x/graders/"
 

playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2

-;  {{ ansible_managed }}
-;
+#
+#  {{ ansible_managed }}
+#
 
 {% set xqwatcher_venv_dir = xqwatcher_app_dir + '/venvs/' + xqwatcher_service_name %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
@@ -11,10 +12,10 @@
 [program:{{ xqwatcher_service_name }}]
 command={{ executable }} -m {{ xqwatcher_module }} -d {{ xqwatcher_conf_dir }} 
 process_name=%(program_name)s
-user={{ xqwatcher_user }}
+user={{ common_web_user }}
 directory={{ xqwatcher_code_dir }}
-stdout_logfile={{ xqwatcher_supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ xqwatcher_supervisor_log_dir }}/%(program_name)s-stderr.log
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}
 killasgroup=true
 stopasgroup=true

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-xqwatcher.j2

-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/bin/kill
-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/usr/bin/pkill
+{{ common_web_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ common_web_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/bin/kill
+{{ common_web_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/usr/bin/pkill

util/vpc-tools/abbey.py

@@ -299,12 +299,26 @@ if [[ ! -x /usr/bin/git || ! -x /usr/bin/pip ]]; then
         libxslt-dev curl libmysqlclient-dev --force-yes
 fi
 
+# python3 is required for certain other things
+# (currently xqwatcher so it can run python2 and 3 grader code,
+# but potentially more in the future). It's not available on Ubuntu 12.04,
+# but in those cases we don't need it anyways.
+if [[ -n "$(apt-cache search --names-only '^python3-pip$')" ]]; then
+    /usr/bin/apt-get update
+    /usr/bin/apt-get install -y python3-pip python3-dev
+fi
+
 # this is missing on 14.04 (base package on 12.04)
 # we need to do this on any build, since the above apt-get
 # only runs on a build from scratch
 /usr/bin/apt-get install -y python-httplib2 --force-yes
 
-# upgrade setuptools early to avoid no distributin errors
+# Must upgrade to latest before pinning to work around bug
+# https://github.com/pypa/pip/issues/3862
+pip install --upgrade pip
+pip install --upgrade pip==8.1.2
+
+# upgrade setuptools early to avoid no distribution errors
 pip install --upgrade setuptools==24.0.3
 
 rm -rf $base_dir