OPS-1536: Add python3 support to xqwatcher

playbooks/roles/xqwatcher/defaults/main.yml

@@ -14,6 +14,7 @@
 # - COURSE: "exampleX-101x"
 #   GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
 #   GIT_REF: "master"
+#   PYTHON_EXECUTABLE: python2
 #   PYTHON_REQUIREMENTS: []
 #   QUEUE_NAME: "exampleX-101x"
 #   QUEUE_CONFIG:
@@ -24,13 +25,15 @@
 #       - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #         CODEJAIL:
 #           name: "exampleX-101x"
-#           python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#           bin_path: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
 #           user: "exampleX-101x"
+#           lang: python2
 #         KWARGS:
 #           grader_root: "../data/exampleX-101x/graders/"
 # - COURSE: "exampleX-202x"
 #   GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
 #   GIT_REF: "master"
+#   PYTHON_EXECUTABLE: python3
 #   PYTHON_REQUIREMENTS: []
 #   QUEUE_NAME: "exampleX-202x"
 #   QUEUE_CONFIG:
@@ -41,10 +44,13 @@
 #       - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #         CODEJAIL:
 #           name: "exampleX-202x"
-#           python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#           bin_path: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
 #           user: "exampleX-202x"
+#           lang: python2
 #         KWARGS:
 #           grader_root: "../data/exampleX-202x/graders/"
+#
+# NB: only python2 and python3 are supported.
 
 XQWATCHER_CONFIG:
   HTTP_BASIC_AUTH: ["{{ COMMON_HTPASSWD_USER }}","{{ COMMON_HTPASSWD_PASS }}"]

playbooks/roles/xqwatcher/tasks/code_jail.yml

@@ -48,7 +48,7 @@
     - manage:sandbox
 
 - name: Create jail virtualenv
-  shell: "/usr/local/bin/virtualenv --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+  shell: "/usr/local/bin/virtualenv --python={{ item.PYTHON_EXECUTABLE }} --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
   with_items: "{{ XQWATCHER_COURSES }}"
   tags:
     - install

playbooks/roles/xqwatcher/tasks/main.yml

@@ -48,6 +48,7 @@
 #   - COURSE: "exampleX-101x"
 #     GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
 #     GIT_REF: "master"
+#     PYTHON_EXECUTABLE: python2
 #     PYTHON_REQUIREMENTS: []
 #     QUEUE_NAME: "exampleX-101x"
 #     QUEUE_CONFIG:
@@ -58,13 +59,15 @@
 #         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #           CODEJAIL:
 #             name: "exampleX-101x"
-#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#             bin_path: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
 #             user: "exampleX-101x"
+#             lang: python2
 #           KWARGS:
 #             grader_root: "../data/exampleX-101x/graders/"
 #   - COURSE: "exampleX-202x"
 #     GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
 #     GIT_REF: "master"
+#     PYTHON_EXECUTABLE: python3
 #     PYTHON_REQUIREMENTS: []
 #     QUEUE_NAME: "exampleX-202x"
 #     QUEUE_CONFIG:
@@ -75,8 +78,9 @@
 #         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #           CODEJAIL:
 #             name: "exampleX-202x"
-#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#             bin_path: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
 #             user: "exampleX-202x"
+#             lang: python2
 #           KWARGS:
 #             grader_root: "../data/exampleX-202x/graders/"
 

util/vpc-tools/abbey.py

@@ -299,12 +299,26 @@ if [[ ! -x /usr/bin/git || ! -x /usr/bin/pip ]]; then
         libxslt-dev curl libmysqlclient-dev --force-yes
 fi
 
+# python3 is required for certain other things
+# (currently xqwatcher so it can run python2 and 3 grader code,
+# but potentially more in the future). It's not available on Ubuntu 12.04,
+# but in those cases we don't need it anyways.
+if [[ -n "$(apt-cache search --names-only '^python3-pip$')" ]]; then
+    /usr/bin/apt-get update
+    /usr/bin/apt-get install -y python3-pip python3-dev
+fi
+
 # this is missing on 14.04 (base package on 12.04)
 # we need to do this on any build, since the above apt-get
 # only runs on a build from scratch
 /usr/bin/apt-get install -y python-httplib2 --force-yes
 
-# upgrade setuptools early to avoid no distributin errors
+# Must upgrade to latest before pinning to work around bug
+# https://github.com/pypa/pip/issues/3862
+pip install --upgrade pip
+pip install --upgrade pip==8.1.2
+
+# upgrade setuptools early to avoid no distribution errors
 pip install --upgrade setuptools==24.0.3
 
 rm -rf $base_dir