Merge branch 'master' into pwattenberger/sailthru

playbooks/callback_plugins/datadog_tasks_timing.py

-import os
-import datetime
-import time
-import logging
-import datadog
-import sys
-
-logging.basicConfig(level=logging.INFO, stream=sys.stdout)
-logging.getLogger("requests").setLevel(logging.WARNING)
-logging.getLogger("dd").setLevel(logging.WARNING)
-logger = logging.getLogger(__name__)
-
-"""
-Originally written by 'Jharrod LaFon'
-#https://github.com/jlafon/ansible-profile/blob/master/callback_plugins/profile_tasks.py
-
-"""
-
-
-class CallbackModule(object):
-
-    """
-    Ansible plugin get the time of each task and total time
-    to run the complete playbook
-    """
-    def __init__(self):
-        self.stats = {}
-        self.current_task = None
-        self.playbook_name = None
-        self.datadog_api_key = os.getenv('DATADOG_API_KEY')
-        self.datadog_api_initialized = False
-
-        if self.datadog_api_key:
-            datadog.initialize(api_key=self.datadog_api_key,
-                               app_key=None)
-            self.datadog_api_initialized = True
-
-    def clean_tag_value(self, value):
-        return value.replace(" | ", ".").replace(" ", "-").lower()
-
-    def playbook_on_play_start(self, pattern):
-            self.playbook_name, _ = os.path.splitext(
-                os.path.basename(self.play.playbook.filename)
-            )
-
-    def playbook_on_task_start(self, name, is_conditional):
-        """
-        Logs the start of each task
-        """
-
-        if self.current_task is not None:
-            # Record the running time of the last executed task
-            self.stats[self.current_task] = (time.time(), time.time() - self.stats[self.current_task])
-
-        # Record the start time of the current task
-        self.current_task = name
-        self.stats[self.current_task] = time.time()
-
-    def playbook_on_stats(self, stats):
-        """
-        Prints the timing of each task and total time to
-        run the complete playbook
-        """
-        # Record the timing of the very last task, we use it here, because we
-        # don't have stop task function by default
-        if self.current_task is not None:
-            self.stats[self.current_task] = (time.time(), time.time() - self.stats[self.current_task])
-
-        # Sort the tasks by their running time
-        results = sorted(self.stats.items(),
-                         key=lambda value: value[1][1], reverse=True)
-
-        # Total time to run the complete playbook
-        total_seconds = sum([x[1][1] for x in self.stats.items()])
-          
-        # send the metric to datadog
-        if self.datadog_api_initialized:
-            datadog_tasks_metrics = []
-            for name, points in results:
-                datadog_tasks_metrics.append({'metric': 'edx.ansible.task_duration',
-                                              'date_happened': points[0],
-                                              'points': points[1],
-                                              'tags': ['task:{0}'.format(self.clean_tag_value(name)),
-                                                       'playbook:{0}'.format(self.clean_tag_value(self.playbook_name))
-                                                       ]
-                                              }
-                                             )
-            try:
-                datadog.api.Metric.send(datadog_tasks_metrics)
-                datadog.api.Metric.send(metric="edx.ansible.playbook_duration",
-                                        date_happened=time.time(),
-                                        points=total_seconds,
-                                        tags=["playbook:{0}".format(self.clean_tag_value(self.playbook_name))]
-                                        )
-            except Exception as ex:
-                logger.error(ex.message)
-
-        # Log the time of each task
-        for name, elapsed in results[:10]:
-            logger.info(
-                "{0:-<80}{1:->8}".format(
-                    '{0} '.format(name),
-                    ' {0:.02f}s'.format(elapsed[1]),
-                )
-            )
-
-        logger.info("\nPlaybook {0} finished: {1}, {2} total tasks.  {3} elapsed. \n".format(
-                self.playbook_name,
-                time.asctime(),
-                len(self.stats.items()),
-                datetime.timedelta(seconds=(int(total_seconds)))
-                )
-          )

playbooks/callback_plugins/task_timing.py

+from datetime import datetime, timedelta
+import json
+import logging
+import os
+from os.path import splitext, basename, exists, dirname
+import sys
+import time
+
+try:
+    from ansible.plugins.callback import CallbackBase
+except ImportError:
+    # Support Ansible 1.9.x
+    CallbackBase = object
+
+import datadog
+
+logging.basicConfig(level=logging.INFO, stream=sys.stdout)
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("dd").setLevel(logging.WARNING)
+LOGGER = logging.getLogger(__name__)
+
+"""
+Originally written by 'Jharrod LaFon'
+#https://github.com/jlafon/ansible-profile/blob/master/callback_plugins/profile_tasks.py
+
+"""
+
+ANSIBLE_TIMER_LOG = os.environ.get('ANSIBLE_TIMER_LOG')
+
+
+class Timestamp(object):
+    """
+    A class for capturing start, end and duration for an action.
+    """
+    def __init__(self):
+        self.start = datetime.utcnow()
+        self.end = None
+
+    def stop(self):
+        """
+        Record the end time of the timed period.
+        """
+        self.end = datetime.utcnow()
+
+    @property
+    def duration(self):
+        """
+        Return the duration that this Timestamp covers.
+        """
+        return self.end - self.start
+
+
+# This class only has a single method (which would ordinarily make it a
+# candidate to be turned into a function). However, the TimingLoggers are
+# instanciated once when ansible starts up, and then called for every play.
+class TimingLogger(object):
+    """
+    Base-class for logging timing about ansible tasks and plays.
+    """
+    def log_play(self, playbook_name, playbook_timestamp, results):
+        """
+        Record the timing results of an ansible play.
+
+        Arguments:
+            playbook_name: the name of the playbook being logged.
+            playbook_timestamp (Timestamp): the timestamps measuring how
+                long the play took.
+            results (dict(string -> Timestamp)): a dict mapping task names
+                to Timestamps that measure how long each task took.
+        """
+        pass
+
+
+class DatadogTimingLogger(TimingLogger):
+    """
+    Record ansible task and play timing to Datadog.
+
+    Requires that the environment variable DATADOG_API_KEY be set in order
+    to log any data.
+    """
+    def __init__(self):
+        super(DatadogTimingLogger, self).__init__()
+
+        self.datadog_api_key = os.getenv('DATADOG_API_KEY')
+        self.datadog_api_initialized = False
+
+        if self.datadog_api_key:
+            datadog.initialize(
+                api_key=self.datadog_api_key,
+                app_key=None
+            )
+            self.datadog_api_initialized = True
+
+    def clean_tag_value(self, value):
+        """
+        Remove any characters that aren't allowed in Datadog tags.
+
+        Arguments:
+            value: the string to be cleaned.
+        """
+        return value.replace(" | ", ".").replace(" ", "-").lower()
+
+    def log_play(self, playbook_name, playbook_timestamp, results):
+        if not self.datadog_api_initialized:
+            return
+
+        datadog_tasks_metrics = []
+        for name, timestamp in results.items():
+            datadog_tasks_metrics.append({
+                'metric': 'edx.ansible.task_duration',
+                'date_happened': time.mktime(timestamp.start.timetuple()),
+                'points': timestamp.duration.total_seconds(),
+                'tags': [
+                    'task:{0}'.format(self.clean_tag_value(name)),
+                    'playbook:{0}'.format(self.clean_tag_value(playbook_name))
+                ]
+            })
+        try:
+            datadog.api.Metric.send(datadog_tasks_metrics)
+            datadog.api.Metric.send(
+                metric="edx.ansible.playbook_duration",
+                date_happened=time.mktime(playbook_timestamp.start.timetuple()),
+                points=playbook_timestamp.duration.total_seconds(),
+                tags=["playbook:{0}".format(self.clean_tag_value(playbook_name))]
+            )
+        except Exception:
+            LOGGER.exception("Failed to log timing data to datadog")
+
+
+class JsonTimingLogger(TimingLogger):
+    """
+    Record task and play timing to a local file in a JSON format.
+
+    Requires that the environment variable ANSIBLE_TIMER_LOG be set in order
+    to log any data. This specifies the file that timing data should be logged
+    to. That variable can include strftime interpolation variables,
+    which will be replaced with the start time of the play.
+    """
+    def log_play(self, playbook_name, playbook_timestamp, results):
+        # N.B. This is intended to provide a consistent interface and message
+        # format across all of Open edX tooling, so it deliberately eschews
+        # standard python logging infrastructure.
+        if ANSIBLE_TIMER_LOG is None:
+            return
+
+        messages = []
+        for name, timestamp in results.items():
+            messages.append({
+                'task': name,
+                'playbook': playbook_name,
+                'started_at': timestamp.start.isoformat(),
+                'ended_at': timestamp.end.isoformat(),
+                'duration': timestamp.duration.total_seconds(),
+            })
+
+        messages.append({
+            'playbook': playbook_name,
+            'started_at': playbook_timestamp.start.isoformat(),
+            'ended_at': playbook_timestamp.end.isoformat(),
+            'duration': playbook_timestamp.duration.total_seconds(),
+        })
+
+        log_path = playbook_timestamp.start.strftime(ANSIBLE_TIMER_LOG)
+
+        try:
+            if not exists(dirname(log_path)):
+                os.makedirs(dirname(log_path))
+
+            with open(log_path, 'a') as outfile:
+                for log_message in messages:
+                    json.dump(
+                        log_message,
+                        outfile,
+                        separators=(',', ':'),
+                        sort_keys=True,
+                    )
+                    outfile.write('\n')
+        except Exception:
+            LOGGER.exception("Unable to write json timing log messages")
+
+
+class LoggingTimingLogger(TimingLogger):
+    """
+    Log timing information for the play and the top 10 tasks to stdout.
+    """
+    def log_play(self, playbook_name, playbook_timestamp, results):
+
+        # Sort the tasks by their running time
+        sorted_results = sorted(
+            results.items(),
+            key=lambda (task, timestamp): timestamp.duration,
+            reverse=True
+        )
+
+        for name, timestamp in sorted_results[:10]:
+            LOGGER.info(
+                "{0:-<80}{1:->8}".format(
+                    '{0} '.format(name),
+                    ' {0:.02f}s'.format(timestamp.duration.total_seconds()),
+                )
+            )
+
+        LOGGER.info(
+            "\nPlaybook %s finished: %s, %d total tasks.  %s elapsed. \n",
+            playbook_name,
+            playbook_timestamp.end,
+            len(results),
+            playbook_timestamp.duration,
+        )
+
+
+class CallbackModule(CallbackBase):
+
+    """
+    Ansible plugin get the time of each task and total time
+    to run the complete playbook
+    """
+    def __init__(self):
+        self.stats = {}
+        self.current_task = None
+        self.playbook_name = None
+        self.playbook_timestamp = None
+        self.play = None
+
+        self.loggers = [
+            DatadogTimingLogger(),
+            LoggingTimingLogger(),
+            JsonTimingLogger(),
+        ]
+
+    def v2_playbook_on_play_start(self, play):
+        self.play = play
+        super(CallbackModule, self).v2_playbook_on_play_start(play)
+
+    def playbook_on_play_start(self, pattern):
+        """
+        Record the start of a play.
+        """
+        self.playbook_name, _ = splitext(
+            basename(self.play.playbook.filename)
+        )
+        self.playbook_timestamp = Timestamp()
+
+    def playbook_on_task_start(self, name, is_conditional):
+        """
+        Logs the start of each task
+        """
+
+        if self.current_task is not None:
+            # Record the running time of the last executed task
+            self.stats[self.current_task].stop()
+
+        # Record the start time of the current task
+        self.current_task = name
+        self.stats[self.current_task] = Timestamp()
+
+    def playbook_on_stats(self, stats):
+        """
+        Prints the timing of each task and total time to
+        run the complete playbook
+        """
+        # Record the timing of the very last task, we use it here, because we
+        # don't have stop task function by default
+        if self.current_task is not None:
+            self.stats[self.current_task].stop()
+
+        self.playbook_timestamp.stop()
+
+        for logger in self.loggers:
+            logger.log_play(
+                self.playbook_name,
+                self.playbook_timestamp,
+                self.stats,
+            )

playbooks/edx-east/manage_edxapp_users_and_groups.yml

@@ -50,6 +50,10 @@
 #     unusable_password: true
 #     groups: []
 #
+#   - username: zoe
+#     email: zoe@example.com
+#     initial_password_hash: 'pbkdf2_sha256$20000$levJ6jdVYCsu$gdBLGf2DNPqfaKdcETXtFocRU8Kk+sMsIvKkmw1dKbY='
+#
 # groups:
 #   - name: group3
 #     remove: true
@@ -91,4 +95,5 @@
         {% if item.get('superuser') %}--superuser{% endif %}
         {% if item.get('staff') %}--staff{% endif %}
         {% if item.get('unusable_password') %}--unusable-password{% endif %}
+        {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %}
       with_items: django_users

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/nginx/sites-available/ROLE_NAME.j2

@@ -4,7 +4,7 @@
 
 
 {{ '{%' }} if nginx_default_sites is defined and "{{ role_name }}" in nginx_default_sites {{ '%}' }}
-  {{ '{%' }} set default_site = "default" {{ '%}' }}
+  {{ '{%' }} set default_site = "default_server" {{ '%}' }}
 {{ '{%' }} else {{ '%}' }}
   {{ '{%' }} set default_site = "" {{ '%}' }}
 {{ '{%' }} endif {{ '%}' }}

playbooks/roles/aws/tasks/main.yml

@@ -87,13 +87,22 @@
  # and add a custom motd.  These do not require an
  # ssh restart
  # Only needed for EC2 instances.
-- name: Update the ssh motd on Ubuntu
-  file:
+- name: Check if files exist so the next task doesn't fail
+  stat:
     path: "{{ item }}"
-    mode: "0644"
-  when: (vagrant_home_dir.stat.exists == false) and (ansible_distribution in common_debian_variants)
+  register: motd_files_exist
   with_items:
     - "/etc/update-motd.d/10-help-text"
     - "/usr/share/landscape/landscape-sysinfo.wrapper"
     - "/etc/update-motd.d/51-cloudguest"
     - "/etc/update-motd.d/91-release-upgrade"
+
+- name: Update the ssh motd on Ubuntu
+  file:
+    path: "{{ item.item }}"
+    mode: "0644"
+  when: >
+    vagrant_home_dir.stat.exists == false and
+    ansible_distribution in common_debian_variants and
+    item.stat.exists
+  with_items: motd_files_exist.results

playbooks/roles/common/tasks/main.yml

@@ -34,24 +34,32 @@
     path: /home/vagrant
   register: vagrant_home_dir
 
+# Ensure that we get a current version of Git
+# GitHub requires version 1.7.10 or later
+# https://help.github.com/articles/https-cloning-errors
+- name: Add git apt repository
+  apt_repository:
+    repo: "{{ common_git_ppa }}"
+  when: ansible_distribution in common_debian_variants
+
+
+# Ensure that we get the latest version of python 2.7
 - name: add edx ppa apt key
   apt_key:
     id: "{{ COMMON_EDX_PPA_KEY_ID }}"
     keyserver: "{{ COMMON_EDX_PPA_KEY_SERVER }}"
     state: "present"
-  when: ansible_distribution in common_debian_variants
-
+  when: >
+    ansible_distribution in common_debian_variants and
+    ansible_distribution_release in common_custom_ppa_releases
 
-# Ensure that we get a current version of Git and latest version of python 2.7
-# GitHub requires version 1.7.10 or later
-# https://help.github.com/articles/https-cloning-errors
-- name: Add git apt repository
+- name: Add custom edX PPA
   apt_repository:
-    repo: "{{ item }}"
-  with_items:
-    - "{{ common_git_ppa }}"
-    - "{{ COMMON_EDX_PPA }}"
-  when: ansible_distribution in common_debian_variants
+    repo: "{{ COMMON_EDX_PPA }}"
+  when: >
+    ansible_distribution in common_debian_variants and
+    ansible_distribution_release in common_custom_ppa_releases
+
 
 - name: Install role-independent useful system packages
   # do this before log dir setup; rsyslog package guarantees syslog user present
@@ -63,6 +71,17 @@
   with_items: "{{ common_debian_pkgs }}"
   when: ansible_distribution in common_debian_variants
 
+- name: Install role-independent useful system packages from custom PPA
+  apt:
+    name: "{{ item }}"
+    install_recommends: yes
+    state: present
+    update_cache: yes
+  with_items: "{{ common_custom_debian_pkgs }}"
+  when: >
+    ansible_distribution in common_debian_variants and 
+    ansible_distribution_release in common_custom_ppa_releases
+
 - name: Install role-independent useful system packages
   yum:
     name: "{{ item }}"

playbooks/roles/common_vars/defaults/main.yml

@@ -112,10 +112,13 @@ common_debian_pkgs:
   - rsyslog
   - git
   - unzip
-  - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1"
   - python-pip
   - python2.7-dev
-  
+
+# Packages that should be installed from our custom PPA, i.e. COMMON_EDX_PPA
+common_custom_debian_pkgs:
+  - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1"
+
 common_pip_pkgs:
   - pip==8.1.2
   - setuptools==24.0.3
@@ -147,6 +150,11 @@ common_debian_variants:
   - Ubuntu
   - Debian
 
+# Only attempt to use our custom PPA for these releases
+common_custom_ppa_releases:
+  - precise
+  - trusty
+
 common_redhat_variants:
   - CentOS
   - Red Hat Enterprise Linux

playbooks/roles/credentials/templates/edx/app/nginx/sites-available/credentials.j2

@@ -4,7 +4,7 @@
 
 
 {% if nginx_default_sites is defined and "credentials" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/discovery/templates/edx/app/nginx/sites-available/discovery.j2

@@ -4,7 +4,7 @@
 
 
 {% if nginx_default_sites is defined and "discovery" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/harstorage/templates/edx/app/nginx/sites-available/harstorage.j2

@@ -4,7 +4,7 @@
 
 
 {% if nginx_default_sites is defined and "harstorage" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2

 {%- if "cms" in nginx_default_sites -%}
-  {%- set default_site = "default" -%}
+  {%- set default_site = "default_server" -%}
 {%- else -%}
   {%- set default_site = "" -%}
 {%- endif -%}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/credentials.j2

@@ -4,7 +4,7 @@
 
 
 {% if "credentials" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/ecommerce.j2

@@ -4,7 +4,7 @@
 
 
 {% if "ecommerce" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2

@@ -10,7 +10,7 @@
 {% endraw %}
 
 {%- if "forum" in nginx_default_sites -%}
-  {%- set default_site = "default" -%}
+  {%- set default_site = "default_server" -%}
 {%- else -%}
   {%- set default_site = "" -%}
 {%- endif -%}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/harstorage.j2

@@ -4,7 +4,7 @@
 
 
 {% if nginx_default_sites is defined and "harstorage" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2

 {%- if "kibana" in nginx_default_sites -%}
-  {%- set default_site = "default" -%}
+  {%- set default_site = "default_server" -%}
 {%- else -%}
   {%- set default_site = "" -%}
 {%- endif -%}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2

 {%- if "lms" in nginx_default_sites -%}
-  {%- set default_site = "default" -%}
+  {%- set default_site = "default_server" -%}
 {%- else -%}
   {%- set default_site = "" -%}
 {%- endif -%}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/nginx_redirect.j2

 {%- if "default" in item.value -%}
-  {%- set default_site = "default" -%}
+  {%- set default_site = "default_server" -%}
 {%- else -%}
   {%- set default_site = "" -%}
 {%- endif -%}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/programs.j2

@@ -4,7 +4,7 @@
 
 
 {% if "programs" in nginx_default_sites %}
-  {% set default_site = "default" %}
+  {% set default_site = "default_server" %}
 {% else %}
   {% set default_site = "" %}
 {% endif %}

playbooks/roles/supervisor/defaults/main.yml

@@ -46,3 +46,36 @@ supervisor_version: 3.2.3
 supervisor_pip_pkgs:
   - boto=="{{ common_boto_version }}"
   - python-simple-hipchat
+
+supervisor_spec:
+  - service: edxapp
+    python: python.edxapp
+    code: "{{ edxapp_code_dir | default(None) }}"
+    env: "{{ edxapp_app_dir | default(None) }}/edxapp_env"
+  - service: xqueue
+    python: python.xqueue
+    code: "{{ xqueue_code_dir | default(None) }}"
+  - service: ecommerce
+    python: python.ecommerce
+    code: "{{ ecommerce_code_dir | default(None) }}"
+    env: "{{ ecommerce_home | default(None) }}/ecommerce_env"
+  - service: insights
+    python: python.insights
+    code: "{{ insights_code_dir | default(None) }}"
+    env: "{{ insights_home | default(None) }}/insights_env"
+  - service: analytics_api
+    python: python.analytics_api
+    code: "{{ analytics_api_code_dir | default(None) }}"
+    env: "{{ analytics_api_home | default(None) }}/analytics_api_env"
+  - service: programs
+    python: python.programs
+    code: "{{ programs_code_dir | default(None) }}"
+    env: "{{ programs_home | default(None) }}/programs_env"
+  - service: credentials
+    python: python.credentials
+    code: "{{ credentials_code_dir | default(None) }}"
+    env: "{{ credentials_home | default(None) }}/credentials_env"
+  - service: discovery
+    python: python.discovery
+    code: "{{ discovery_code_dir | default(None) }}"
+    env: "{{ discovery_home | default(None) }}/discovery_env"

playbooks/roles/supervisor/tasks/main.yml

@@ -61,7 +61,7 @@
   tags:
     - install
     - install:base
-    
+
 - name: Create supervisor and service user accessible directories
   file:
     path: "{{ item }}"
@@ -121,6 +121,7 @@
     dest: "/etc/init/{{ supervisor_service }}.conf"
     owner: root
     group: root
+  when: ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 16
   tags:
     - install
     - install:base
@@ -134,11 +135,25 @@
     dest: "/etc/init/pre_supervisor.conf"
     owner: root
     group: root
-  when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+  when: >
+    supervisor_service == "supervisor" and disable_edx_services and not devstack
+    and ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 16
   tags:
     - to-remove
     - aws-specfic
 
+# NB: with systemd, pre_supervisor is a pre-task for supervisor, not a separate service
+- name: Create supervisor systemd job
+  template:
+    src: "etc/init/supervisor-systemd.service.j2"
+    dest: "/etc/systemd/system/{{ supervisor_service }}.service"
+    owner: root
+    group: root
+  when: not (ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 16)
+  tags:
+    - install
+    - install:base
+
 - name: Write the pre_suprevisor python script
   copy:
     src: pre_supervisor_checks.py
@@ -198,11 +213,21 @@
     - install
     - install:configuration
 
+- name: Enable supervisor to start on boot
+  service:
+    name: "{{ supervisor_service }}.service"
+    enabled: yes
+  when: not (ansible_distribution == 'Ubuntu' and ansible_distribution_major_version|int < 16)
+  tags:
+    - install
+    - install:base
+
 - name: Start supervisor
   service:
     name: "{{ supervisor_service }}"
     state: started
   register: start_supervisor
+  when: not disable_edx_services
   tags:
     - manage
     - manage:start
@@ -226,6 +251,7 @@
   # See https://github.com/ansible/ansible/issues/4853
 - name: Update supervisor configuration
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
   register: supervisor_update
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
   tags:

playbooks/roles/supervisor/templates/etc/init/supervisor-systemd.service.j2

+[Unit]
+Description=supervisord - Supervisor process control system
+Documentation=http://supervisord.org
+After=network.target
+
+
+[Service]
+{% if disable_edx_services and not devstack -%}
+# Run pre_supervisor
+ExecStartPre={{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py \
+  {% if SUPERVISOR_HIPCHAT_API_KEY is defined -%}
+    --hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} \
+  {% endif -%}
+  {%- for item in supervisor_spec -%}
+    {%- if item.code -%}
+      {%- set name = item.service.replace('_', '-') -%}
+      --{{ name }}-python {{ COMMON_BIN_DIR }}/{{ item.python }} --{{ name }}-code-dir {{ item.code }}
+      {%- if item.env is defined %} --{{ name }}-env {{ item.env }}{% endif %} \
+    {% endif %}
+  {%- endfor -%}
+  --available={{ supervisor_available_dir }} --enabled={{ supervisor_cfg_dir }}
+{% endif %}
+
+# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre)
+# This is needed because pre_supervisor needs to write to supervisor/conf.d, which
+# supervisor_service_user does not have permission to do.
+PermissionsStartOnly=true
+User={{ supervisor_service_user }}
+
+Type=forking
+TimeoutStartSec=432000
+
+ExecStart={{ supervisor_venv_dir }}/bin/supervisord --configuration {{ supervisor_cfg }}
+ExecReload={{ supervisor_venv_dir }}/bin/supervisorctl reload
+ExecStop={{ supervisor_venv_dir }}/bin/supervisorctl shutdown
+
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/xqueue/tasks/deploy.yml

@@ -152,3 +152,12 @@
   with_items:
     - xqueue
     - xqueue_consumer
+  tags:
+    - install
+    - install:configuration
+    - install:code
+    - install:app-requirements
+    - migrate
+    - migrate:db
+    - manage
+    - manage:app-users

playbooks/roles/xqwatcher/defaults/main.yml

@@ -14,6 +14,7 @@
 # - COURSE: "exampleX-101x"
 #   GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
 #   GIT_REF: "master"
+#   PYTHON_EXECUTABLE: python2
 #   PYTHON_REQUIREMENTS: []
 #   QUEUE_NAME: "exampleX-101x"
 #   QUEUE_CONFIG:
@@ -24,13 +25,15 @@
 #       - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #         CODEJAIL:
 #           name: "exampleX-101x"
-#           python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#           bin_path: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
 #           user: "exampleX-101x"
+#           lang: python2
 #         KWARGS:
 #           grader_root: "../data/exampleX-101x/graders/"
 # - COURSE: "exampleX-202x"
 #   GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
 #   GIT_REF: "master"
+#   PYTHON_EXECUTABLE: python3
 #   PYTHON_REQUIREMENTS: []
 #   QUEUE_NAME: "exampleX-202x"
 #   QUEUE_CONFIG:
@@ -41,10 +44,13 @@
 #       - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #         CODEJAIL:
 #           name: "exampleX-202x"
-#           python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#           bin_path: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
 #           user: "exampleX-202x"
+#           lang: python2
 #         KWARGS:
 #           grader_root: "../data/exampleX-202x/graders/"
+#
+# NB: only python2 and python3 are supported.
 
 XQWATCHER_CONFIG:
   HTTP_BASIC_AUTH: ["{{ COMMON_HTPASSWD_USER }}","{{ COMMON_HTPASSWD_PASS }}"]
@@ -94,21 +100,6 @@ xqwatcher_module: "xqueue_watcher"
 xqwatcher_venv_base: "{{ xqwatcher_app_dir }}/venvs"
 xqwatcher_venv_dir: "{{ xqwatcher_venv_base }}/{{ xqwatcher_service_name }}"
 
-
-#
-# supervisor related config
-#
-xqwatcher_supervisor_app_dir: "{{ xqwatcher_app_dir }}/supervisor"
-xqwatcher_supervisor_http_port: 9003
-xqwatcher_supervisor_data_dir: "{{ COMMON_DATA_DIR }}/{{ xqwatcher_service_name }}"
-xqwatcher_supervisor_log_dir: "{{ xqwatcher_log_dir }}"
-xqwatcher_supervisor_venv_dir: "{{ xqwatcher_venv_base }}/supervisor"
-xqwatcher_supervisor_user: "{{ xqwatcher_user }}"
-xqwatcher_supervisor_venv_bin: "{{ xqwatcher_supervisor_venv_dir }}/bin"
-xqwatcher_supervisor_ctl: "{{ xqwatcher_supervisor_venv_bin }}/supervisorctl"
-xqwatcher_supervisor_cfg_dir: "{{ xqwatcher_supervisor_app_dir }}/conf.d"
-xqwatcher_supervisor_available_dir: "{{ xqwatcher_supervisor_app_dir }}/conf.available.d"
-
 #
 # OS packages
 #

playbooks/roles/xqwatcher/meta/main.yml

@@ -14,6 +14,7 @@
 # random corners of ansible/jinga/python variable expansion.
 dependencies:
   - common
+  - role: supervisor
   - role: edx_service
     edx_service_name: "{{ xqwatcher_service_name }}"
     edx_service_repos: "{{ XQWATCHER_REPOS }}"
@@ -22,12 +23,3 @@ dependencies:
     edx_service_packages:
         debian: "{{ xqwatcher_debian_pkgs }}"
         redhat: "{{ xqwatcher_redhat_pkgs }}"
-  - role: supervisor
-    supervisor_app_dir: "{{ xqwatcher_supervisor_app_dir }}"
-    supervisor_data_dir: "{{ xqwatcher_supervisor_data_dir }}"
-    supervisor_log_dir: "{{ xqwatcher_supervisor_log_dir }}"
-    supervisor_venv_dir: "{{ xqwatcher_supervisor_venv_dir }}"
-    supervisor_service_user: "{{ xqwatcher_supervisor_user }}"
-    supervisor_available_dir: "{{ xqwatcher_supervisor_available_dir }}"
-    supervisor_service: "supervisor.xqwatcher"
-    supervisor_http_bind_port: "{{ xqwatcher_supervisor_http_port }}"

playbooks/roles/xqwatcher/tasks/code_jail.yml

@@ -48,7 +48,7 @@
     - manage:sandbox
 
 - name: Create jail virtualenv
-  shell: "/usr/local/bin/virtualenv --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
+  shell: "/usr/local/bin/virtualenv --python={{ item.PYTHON_EXECUTABLE }} --no-site-packages {{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}"
   with_items: "{{ XQWATCHER_COURSES }}"
   tags:
     - install

playbooks/roles/xqwatcher/tasks/deploy_watcher.yml

@@ -28,30 +28,20 @@
 - name: Write supervisord config
   template:
     src: "edx/app/supervisor/conf.d/xqwatcher.conf.j2"
-    dest: "{{ xqwatcher_supervisor_available_dir }}/xqwatcher.conf"
-    group: "{{ xqwatcher_user }}"
-    mode: "0650"
-  tags:
-    - install
-    - install:configuration
-
-- name: Enable supervisor script
-  file:
-    src: "{{ xqwatcher_supervisor_available_dir }}/xqwatcher.conf"
-    dest: "{{ xqwatcher_supervisor_cfg_dir }}/xqwatcher.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
+    dest: "{{ supervisor_available_dir }}/xqwatcher.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: "0644"
   tags:
     - install
     - install:configuration
 
 - name: Update supervisor configuration
-  shell: "{{ xqwatcher_supervisor_ctl }} -c {{ xqwatcher_supervisor_app_dir }}/supervisord.conf update"
+  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   when: not disable_edx_services
   tags:
     - manage
-    - manage:update
+    - manage:start
 
 - name: Restart xqwatcher
   supervisorctl:

playbooks/roles/xqwatcher/tasks/main.yml

@@ -48,6 +48,7 @@
 #   - COURSE: "exampleX-101x"
 #     GIT_REPO: "git@github.com:foo/graders-exampleX-101x.git"
 #     GIT_REF: "master"
+#     PYTHON_EXECUTABLE: python2
 #     PYTHON_REQUIREMENTS: []
 #     QUEUE_NAME: "exampleX-101x"
 #     QUEUE_CONFIG:
@@ -58,13 +59,15 @@
 #         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #           CODEJAIL:
 #             name: "exampleX-101x"
-#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
+#             bin_path: "{{ xqwatcher_venv_base }}/exampleX-101x/bin/python"
 #             user: "exampleX-101x"
+#             lang: python2
 #           KWARGS:
 #             grader_root: "../data/exampleX-101x/graders/"
 #   - COURSE: "exampleX-202x"
 #     GIT_REPO: "git@github.com:foo/graders-exampleX-202x.git"
 #     GIT_REF: "master"
+#     PYTHON_EXECUTABLE: python3
 #     PYTHON_REQUIREMENTS: []
 #     QUEUE_NAME: "exampleX-202x"
 #     QUEUE_CONFIG:
@@ -75,8 +78,9 @@
 #         - HANDLER: "xqueue_watcher.jailedgrader.JailedGrader"
 #           CODEJAIL:
 #             name: "exampleX-202x"
-#             python_bin: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
+#             bin_path: "{{ xqwatcher_venv_base }}/exampleX-202x/bin/python"
 #             user: "exampleX-202x"
+#             lang: python2
 #           KWARGS:
 #             grader_root: "../data/exampleX-202x/graders/"
 

playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2

-;  {{ ansible_managed }}
-;
+#
+#  {{ ansible_managed }}
+#
 
 {% set xqwatcher_venv_dir = xqwatcher_app_dir + '/venvs/' + xqwatcher_service_name %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
@@ -11,10 +12,10 @@
 [program:{{ xqwatcher_service_name }}]
 command={{ executable }} -m {{ xqwatcher_module }} -d {{ xqwatcher_conf_dir }} 
 process_name=%(program_name)s
-user={{ xqwatcher_user }}
+user={{ common_web_user }}
 directory={{ xqwatcher_code_dir }}
-stdout_logfile={{ xqwatcher_supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ xqwatcher_supervisor_log_dir }}/%(program_name)s-stderr.log
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}
 killasgroup=true
 stopasgroup=true

playbooks/roles/xqwatcher/templates/etc/sudoers.d/95-xqwatcher.j2

-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/bin/kill
-{{ xqwatcher_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/usr/bin/pkill
+{{ common_web_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }})  SETENV:NOPASSWD:{{ xqwatcher_app_dir }}/venvs/{{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.name }}/bin/python
+{{ common_web_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/bin/kill
+{{ common_web_user }} ALL=({{ item.QUEUE_CONFIG.HANDLERS[0].CODEJAIL.user }}) NOPASSWD:/usr/bin/pkill

util/vpc-tools/abbey.py

@@ -299,12 +299,26 @@ if [[ ! -x /usr/bin/git || ! -x /usr/bin/pip ]]; then
         libxslt-dev curl libmysqlclient-dev --force-yes
 fi
 
+# python3 is required for certain other things
+# (currently xqwatcher so it can run python2 and 3 grader code,
+# but potentially more in the future). It's not available on Ubuntu 12.04,
+# but in those cases we don't need it anyways.
+if [[ -n "$(apt-cache search --names-only '^python3-pip$')" ]]; then
+    /usr/bin/apt-get update
+    /usr/bin/apt-get install -y python3-pip python3-dev
+fi
+
 # this is missing on 14.04 (base package on 12.04)
 # we need to do this on any build, since the above apt-get
 # only runs on a build from scratch
 /usr/bin/apt-get install -y python-httplib2 --force-yes
 
-# upgrade setuptools early to avoid no distributin errors
+# Must upgrade to latest before pinning to work around bug
+# https://github.com/pypa/pip/issues/3862
+pip install --upgrade pip
+pip install --upgrade pip==8.1.2
+
+# upgrade setuptools early to avoid no distribution errors
 pip install --upgrade setuptools==24.0.3
 
 rm -rf $base_dir