Merge pull request #2976 from edx/ned/fix-dogwood

Latest cherry-picked changes for Dogwood.

playbooks/edx-east/devpi.yml

-- name: Deploy devpi
-  hosts: all
-  sudo: True
-  gather_facts: True
-  vars:
-    serial_count: 1
-  serial: "{{ serial_count }}"
-  roles:
-    - aws
-    - devpi

playbooks/edx-east/edx_mirror.yml

@@ -6,14 +6,8 @@
   roles:
     - role: nginx
       nginx_sites:
-      - devpi
       - gh_mirror
       tags: ['r_nginx']
-    - role: supervisor
-      supervisor_servers:
-      - devpi
-    - role: devpi
-      tags: ['r_devpi']
     - role: gh_mirror
       tags: ['r_gh_mirror']
 

playbooks/roles/common_vars/defaults/main.yml

@@ -105,7 +105,7 @@ common_pip_pkgs:
   - pip==7.1.2
   - setuptools==18.3.2
   - virtualenv==13.1.2
-  - virtualenvwrapper
+  - virtualenvwrapper==4.7.1
 
 common_web_user: www-data
 common_web_group: www-data

playbooks/roles/devpi/defaults/main.yml

----
-devpi_app_dir: "{{ COMMON_APP_DIR }}/devpi"
-devpi_data_dir: "{{ COMMON_DATA_DIR }}/devpi"
-devpi_mirror_dir: "{{ devpi_data_dir }}/data"
-devpi_log_dir: "{{ COMMON_LOG_DIR }}/devpi"
-devpi_venvs_dir: "{{ devpi_app_dir }}/venvs"
-devpi_venv_dir: "{{ devpi_venvs_dir }}/devpi"
-devpi_venv_bin: "{{ devpi_venv_dir }}/bin"
-devpi_pip_pkgs:
-  - devpi-server
-  - eventlet
-devpi_nginx_port: 80
-devpi_port: 4040
-devpi_user: devpi
-devpi_group: devpi
-devpi_server_name: 'pypy.*'
-
-devpi_supervisor_user: devpi.supervisor
-devpi_supervisor_app_dir: "{{ devpi_app_dir }}/supervisor"
-devpi_supervisor_cfg_dir: "{{ devpi_supervisor_app_dir }}/conf.d"
-devpi_supervisor_data_dir: "{{ devpi_data_dir }}/supervisor"
-devpi_supervisor_cfg: "{{ devpi_supervisor_app_dir }}/supervisord.conf"
-devpi_supervisor_log_dir: "{{ devpi_log_dir }}/supervisor"
-devpi_supervisor_venv_dir: "{{ devpi_app_dir }}/venvs/supervisor"
-devpi_supervisor_venv_bin: "{{ devpi_supervisor_venv_dir }}/bin"
-devpi_supervisor_ctl: "{{ devpi_supervisor_venv_bin }}/supervisorctl"

playbooks/roles/devpi/handlers/main.yml

-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Defaults for role devpi
-#
----
-- name: restart devpi
-  supervisorctl: >
-    state=restarted
-    supervisorctl_path={{ devpi_supervisor_ctl }}
-    config={{ devpi_supervisor_cfg }}
-    name=devpi-server
-  sudo_user: "{{ devpi_supervisor_user }}"

playbooks/roles/devpi/meta/main.yml

----
-dependencies:
-  - common
-  - role: supervisor
-    supervisor_app_dir: "{{ devpi_supervisor_app_dir }}"
-    supervisor_data_dir: "{{ devpi_supervisor_data_dir }}"
-    supervisor_log_dir: "{{ devpi_supervisor_log_dir }}"
-    supervisor_venv_dir: "{{ devpi_supervisor_venv_dir }}"
-    supervisor_service_user: "{{ devpi_supervisor_user }}"
-    supervisor_service: "supervisor.devpi"
-    supervisor_http_bind_port: '9002'

playbooks/roles/devpi/tasks/main.yml

-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Tasks for role devpi
-#
-# Overview:
-#   Creates a pypi caching server
-#
-# Dependencies:
-#   - common
-#   - nginx
-#   - supervisor
-#
-# Example play:
-#   roles:
-#     - common
-#     - role: nginx
-#       nginx_sites:
-#       - devpi
-#     - role: supervisor
-#       supervisor_servers:
-#       - devpi
-#     - devpi
-
----
-- name: create devpi user
-  user: >
-    name={{ devpi_user }}
-    shell=/bin/false createhome=no
-  notify: restart devpi
-
-- name: create devpi application directories
-  file: >
-    path={{ item }}
-    state=directory
-    owner={{ devpi_user }}
-    group={{ devpi_supervisor_user }}
-  with_items:
-  - "{{ devpi_app_dir }}"
-  - "{{ devpi_venv_dir }}"
-  notify: restart devpi
-
-- name: create the devpi data directory, needs write access by the service user
-  file: >
-    path={{ item }}
-    state=directory
-    owner={{ devpi_supervisor_user }}
-    group={{ devpi_user }}
-  with_items:
-  - "{{ devpi_data_dir }}"
-  - "{{ devpi_mirror_dir }}"
-  notify: restart devpi
-
-- name: install devpi pip pkgs
-  pip: >
-    name={{ item }}
-    state=present
-    virtualenv={{ devpi_venv_dir }}
-  sudo_user: "{{ devpi_user }}"
-  with_items: devpi_pip_pkgs
-  notify: restart devpi
-
-- name: writing supervisor script
-  template: >
-    src=devpi.conf.j2 dest={{ devpi_supervisor_cfg_dir }}/devpi.conf
-    owner={{ devpi_user }} group={{ devpi_user }} mode=0644
-  notify: restart devpi
-
-- name: create a symlink for venv python, pip
-  file: >
-    src="{{ devpi_venv_bin }}/{{ item }}"
-    dest={{ COMMON_BIN_DIR }}/{{ item }}.devpi
-    state=link
-  notify: restart devpi
-  with_items:
-  - python
-  - pip
-
-- name: create a symlink for venv supervisor
-  file: >
-    src="{{ devpi_supervisor_venv_bin }}/supervisorctl"
-    dest={{ COMMON_BIN_DIR }}/supervisorctl.devpi
-    state=link
-
-- name: create a symlink for supervisor config
-  file: >
-    src="{{ devpi_supervisor_app_dir }}/supervisord.conf"
-    dest={{ COMMON_CFG_DIR }}/supervisord.conf.devpi
-    state=link
-
-  # call supervisorctl update. this reloads
-  # the supervisorctl config and restarts
-  # the services if any of the configurations
-  # have changed.
-  #
-- name: update devpi supervisor configuration
-  shell:  "{{ devpi_supervisor_ctl }} -c {{ devpi_supervisor_cfg }} update"
-  register: supervisor_update
-  changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
-
-- name: ensure devpi is started
-  supervisorctl: >
-    state=started
-    supervisorctl_path={{ devpi_supervisor_ctl }}
-    config={{ devpi_supervisor_cfg }}
-    name=devpi-server
-  sudo_user: "{{ devpi_supervisor_user }}"

playbooks/roles/devpi/templates/devpi.conf.j2

-[program:devpi-server]
-command={{ devpi_venv_bin }}/devpi-server --port {{ devpi_port }} --serverdir {{ devpi_mirror_dir }}
-user={{ devpi_supervisor_user }}
-priority=999
-stdout_logfile={{ devpi_supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ devpi_supervisor_log_dir }}/%(program_name)s-stderr.log
-autostart=True
-killasgroup=true
-stopasgroup=true

playbooks/roles/edxapp/meta/main.yml

@@ -7,7 +7,6 @@ dependencies:
     rbenv_user: "{{ edxapp_user }}"
     rbenv_dir: "{{ edxapp_app_dir }}"
     rbenv_ruby_version: "{{ edxapp_ruby_version }}"
-  - devpi
   - nltk
   - role: user
     user_info:

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/devpi.j2

-server {
-    listen {{ devpi_nginx_port }};
-    server_name {{ devpi_server_name }};
-    gzip             on;
-    gzip_min_length  2000;
-    gzip_proxied     any;
-    gzip_types       text/html application/json;
-
-    location / {
-        root {{ devpi_data_dir }};
-        proxy_pass http://localhost:{{ devpi_port }};
-        proxy_set_header  X-outside-url $scheme://$host;
-        proxy_set_header  X-Real-IP $remote_addr;
-    }
-
-    {% include robots.j2 %}
-}
-

playbooks/roles/stop_all_edx_services/handlers/main.yml

@@ -21,9 +21,6 @@
 - name: stop supervisor
   service: name=supervisor state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
 
-- name: stop supervisor.devpi
-  service: name=supervisor.devpi state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
-
 - name: stop nginx
   service: name=nginx state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
 
@@ -36,9 +33,6 @@
 - name: stop memcached
   service: name=memcached state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
 
-- name: stop supervisor.devpi
-  service: name=supervisor.devpi state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
-
 - name: stop mongodb
   service: name=mongodb state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
 
@@ -47,5 +41,4 @@
   shell: pgrep -u {{ item }} -lf | grep -v celery | grep -v supervisord | grep -v gunicorn |  awk '{ print $1}' | xargs -I {} kill {} || true
   with_items:
   - www-data
-  - devpi.supervisor
   - rabbitmq

playbooks/roles/stop_all_edx_services/tasks/main.yml

@@ -28,12 +28,6 @@
   changed_when: stat_out is defined and stat_out.stat.exists
   notify: stop supervisor
 
-- name: stop supervisor.devpi
-  stat: path=/etc/init/supervisor.devpi.conf
-  register: stat_out
-  changed_when: stat_out is defined and stat_out.stat.exists
-  notify: stop supervisor
-
 - name: stop nginx
   stat: path=/etc/init.d/nginx
   register: stat_out

playbooks/roles/supervisor/defaults/main.yml

@@ -41,6 +41,8 @@ supervisor_cfg: "{{ supervisor_app_dir }}/supervisord.conf"
 supervisor_service: supervisor
 supervisor_service_user: "{{ common_web_user }}"
 
+supervisor_version: 3.2.3
+
 supervisor_pip_pkgs:
   - boto=="{{ common_boto_version }}"
   - python-simple-hipchat

playbooks/roles/supervisor/tasks/main.yml

@@ -110,9 +110,12 @@
     - install:base
 
 - name: install supervisor in its venv
-  pip: >
-    name=supervisor virtualenv="{{ supervisor_venv_dir }}" state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+  pip:
+    name: supervisor
+    version: "{{ supervisor_version }}"
+    virtualenv: "{{ supervisor_venv_dir }}"
+    state: present
+    extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
   sudo_user: "{{ supervisor_user }}"
   tags:
     - install
@@ -235,4 +238,4 @@
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
   tags:
     - manage
-    - manage:start
\ No newline at end of file
+    - manage:start

playbooks/vagrant/ansible.cfg

@@ -2,6 +2,7 @@
 
 jinja2_extensions=jinja2.ext.do
 host_key_checking = False
+library=../library
 roles_path=../roles
 callback_plugins=../callback_plugins
 ansible_managed=This file is created and updated by ansible, edit at your peril

requirements.txt

@@ -11,7 +11,7 @@ docopt==0.6.1
 python-simple-hipchat==0.2
 prettytable==0.7.2
 awscli==1.4.2
-requests
+requests==2.9.1
 datadog==0.8.0
 
 # Needed for the mongo_* modules (playbooks/library/mongo_*)