Merge pull request #4309 from edx/jeskew/devpi_container_addition

Add devpi and devpi_consumer roles and devpi Docker container.

CHANGELOG.md

+- Role: devpi
+  - New role added to configure a devpi service as a pass-through cache for PyPI.
+
+- Role: devpi_consumer
+  - Added role to configure Python containers to use devpi for Docker Devstack
+
 - Role: xqueue
   - Remove S3_BUCKET and S3_PATH_PREFIX - they were deprecated prior to ginkgo
   - Remove SERVICE_VARIANT - it was copied from edxapp but never truly used (except to complicate things)

docker/build/devpi/Dockerfile

+# To build this Dockerfile:
+#
+# From the root of configuration:
+#
+# docker build -f docker/build/devpi/Dockerfile .
+#
+# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
+# with the currently checked-out configuration repo.
+
+FROM edxops/xenial-common:latest
+MAINTAINER edxops
+
+ARG ARG_DEVPI_SERVER_VERSION=4.4.0
+ARG ARG_DEVPI_WEB_VERSION=3.2.2
+ARG ARG_DEVPI_CLIENT_VERSION=4.0.0
+
+ADD . /edx/app/edx_ansible/edx_ansible
+
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
+RUN apt-get update
+
+COPY docker/build/devstack/ansible_overrides.yml /devstack/ansible_overrides.yml
+
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook devpi.yml \
+    -c local -i '127.0.0.1,' \
+    -t "install,devstack" \
+    --extra-vars="@/devstack/ansible_overrides.yml" \
+    --extra-vars="DEVPI_SERVER_VERSION=$ARG_DEVPI_SERVER_VERSION" \
+    --extra-vars="DEVPI_WEB_VERSION=$ARG_DEVPI_WEB_VERSION" \
+    --extra-vars="DEVPI_CLIENT_VERSION=$ARG_DEVPI_CLIENT_VERSION"
+
+EXPOSE 3141
+VOLUME /data
+
+COPY docker/build/devpi/docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+USER root
+ENV HOME /data
+WORKDIR /data
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["devpi"]

docker/build/devpi/docker-entrypoint.sh

+#!/bin/bash
+
+function defaults {
+    : ${DEVPI_SERVERDIR="/data/server"}
+    : ${DEVPI_CLIENTDIR="/data/client"}
+
+    echo "DEVPI_SERVERDIR is ${DEVPI_SERVERDIR}"
+    echo "DEVPI_CLIENTDIR is ${DEVPI_CLIENTDIR}"
+
+    export DEVPI_SERVERDIR DEVPI_CLIENTDIR
+}
+
+function initialize_devpi {
+    echo "[RUN]: Initializing devpi-server..."
+    if [ ! -d  $DEVPI_SERVERDIR ]; then
+        devpi-server --restrict-modify root --init --start --host 127.0.0.1 --port 3141
+    else
+        devpi-server --restrict-modify root --start --host 127.0.0.1 --port 3141
+    fi
+    devpi-server --status
+    devpi use http://localhost:3141
+    devpi login root --password=''
+    DEVPI_PASSWORD=`date +%s | sha256sum | base64 | head -c 32`
+    devpi user -m root password="${DEVPI_PASSWORD}"
+    echo "[RUN]: devpi-server password set to '${DEVPI_PASSWORD}'" > $DEVPI_SERVERDIR/.serverpassword
+    devpi index -y -c public pypi_whitelist='*'
+    devpi-server --stop
+    devpi-server --status
+}
+
+defaults
+
+if [ "$1" = 'devpi' ]; then
+    source /home/devpi/venvs/devpi_venv/bin/activate
+
+    if [ ! -f  $DEVPI_SERVERDIR/.serverversion ]; then
+        initialize_devpi
+    fi
+
+    echo "[RUN]: Launching devpi-server..."
+    exec devpi-server --restrict-modify root --host 0.0.0.0 --port 3141
+fi
+
+echo "[RUN]: Builtin command not provided [devpi]"
+echo "[RUN]: $@"
+
+exec "$@"

docker/plays/devpi.yml

+# Usage: ansible-playbook devpi.yml -i <admin-host>, -e <secure-repo>/admin/edx_admin.yml -e <secure-repo>/admin/admin.yml
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - common_vars
+    - docker
+    - devpi

docker/plays/edxapp.yml

@@ -8,6 +8,7 @@
   roles:
     - common_vars
     - docker
+    - devpi_consumer
     - role: nginx
       nginx_sites:
       - lms

playbooks/roles/devpi/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role devpi
+
+devpi_group: devpi
+devpi_user: devpi
+devpi_app_dir: /home/devpi
+devpi_venv_dir: devpi_venv
+devpi_venv_path: "{{ devpi_app_dir }}/venvs/{{ devpi_venv_dir }}"
+
+devpi_environment:
+  PIP_NO_CACHE_DIR: "off"
+  PIP_INDEX_URL: "https://pypi.python.org/simple"
+  PIP_TRUSTED_HOST: "127.0.0.1"
+  VIRTUAL_ENV: "{{ devpi_venv_path }}"
+  PATH: $VIRTUAL_ENV/bin:$PATH
+
+# The versions below are required to be passed-in to the role.
+DEVPI_REQUIREMENTS:
+  - name: devpi-server
+    version: "{{ DEVPI_SERVER_VERSION }}"
+  - name: devpi-web
+    version: "{{ DEVPI_WEB_VERSION }}"
+  - name: devpi-client
+    version: "{{ DEVPI_CLIENT_VERSION }}"

playbooks/roles/devpi/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role devpi
+#
+# Overview:
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+
+- name: Create the application group
+  group:
+    name: "{{ devpi_group }}"
+    state: present
+  tags:
+    - "install"
+
+- name: Create application user
+  user:
+    name: "{{ devpi_user }}"
+    home: "{{ devpi_app_dir }}"
+    group: "{{ devpi_group }}"
+    createhome: no
+    shell: /bin/false
+  tags:
+    - "install"
+
+- name: Create devpi user dirs
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ devpi_user }}"
+    group: "{{ devpi_group }}"
+  with_items:
+    - "{{ devpi_app_dir }}"
+    - "{{ devpi_venv_dir }}"
+  tags:
+    - "install"
+
+- name: install python requirements
+  pip:
+    name: "{{ item.name }}"
+    version: "{{ item.version|default(omit) }}"
+    extra_args: "--exists-action w {{ item.extra_args|default('') }}"
+    virtualenv: "{{ devpi_venv_path }}"
+    state: present
+  with_items: "{{ DEVPI_REQUIREMENTS }}"
+  become_user: "{{ devpi_user }}"
+  tags:
+    - "install"

playbooks/roles/devpi_consumer/defaults/main.yml

+# Variables for the devpi_consumer role
+# This role allows a host to use the configured devpi server as a primary source for pip
+# Defaults are for Docker Devstack
+#
+
+# This should be a directory, pip.conf will be appended
+DEVPI_PIP_CONF_PATH: /root/.pip
+DEVPI_PIP_CONF_OWNER: root
+DEVPI_PIP_CONF_GROUP: root
+
+DEVPI_HOST: edx.devstack.devpi
+DEVPI_PORT: 3141
+
+# http or https
+#
+DEVPI_PROTOCOL: http
+
+# Use a leading slash, but no trailing slash here
+#
+DEVPI_INDEX: /root/pypi

playbooks/roles/devpi_consumer/tasks/main.yml

+- name: create pip conf directory
+  file:
+    path: "{{ DEVPI_PIP_CONF_PATH }}"
+    state: directory
+    owner: "{{ DEVPI_PIP_CONF_OWNER }}"
+    group: "{{ DEVPI_PIP_CONF_GROUP }}"
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
+- name: write pip.conf to devstack
+  template:
+    src: "pip.conf.j2"
+    dest: "{{ DEVPI_PIP_CONF_PATH }}/pip.conf"
+    owner: "{{ DEVPI_PIP_CONF_OWNER }}"
+    group: "{{ DEVPI_PIP_CONF_GROUP }}"
+    mode: 0744
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install

playbooks/roles/devpi_consumer/templates/pip.conf.j2

+[global]
+index-url = {{ DEVPI_PROTOCOL }}://{{ DEVPI_HOST }}:{{ DEVPI_PORT }}{{ DEVPI_INDEX }}/+simple/
+
+# Enables failover to PyPI if devpi is broken
+extra-index-url = https://pypi.python.org/simple
+
+# This is necessary if you aren't using TLS on the DevPI host
+trusted-host = {{ DEVPI_HOST }}
+
+[search]
+index = {{ DEVPI_PROTOCOL }}://{{ DEVPI_HOST }}:{{ DEVPI_PORT }}{{ DEVPI_INDEX }}/

util/parsefiles_config.yml

@@ -30,3 +30,4 @@ weights:
   - notes: 2
   - notifier: 2
   - mongo: 1
+  - devpi: 1