Some environment changes to speed up deploys

* Instead of using pypi.edx.org why not cache on localhost?
  Seems to work well and it does result in a speed improvement
  for when we need to download packages. This adds the devpi
  role as dependency for edxapp
* For edxapp we md5sum all fo the requirements files and only
  run the pip installs, bundle, npm tasks if any one of those
  files have changed (it treats them as a group)
* Adds convenience symlinks for pip

playbooks/roles/certs/tasks/main.yml

@@ -77,6 +77,9 @@
 
 - name: certs | create a symlink for venv python
   file: >
-    src="{{ certs_venv_bin }}/python"
-    dest={{ COMMON_BIN_DIR }}/python.certs
+    src="{{ certs_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.certs
     state=link
+  with_items:
+  - python
+  - pip

playbooks/roles/devpi/defaults/main.yml

 ---
-devpi_venv_dir: "{{ COMMON_APP_DIR }}/devpi/venvs/devpi"
+devpi_app_dir: "{{ COMMON_APP_DIR }}/devpi"
+devpi_venvs_dir: "{{ devpi_app_dir }}/venvs"
+devpi_venv_dir: "{{ devpi_venvs_dir }}/devpi"
+devpi_venv_bin: "{{ devpi_venv_dir }}/bin"
 devpi_pip_pkgs:
   - devpi-server
   - eventlet
 devpi_nginx_port: 80
 devpi_port: 4040
-devpi_data_dir: /var/devpi/data
+devpi_data_dir: "{{ COMMON_DATA_DIR }}/devpi"
 devpi_user: devpi
 devpi_group: devpi
 devpi_server_name: 'pypy.*'

playbooks/roles/devpi/handlers/main.yml

@@ -13,13 +13,15 @@
 ---
 - name: devpi | restart devpi
   supervisorctl: >
-    state=restarted 
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     name=devpi-server
 
 - name: devpi | start devpi
   supervisorctl: >
     state=started
+    supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     name=devpi-server
 

playbooks/roles/devpi/tasks/main.yml

@@ -33,7 +33,8 @@
 - name: devpi | create devpi user
   user: >
     name={{ devpi_user }}
-    state=present
+    home={{ devpi_app_dir }}
+    shell=/bin/false createhome=no
 
 - name: devpi | create virtualenv directory
   file: >
@@ -47,22 +48,34 @@
   file: >
     path={{ devpi_data_dir }}
     state=directory
-    owner={{ devpi_user }}
+    owner={{ common_web_user }}
     group={{ devpi_group }}
 
 - name: devpi | install devpi pip pkgs
   pip: >
     name={{ item }}
-    state=present 
+    state=present
     virtualenv={{ devpi_venv_dir }}
+  sudo_user: "{{ devpi_user }}"
   with_items: devpi_pip_pkgs
   notify: devpi | restart devpi
 
-- name: supervisor | ensure supervisor is started
-  service: name=supervisor state=started
+- name: devpi | writing supervisor script
+  template: >
+    src=devpi.conf.j2 dest={{ supervisor_cfg_dir }}/devpi.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  notify: devpi | restart devpi
+
+- name: devpi | create a symlink for venv python
+  file: >
+    src="{{ devpi_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.devpi
+    state=link
+  with_items:
+  - python
+  - pip
 
-- name: devpi | ensure devpi is running
-  supervisorctl: >
-    state=started
-    config={{ supervisor_cfg }}
-    name=devpi-server
+# flush queued up handlers so that devpi
+# is restarted since downstream roles
+# will need to use it
+- meta: flush_handlers

playbooks/roles/devpi/templates/devpi.conf.j2

+[program:devpi-server]
+command={{ devpi_venv_bin }}/devpi-server --port {{ devpi_port }} --serverdir {{ devpi_data_dir }}
+user={{ common_web_user }}
+priority=999
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+autostart=True

playbooks/roles/edxapp/defaults/main.yml

@@ -111,6 +111,7 @@ edxapp_staticfile_dir: "{{ edxapp_data_dir }}/staticfiles"
 edxapp_course_data_dir: "{{ edxapp_data_dir }}/data"
 edxapp_upload_dir: "{{ edxapp_data_dir }}/uploads"
 edxapp_theme_dir: "{{ edxapp_data_dir }}/themes"
+edxapp_pypi_local_mirror: "http://localhost:{{ devpi_port }}/root/pypi/+simple"
 edxapp_workers:
   - queue: low
     service_variant: cms
@@ -134,15 +135,21 @@ edxapp_workers:
     service_variant: lms
     concurrency: 2
 
-
-
-
-
-
+edxapp_all_req_files:
+  - "{{ pre_requirements_file }}"
+  - "{{ post_requirements_file }}"
+  - "{{ repo_requirements_file }}"
+  - "{{ github_requirements_file }}"
+  - "{{ local_requirements_file }}"
+  - "{{ sandbox_base_requirements }}"
+  - "{{ sandbox_local_requirements }}"
+  - "{{ sandbox_post_requirements }}"
+  - "{{ edxapp_code_dir }}/package.json"
+  - "{{ edxapp_code_dir }}/Gemfile"
 
 # TODO: old style variable syntax is necessary
-# here until ansible 1.4
-edxapp_deploy_environment:
+# for lists and dictionaries
+edxapp_environment:
   LANG: "en_US.UTF-8"
   NO_PREREQ_INSTALL: 1
   SKIP_WS_MIGRATIONS: 1
@@ -151,7 +158,6 @@ edxapp_deploy_environment:
   GEM_PATH: $edxapp_gem_root
   PATH: $edxapp_deploy_path
 
-
 edxapp_generic_auth_config:  &edxapp_generic_auth
   AWS_ACCESS_KEY_ID:  $EDXAPP_AWS_ACCESS_KEY_ID
   AWS_SECRET_ACCESS_KEY: $EDXAPP_AWS_SECRET_ACCESS_KEY

playbooks/roles/edxapp/meta/main.yml

@@ -4,3 +4,4 @@ dependencies:
     rbenv_user: "{{ edxapp_user }}"
     rbenv_dir: "{{ edxapp_app_dir }}"
     rbenv_ruby_version: "{{ edxapp_ruby_version }}"
+  - devpi

playbooks/roles/edxapp/tasks/deploy.yml

@@ -35,7 +35,6 @@
   tags:
   - deploy
 
-
 - name: edxapp | git clean after checking out edx-platform
   shell: cd {{edxapp_code_dir}} && git clean -xdf
   when: edx_platform_checkout.changed
@@ -51,16 +50,42 @@
   tags:
   - deploy
 
+- name: edxapp | create checksum for requirements, package.json and Gemfile
+  shell: >
+    /usr/bin/md5sum {{ " ".join(edxapp_all_req_files) }} 2>/dev/null > /var/tmp/edxapp.req.new
+  sudo_user: "{{ edxapp_user }}"
+  ignore_errors: true
+  tags:
+  - deploy
+
+- stat: path=/var/tmp/edxapp.req.new
+  register: new
+  tags: deploy
+
+- stat: path=/var/tmp/edxapp.req.installed
+  register: inst
+  tags: deploy
+
+# Substitute github mirror in all requirements files
+- name: edxapp | Updating requirement files for git mirror
+  command: |
+    /bin/sed -i -e 's/github\.com/{{ COMMON_GIT_MIRROR }}/g' {{ item }}
+  with_items: edxapp_all_req_files
+  sudo_user: "{{ edxapp_user }}"
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
+
+
 # Ruby plays that need to be run after platform updates.
 - name: edxapp | gem install bundler
   shell: >
     gem install bundle
       chdir={{ edxapp_code_dir }}
       executable=/bin/bash
-  environment: "{{ edxapp_deploy_environment }}"
+  environment: "{{ edxapp_environment }}"
   sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
 - name: edxapp | bundle install
   shell: >
@@ -68,37 +93,18 @@
       chdir={{ edxapp_code_dir }}
       executable=/bin/bash
   sudo_user: "{{ edxapp_user }}"
-  environment: "{{ edxapp_deploy_environment }}"
-  tags:
-  - deploy
+  environment: "{{ edxapp_environment }}"
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
 # Node play that need to be run after platform updates.
 - name: edxapp | Install edx-platform npm dependencies
   shell: npm install chdir={{ edxapp_code_dir }}
   sudo_user: "{{ edxapp_user }}"
-  environment: "{{ edxapp_deploy_environment }}"
-  tags:
-  - deploy
+  environment: "{{ edxapp_environment }}"
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
-# Python plays that need to be run after platform updates.
-
-# Substitute github mirror in all requirements files
-#
-- name: edxapp | Updating requirement files for git mirror
-  command: |
-    /bin/sed -i -e 's/github\.com/{{ COMMON_GIT_MIRROR }}/g' {{ item }}
-  with_items:
-    - "{{ pre_requirements_file }}"
-    - "{{ post_requirements_file }}"
-    - "{{ repo_requirements_file }}"
-    - "{{ github_requirements_file }}"
-    - "{{ local_requirements_file }}"
-    - "{{ sandbox_base_requirements }}"
-    - "{{ sandbox_local_requirements }}"
-    - "{{ sandbox_post_requirements }}"
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-    - deploy
 
 # Install the python pre requirements into {{ edxapp_venv_dir }}
 - name : edxapp | install python pre-requirements
@@ -106,20 +112,24 @@
     requirements="{{pre_requirements_file}}"
     virtualenv="{{edxapp_venv_dir}}"
     state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+    extra_args="-i {{ edxapp_pypi_local_mirror }}"
   sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
+  environment: "{{ edxapp_environment }}"
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
 # Install the python modules into {{ edxapp_venv_dir }}
 - name : edxapp | install python base-requirements
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ edxapp_code_dir }} && {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w --use-mirrors -r {{ base_requirements_file }}
+  shell: >
+    {{ edxapp_venv_dir }}/bin/pip install -i {{ edxapp_pypi_local_mirror }} --exists-action w --use-mirrors -r {{ base_requirements_file }}
+    chdir={{ edxapp_code_dir }}
+  environment: "{{ edxapp_environment }}"
   sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
 # Install the python post requirements into {{ edxapp_venv_dir }}
 - name : edxapp | install python post-requirements
@@ -127,24 +137,25 @@
     requirements="{{post_requirements_file}}"
     virtualenv="{{edxapp_venv_dir}}"
     state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
+    extra_args="-i {{ edxapp_pypi_local_mirror }}"
   sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
+  environment: "{{ edxapp_environment }}"
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
 # Install the final python modules into {{ edxapp_venv_dir }}
 - name : edxapp | install python post-post requirements
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ edxapp_code_dir }} && {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w --use-mirrors -r {{ item }}
+  shell: cd {{ edxapp_code_dir }} && {{ edxapp_venv_dir }}/bin/pip install -i {{ edxapp_pypi_local_mirror }} --exists-action w --use-mirrors -r {{ item }}
   with_items:
   - "{{ repo_requirements_file }}"
   - "{{ github_requirements_file }}"
   - "{{ local_requirements_file }}"
   sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
 
 
 # Install the sandbox python modules into {{ edxapp_venv_dir }}
@@ -152,15 +163,21 @@
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ edxapp_code_dir }} && {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w --use-mirrors -r {{ item }}
+  shell: cd {{ edxapp_code_dir }} && {{ edxapp_venv_dir }}/bin/pip install -i {{ edxapp_pypi_local_mirror }} --exists-action w --use-mirrors -r {{ item }}
   with_items:
   - "{{ sandbox_base_requirements }}"
   - "{{ sandbox_local_requirements }}"
   - "{{ sandbox_post_requirements }}"
   when: install_sandbox_reqs_into_regular_venv
   sudo_user: "{{ edxapp_user }}"
-  tags:
-  - deploy
+  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+  tags: deploy
+
+- name: edxapp | create checksum for installed requirements
+  shell: cp /var/tmp/edxapp.req.new /var/tmp/edxapp.req.installed
+  sudo_user: "{{ edxapp_user }}"
+  tags: deploy
+
 
 # https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
 # This is necessary for when syncdb is run and the django_openid_auth module is installed,

playbooks/roles/edxapp/tasks/main.yml

@@ -12,28 +12,21 @@
     name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"
     createhome=no shell=/bin/false
 
-- name: edxapp | create edxapp app dir
+- name: edxapp | create edxapp user dirs
   file: >
     path="{{ item }}" state=directory
     owner="{{ edxapp_user }}" group="{{ common_web_group }}"
   with_items:
     - "{{ edxapp_app_dir }}"
     - "{{ edxapp_venvs_dir }}"
+    - "{{ edxapp_theme_dir }}"
+    - "{{ edxapp_pip_cache_dir }}"
 
 - name: edxapp | create edxapp log dir
   file: >
     path="{{ edxapp_log_dir }}" state=directory
     owner="{{ common_log_user }}" group="{{ common_log_user }}"
 
-- name: edxapp | create edxapp writable dirs
-  file: >
-    path="{{ item }}" state=directory
-    owner="{{ edxapp_user }}" group="{{ edxapp_user }}"
-  with_items:
-    - "{{ edxapp_staticfile_dir }}"
-    - "{{ edxapp_theme_dir }}"
-
-
 - name: edxapp | create web-writable edxapp data dirs
   file: >
     path="{{ item }}" state=directory
@@ -53,10 +46,20 @@
     mode=0750
   with_items: service_variants_enabled
 
+- name: edxapp | setup the edxapp env
+  template: >
+    src=edxapp_env.j2 dest={{ edxapp_app_dir }}/edxapp_env
+    owner={{ edxapp_user }}  group={{ common_web_user }}
+    mode=0644
+
 - include: deploy.yml
 
 - name: edxapp | create a symlink for venv python
   file: >
-    src="{{ edxapp_venv_bin }}/python"
-    dest={{ COMMON_BIN_DIR }}/python.edxapp
+    src="{{ edxapp_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.edxapp
     state=link
+  with_items:
+  - python
+  - pip
+

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -61,7 +61,7 @@
   sudo_user: "{{ edxapp_user }}"
   when: celery_worker is not defined
   with_items: service_variants_enabled
-  environment: "{{ edxapp_deploy_environment }}"
+  environment: "{{ edxapp_environment }}"
   tags:
   - deploy
 

playbooks/roles/edxapp/templates/edxapp_env.j2

+# {{ ansible_managed }}
+{% for name,value in edxapp_environment.items() %}
+{%- if value %}
+export {{ name }}="{{ value }}"
+{%- endif %}                 
+{% endfor %}

playbooks/roles/ora/tasks/main.yml

@@ -30,5 +30,8 @@
 
 - name: ora | create a symlink for venv python
   file: >
-    src="{{ ora_venv_bin }}/python"
-    dest={{ COMMON_BIN_DIR }}/python.ora state=link
+    src="{{ ora_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.ora state=link
+  with_items:
+  - python
+  - pip

playbooks/roles/supervisor/templates/devpi.conf.j2

-[program:devpi-server]
-command={{ devpi_venv_dir }}/bin/devpi-server --port {{ devpi_port }} --serverdir {{ devpi_data_dir }}
-priority=999
-startsecs = 5
-redirect_stderr = True
-autostart=True
-user={{ devpi_user }}

playbooks/roles/xqueue/tasks/main.yml

@@ -44,6 +44,9 @@
 
 - name: xqueue | create a symlink for venv python
   file: >
-    src="{{ xqueue_venv_bin }}/python"
-    dest={{ COMMON_BIN_DIR }}/python.xqueue
+    src="{{ xqueue_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.xqueue
     state=link
+  with_items:
+  - python
+  - pip

playbooks/roles/xserver/tasks/main.yml

@@ -73,7 +73,10 @@
 
 - name: xserver | create a symlink for venv python
   file: >
-    src="{{ xserver_venv_bin }}/python"
-    dest={{ COMMON_BIN_DIR }}/python.xserver
+    src="{{ xserver_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.xserver
     state=link
+  with_item:
+  - python
+  - pip