Refactor installation of Python requirements.

The current code installs the Python requirements files in a different order
than "paver install_prereqs".  This has been causing problems for the
edx-solutions fork of edx-platform.

The code is also rather repetitive.  This patch cleans the code up a bit, while
at the same time making sure that the order in which the requirements files are
processed matches the order paver uses.

Additionally, the patch extends the list of requirements files which are scanned
for Github URLs by two items that seemed to be missing.

playbooks/roles/edxapp/defaults/main.yml

@@ -595,18 +595,6 @@ edxapp_sandbox_user: 'sandbox'  # I think something about the codejail requires
 # apparmor command
 edxapp_aa_command: "{% if EDXAPP_SANDBOX_ENFORCE %}aa-enforce{% else %}aa-complain{% endif %}"
 
-# all edxapp requirements files
-edxapp_requirements_with_github_urls:
-  - "{{ pre_requirements_file }}"
-  - "{{ post_requirements_file }}"
-  - "{{ base_requirements_file }}"
-  - "{{ paver_requirements_file }}"
-  - "{{ github_requirements_file }}"
-  - "{{ sandbox_post_requirements }}"
-  - "{{ sandbox_local_requirements }}"
-  - "{{ sandbox_base_requirements }}"
-
-
 # TODO: old style variable syntax is necessary
 # for lists and dictionaries
 edxapp_helper_scripts:
@@ -995,19 +983,48 @@ edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-platform.git"
 # `edx_platform_version` can be anything that git recognizes as a commit
 # reference, including a tag, a branch name, or a commit hash
 edx_platform_version: 'release'
-local_requirements_file:  "{{ edxapp_code_dir }}/requirements/edx/local.txt"
 pre_requirements_file:    "{{ edxapp_code_dir }}/requirements/edx/pre.txt"
-post_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/post.txt"
-base_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/base.txt"
+github_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/github.txt"
 custom_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/custom.txt"
+local_requirements_file:  "{{ edxapp_code_dir }}/requirements/edx/local.txt"
+base_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/base.txt"
+post_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/post.txt"
 paver_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/paver.txt"
-github_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/github.txt"
 private_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/edx-private.txt"
 
 sandbox_base_requirements:  "{{ edxapp_code_dir }}/requirements/edx-sandbox/base.txt"
 sandbox_local_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/local.txt"
 sandbox_post_requirements:  "{{ edxapp_code_dir }}/requirements/edx-sandbox/post.txt"
 
+# The Python requirements files in the order they should be installed.  This order should
+# match the order of PYTHON_REQ_FILES in edx-platform/pavelib/prereqs.py.  (Also check the
+# edx-solutions fork of edx-platform when if you consider reordering this list.)
+# private_requirements_file is not included in this list, since it is only installed
+# conditionally based on the value of EDXAPP_INSTALL_PRIVATE_REQUIREMENTS.
+edxapp_requirements_files:
+  - "{{ pre_requirements_file }}"
+  - "{{ github_requirements_file }}"
+  - "{{ custom_requirements_file }}"
+  - "{{ local_requirements_file }}"
+  - "{{ base_requirements_file }}"
+  - "{{ post_requirements_file }}"
+  - "{{ paver_requirements_file }}"
+
+# All edxapp requirements files potentially containing Github URLs.  When using a custom
+# Github mirror, occurrences of "github.com" are replaced by the custom mirror in these
+# files.
+edxapp_requirements_with_github_urls:
+  - "{{ pre_requirements_file }}"
+  - "{{ github_requirements_file }}"
+  - "{{ custom_requirements_file }}"
+  - "{{ base_requirements_file }}"
+  - "{{ post_requirements_file }}"
+  - "{{ paver_requirements_file }}"
+  - "{{ private_requirements_file }}"
+  - "{{ sandbox_post_requirements }}"
+  - "{{ sandbox_local_requirements }}"
+  - "{{ sandbox_base_requirements }}"
+
 edxapp_chrislea_ppa: "ppa:chris-lea/node.js"
 
 edxapp_debian_pkgs:

playbooks/roles/edxapp/tasks/deploy.yml

@@ -87,7 +87,7 @@
     - install
     - install:code
 
-- name: Stat each requirements file to ensure it exists
+- name: Stat each requirements file with Github URLs to ensure it exists
   stat: path="{{ item }}"
   with_items: "{{ edxapp_requirements_with_github_urls }}"
   register: requirement_file_stats
@@ -162,105 +162,41 @@
     - install
     - install:app-requirements
 
-# Install the python pre requirements into {{ edxapp_venv_dir }}
-- name : install python pre-requirements
-  pip: >
-    requirements="{{ pre_requirements_file }}"
-    virtualenv="{{ edxapp_venv_dir }}"
-    state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
-  sudo_user: "{{ edxapp_user }}"
-  environment: "{{ edxapp_environment }}"
-  tags:
-    - install
-    - install:app-requirements
-
-# Install the python modules into {{ edxapp_venv_dir }}
-- name : install python base-requirements
-  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
-  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
-  # installs everything into that virtual environment.
-  shell: >
-    {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ base_requirements_file }}
+- name: Create the virtualenv to install the Python requirements
+  command: >
+    virtualenv {{ edxapp_venv_dir }}
     chdir={{ edxapp_code_dir }}
-  environment: "{{ edxapp_environment }}"
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-    - install
-    - install:app-requirements
-        
-- stat: path="{{ post_requirements_file }}"
-  register: post_requirements
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-    - install
-    - install:app-requirements
-    
-# Install the python post requirements into {{ edxapp_venv_dir }}
-- name : install python post-requirements
-  pip: >
-    requirements="{{ post_requirements_file }}"
-    virtualenv="{{ edxapp_venv_dir }}"
-    state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
+    creates={{ edxapp_venv_dir }}/bin/pip
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
-  when: post_requirements.stat.exists
   tags:
     - install
     - install:app-requirements
-    
-# Install the python paver requirements into {{ edxapp_venv_dir }}
-- name : install python paver-requirements
-  pip: >
-    requirements="{{ paver_requirements_file }}"
-    virtualenv="{{ edxapp_venv_dir }}"
-    state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
-  sudo_user: "{{ edxapp_user }}"
-  environment: "{{ edxapp_environment }}"
-  tags:
-    - install
-    - install:app-requirements
-    
-# Install the python custom requirements into {{ edxapp_venv_dir }}
 
-- stat: path="{{ custom_requirements_file }}"
-  register: custom_requirements
-  sudo_user: "{{ edxapp_user }}"
-  tags:
-    - install
-    - install:app-requirements
-    
-- name : install python custom-requirements
-  pip: >
-    requirements="{{ custom_requirements_file }}"
-    virtualenv="{{ edxapp_venv_dir }}"
-    state=present
-    extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
-  sudo_user: "{{ edxapp_user }}"
-  environment: "{{ edxapp_environment }}"
-  when: custom_requirements.stat.exists
+- name: Stat each Python requirements file to ensure it exists
+  stat: path="{{ item }}"
+  with_items: "{{ edxapp_requirements_files }}"
+  register: python_requirement_files
   tags:
     - install
     - install:app-requirements
-    
-# Install the final python modules into {{ edxapp_venv_dir }}
-- name : install python post-post requirements
-  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
+
+# Install the python requirements into {{ edxapp_venv_dir }}
+- name : install python requirements
+  # Need to use command rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: >
-    {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ item }}
+  command: >
+    {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ item.item }}
     chdir={{ edxapp_code_dir }}
-  with_items:
-  - "{{ github_requirements_file }}"
-  - "{{ local_requirements_file }}"
   sudo_user: "{{ edxapp_user }}"
+  environment: "{{ edxapp_environment }}"
+  when: item.stat.exists
+  with_items: "{{ python_requirement_files.results }}"
   tags:
     - install
     - install:app-requirements
-    
+
 # Private requriements require a ssh key to install, use the same key as the private key for edx-platform
 # If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true
 - name : install python private requirements
@@ -279,7 +215,7 @@
   tags:
     - install
     - install:app-requirements
-    
+
 # Install any custom extra requirements if defined in EDXAPP_EXTRA_REQUIREMENTS.
 - name: install python extra requirements
   pip: >