Commit 0ca7bdd2 by Sven Marnach

Refactor installation of Python requirements.

The current code installs the Python requirements files in a different order
than "paver install_prereqs".  This has been causing problems for the
edx-solutions fork of edx-platform.

The code is also rather repetitive.  This patch cleans the code up a bit, while
at the same time making sure that the order in which the requirements files are
processed matches the order paver uses.

Additionally, the patch extends the list of requirements files which are scanned
for Github URLs by two items that seemed to be missing.
parent c5563061
...@@ -595,18 +595,6 @@ edxapp_sandbox_user: 'sandbox' # I think something about the codejail requires ...@@ -595,18 +595,6 @@ edxapp_sandbox_user: 'sandbox' # I think something about the codejail requires
# apparmor command # apparmor command
edxapp_aa_command: "{% if EDXAPP_SANDBOX_ENFORCE %}aa-enforce{% else %}aa-complain{% endif %}" edxapp_aa_command: "{% if EDXAPP_SANDBOX_ENFORCE %}aa-enforce{% else %}aa-complain{% endif %}"
# all edxapp requirements files
edxapp_requirements_with_github_urls:
- "{{ pre_requirements_file }}"
- "{{ post_requirements_file }}"
- "{{ base_requirements_file }}"
- "{{ paver_requirements_file }}"
- "{{ github_requirements_file }}"
- "{{ sandbox_post_requirements }}"
- "{{ sandbox_local_requirements }}"
- "{{ sandbox_base_requirements }}"
# TODO: old style variable syntax is necessary # TODO: old style variable syntax is necessary
# for lists and dictionaries # for lists and dictionaries
edxapp_helper_scripts: edxapp_helper_scripts:
...@@ -995,19 +983,48 @@ edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-platform.git" ...@@ -995,19 +983,48 @@ edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-platform.git"
# `edx_platform_version` can be anything that git recognizes as a commit # `edx_platform_version` can be anything that git recognizes as a commit
# reference, including a tag, a branch name, or a commit hash # reference, including a tag, a branch name, or a commit hash
edx_platform_version: 'release' edx_platform_version: 'release'
local_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/local.txt"
pre_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/pre.txt" pre_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/pre.txt"
post_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/post.txt" github_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/github.txt"
base_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/base.txt"
custom_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/custom.txt" custom_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/custom.txt"
local_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/local.txt"
base_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/base.txt"
post_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/post.txt"
paver_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/paver.txt" paver_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/paver.txt"
github_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/github.txt"
private_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/edx-private.txt" private_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/edx-private.txt"
sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/base.txt" sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/base.txt"
sandbox_local_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/local.txt" sandbox_local_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/local.txt"
sandbox_post_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/post.txt" sandbox_post_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/post.txt"
# The Python requirements files in the order they should be installed. This order should
# match the order of PYTHON_REQ_FILES in edx-platform/pavelib/prereqs.py. (Also check the
# edx-solutions fork of edx-platform when if you consider reordering this list.)
# private_requirements_file is not included in this list, since it is only installed
# conditionally based on the value of EDXAPP_INSTALL_PRIVATE_REQUIREMENTS.
edxapp_requirements_files:
- "{{ pre_requirements_file }}"
- "{{ github_requirements_file }}"
- "{{ custom_requirements_file }}"
- "{{ local_requirements_file }}"
- "{{ base_requirements_file }}"
- "{{ post_requirements_file }}"
- "{{ paver_requirements_file }}"
# All edxapp requirements files potentially containing Github URLs. When using a custom
# Github mirror, occurrences of "github.com" are replaced by the custom mirror in these
# files.
edxapp_requirements_with_github_urls:
- "{{ pre_requirements_file }}"
- "{{ github_requirements_file }}"
- "{{ custom_requirements_file }}"
- "{{ base_requirements_file }}"
- "{{ post_requirements_file }}"
- "{{ paver_requirements_file }}"
- "{{ private_requirements_file }}"
- "{{ sandbox_post_requirements }}"
- "{{ sandbox_local_requirements }}"
- "{{ sandbox_base_requirements }}"
edxapp_chrislea_ppa: "ppa:chris-lea/node.js" edxapp_chrislea_ppa: "ppa:chris-lea/node.js"
edxapp_debian_pkgs: edxapp_debian_pkgs:
......
...@@ -87,7 +87,7 @@ ...@@ -87,7 +87,7 @@
- install - install
- install:code - install:code
- name: Stat each requirements file to ensure it exists - name: Stat each requirements file with Github URLs to ensure it exists
stat: path="{{ item }}" stat: path="{{ item }}"
with_items: "{{ edxapp_requirements_with_github_urls }}" with_items: "{{ edxapp_requirements_with_github_urls }}"
register: requirement_file_stats register: requirement_file_stats
...@@ -162,105 +162,41 @@ ...@@ -162,105 +162,41 @@
- install - install
- install:app-requirements - install:app-requirements
# Install the python pre requirements into {{ edxapp_venv_dir }} - name: Create the virtualenv to install the Python requirements
- name : install python pre-requirements command: >
pip: > virtualenv {{ edxapp_venv_dir }}
requirements="{{ pre_requirements_file }}"
virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
sudo_user: "{{ edxapp_user }}"
environment: "{{ edxapp_environment }}"
tags:
- install
- install:app-requirements
# Install the python modules into {{ edxapp_venv_dir }}
- name : install python base-requirements
# Need to use shell rather than pip so that we can maintain the context of our current working directory; some
# requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
# installs everything into that virtual environment.
shell: >
{{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ base_requirements_file }}
chdir={{ edxapp_code_dir }} chdir={{ edxapp_code_dir }}
environment: "{{ edxapp_environment }}" creates={{ edxapp_venv_dir }}/bin/pip
sudo_user: "{{ edxapp_user }}"
tags:
- install
- install:app-requirements
- stat: path="{{ post_requirements_file }}"
register: post_requirements
sudo_user: "{{ edxapp_user }}"
tags:
- install
- install:app-requirements
# Install the python post requirements into {{ edxapp_venv_dir }}
- name : install python post-requirements
pip: >
requirements="{{ post_requirements_file }}"
virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
sudo_user: "{{ edxapp_user }}" sudo_user: "{{ edxapp_user }}"
environment: "{{ edxapp_environment }}" environment: "{{ edxapp_environment }}"
when: post_requirements.stat.exists
tags: tags:
- install - install
- install:app-requirements - install:app-requirements
# Install the python paver requirements into {{ edxapp_venv_dir }}
- name : install python paver-requirements
pip: >
requirements="{{ paver_requirements_file }}"
virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
sudo_user: "{{ edxapp_user }}"
environment: "{{ edxapp_environment }}"
tags:
- install
- install:app-requirements
# Install the python custom requirements into {{ edxapp_venv_dir }}
- stat: path="{{ custom_requirements_file }}" - name: Stat each Python requirements file to ensure it exists
register: custom_requirements stat: path="{{ item }}"
sudo_user: "{{ edxapp_user }}" with_items: "{{ edxapp_requirements_files }}"
tags: register: python_requirement_files
- install
- install:app-requirements
- name : install python custom-requirements
pip: >
requirements="{{ custom_requirements_file }}"
virtualenv="{{ edxapp_venv_dir }}"
state=present
extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
sudo_user: "{{ edxapp_user }}"
environment: "{{ edxapp_environment }}"
when: custom_requirements.stat.exists
tags: tags:
- install - install
- install:app-requirements - install:app-requirements
# Install the final python modules into {{ edxapp_venv_dir }} # Install the python requirements into {{ edxapp_venv_dir }}
- name : install python post-post requirements - name : install python requirements
# Need to use shell rather than pip so that we can maintain the context of our current working directory; some # Need to use command rather than pip so that we can maintain the context of our current working directory; some
# requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
# installs everything into that virtual environment. # installs everything into that virtual environment.
shell: > command: >
{{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ item }} {{ edxapp_venv_dir }}/bin/pip install -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ item.item }}
chdir={{ edxapp_code_dir }} chdir={{ edxapp_code_dir }}
with_items:
- "{{ github_requirements_file }}"
- "{{ local_requirements_file }}"
sudo_user: "{{ edxapp_user }}" sudo_user: "{{ edxapp_user }}"
environment: "{{ edxapp_environment }}"
when: item.stat.exists
with_items: "{{ python_requirement_files.results }}"
tags: tags:
- install - install
- install:app-requirements - install:app-requirements
# Private requriements require a ssh key to install, use the same key as the private key for edx-platform # Private requriements require a ssh key to install, use the same key as the private key for edx-platform
# If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true # If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true
- name : install python private requirements - name : install python private requirements
...@@ -279,7 +215,7 @@ ...@@ -279,7 +215,7 @@
tags: tags:
- install - install
- install:app-requirements - install:app-requirements
# Install any custom extra requirements if defined in EDXAPP_EXTRA_REQUIREMENTS. # Install any custom extra requirements if defined in EDXAPP_EXTRA_REQUIREMENTS.
- name: install python extra requirements - name: install python extra requirements
pip: > pip: >
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment