Skip to content

C
configuration
Commit 134935a7 by Ben Patterson
Options

Merge pull request #1397 from edx/benp/remove-wheels-jenkins-worker 

Remove usage of wheels during jenkins worker provisioning.
parents 155904e5 cb076915
Showing with 28 additions and 156 deletions
playbooks/roles/jenkins_worker/defaults/main.yml
......@@ -29,104 +29,3 @@ jenkins_ruby_version: "1.9.3-p374"
# JSCover direct download URL
jscover_url: "http://files.edx.org/testeng/JSCover-1.0.2.zip"
jscover_version: "1.0.2"
# Python
jenkins_venv: "{{ jenkins_home }}/wheel_venv"
jenkins_pip: "{{ jenkins_venv }}/bin/pip"
jenkins_wheel_dir: "{{ jenkins_home }}/wheelhouse"
jenkins_wheels:
- { pkg: "numpy==1.6.2", wheel: "numpy-1.6.2-cp27-none-linux_x86_64.whl" }
- { pkg: "django-celery==3.0.17", wheel: "django_celery-3.0.17-py27-none-any.whl" }
- { pkg: "beautifulsoup4==4.1.3", wheel: "beautifulsoup4-4.1.3-py27-none-any.whl"}
- { pkg: "beautifulsoup==3.2.1", wheel: "BeautifulSoup-3.2.1-py27-none-any.whl" }
- { pkg: "bleach==1.4", wheel: "bleach-1.4-py27-none-any.whl" }
- { pkg: "html5lib==0.999", wheel: "html5lib-0.999-py27-none-any.whl" }
- { pkg: "boto==2.13.3", wheel: "boto-2.13.3-py27-none-any.whl" }
- { pkg: "celery==3.0.19", wheel: "celery-3.0.19-py27-none-any.whl" }
- { pkg: "dealer==0.2.3", wheel: "dealer-0.2.3-py27-none-any.whl" }
- { pkg: "django-countries==1.5", wheel: "django_countries-1.5-py27-none-any.whl" }
- { pkg: "django-filter==0.6.0", wheel: "django_filter-0.6-py27-none-any.whl" }
- { pkg: "django-followit==0.0.3", wheel: "django_followit-0.0.3-py27-none-any.whl" }
- { pkg: "django-kombu==0.9.4", wheel: "kombu-2.5.16-py27-none-any.whl" }
- { pkg: "django-mako==0.1.5pre", wheel: "django_mako-0.1.5pre-py27-none-any.whl" }
- { pkg: "django-model-utils==1.4.0", wheel: "django_model_utils-1.4.0-py27-none-any.whl" }
- { pkg: "django-masquerade==0.1.6", wheel: "django_masquerade-0.1.6-py27-none-any.whl" }
- { pkg: "django-mptt==0.5.5", wheel: "django_mptt-0.5.5-py27-none-any.whl" }
- { pkg: "django-openid-auth==0.4", wheel: "python_openid-2.2.5-py27-none-any.whl" }
- { pkg: "django-robots==0.9.1", wheel: "django_robots-0.9.1-py27-none-any.whl" }
- { pkg: "django-sekizai==0.6.1", wheel: "django_sekizai-0.6.1-py27-none-any.whl" }
- { pkg: "django-ses==0.4.1", wheel: "django_ses-0.4.1-py27-none-any.whl" }
- { pkg: "django-storages==1.1.5", wheel: "django_storages-1.1.5-py27-none-any.whl" }
- { pkg: "django-method-override==0.1.0", wheel: "django_method_override-0.1.0-py27-none-any.whl" }
- { pkg: "djangorestframework==2.3.5", wheel: "djangorestframework-2.3.5-py27-none-any.whl" }
- { pkg: "django==1.4.8", wheel: "Django-1.4.8-py27-none-any.whl" }
- { pkg: "feedparser==5.1.3", wheel: "feedparser-5.1.3-py27-none-any.whl" }
- { pkg: "fs==0.4.0", wheel: "fs-0.4.0-py27-none-any.whl" }
- { pkg: "GitPython==0.3.2.RC1", wheel: "GitPython-0.3.2.RC1-py27-none-any.whl" }
- { pkg: "glob2==0.3", wheel: "glob2-0.3-py27-none-any.whl" }
- { pkg: "gunicorn==0.17.4", wheel: "gunicorn-0.17.4-py27-none-any.whl" }
- { pkg: "lazy==1.1", wheel: "lazy-1.1-py27-none-any.whl" }
- { pkg: "lxml==3.0.1", wheel: "lxml-3.0.1-cp27-none-linux_x86_64.whl" }
- { pkg: "mako==0.9.1", wheel: "Mako-0.9.1-py2.py3-none-any.whl" }
- { pkg: "Markdown==2.2.1", wheel: "Markdown-2.2.1-py27-none-any.whl" }
- { pkg: "mongoengine==0.7.10", wheel: "mongoengine-0.7.10-py27-none-any.whl" }
- { pkg: "networkx==1.7", wheel: "networkx-1.7-py27-none-any.whl" }
- { pkg: "nltk==2.0.4", wheel: "nltk-2.0.4-py27-none-any.whl" }
- { pkg: "oauthlib==0.6.3", wheel: "oauthlib-0.6.3-py27-none-any.whl" }
- { pkg: "paramiko==1.9.0", wheel: "paramiko-1.9.0-py27-none-any.whl" }
- { pkg: "path.py==3.0.1", wheel: "path.py-3.0.1-py27-none-any.whl" }
- { pkg: "Pillow==1.7.8", wheel: "Pillow-1.7.8-cp27-none-linux_x86_64.whl" }
- { pkg: "polib==1.0.3", wheel: "polib-1.0.3-py27-none-any.whl" }
- { pkg: "pycrypto>=2.6", wheel: "pycrypto-2.6.1-cp27-none-linux_x86_64.whl" }
- { pkg: "pygments==1.6", wheel: "Pygments-1.6-py27-none-any.whl" }
- { pkg: "pygraphviz==1.1", wheel: "pygraphviz-1.1-cp27-none-linux_x86_64.whl" }
- { pkg: "pymongo==2.4.1", wheel: "pymongo-2.4.1-cp27-none-linux_x86_64.whl" }
- { pkg: "pyparsing==1.5.6", wheel: "pyparsing-1.5.6-py27-none-any.whl" }
- { pkg: "python-memcached==1.48", wheel: "python_memcached-1.48-py27-none-any.whl" }
- { pkg: "python-openid==2.2.5", wheel: "python_openid-2.2.5-py27-none-any.whl" }
- { pkg: "python-dateutil==2.1", wheel: "python_dateutil-2.1-py27-none-any.whl" }
- { pkg: "python-social-auth==0.1.21", wheel: "python_social_auth-0.1.21-py27-none-any.whl" }
- { pkg: "pytz==2012h", wheel: "pytz-2012h-py27-none-any.whl" }
- { pkg: "pysrt==0.4.7", wheel: "pysrt-0.4.7-py27-none-any.whl" }
- { pkg: "PyYAML==3.10", wheel: "PyYAML-3.10-cp27-none-linux_x86_64.whl" }
- { pkg: "requests==2.3.0", wheel: "requests-2.3.0-py27-none-any.whl" }
- { pkg: "scipy==0.11.0", wheel: "scipy-0.11.0-cp27-none-linux_x86_64.whl" }
- { pkg: "Shapely==1.2.16", wheel: "Shapely-1.2.16-cp27-none-linux_x86_64.whl" }
- { pkg: "singledispatch==3.4.0.2", wheel: "singledispatch-3.4.0.2-py27-none-any.whl" }
- { pkg: "sorl-thumbnail==11.12", wheel: "sorl_thumbnail-11.12-py27-none-any.whl" }
- { pkg: "South==0.7.6", wheel: "South-0.7.6-py27-none-any.whl" }
- { pkg: "sympy==0.7.1", wheel: "sympy-0.7.1-py27-none-any.whl" }
- { pkg: "xmltodict==0.4.1", wheel: "xmltodict-0.4.1-py27-none-any.whl" }
- { pkg: "django-ratelimit-backend==0.6", wheel: "django_ratelimit_backend-0.6-py27-none-any.whl" }
- { pkg: "unicodecsv==0.9.4", wheel: "unicodecsv-0.9.4-py27-none-any.whl" }
- { pkg: "ipython==0.13.1", wheel: "ipython-0.13.1-py27-none-any.whl" }
- { pkg: "watchdog==0.6.0", wheel: "watchdog-0.6.0-py27-none-any.whl" }
- { pkg: "dogapi==1.2.1", wheel: "dogapi-1.2.1-py27-none-any.whl" }
- { pkg: "newrelic==2.4.0.4", wheel: "newrelic-2.4.0.4-cp27-none-linux_x86_64.whl" }
- { pkg: "sphinx==1.1.3", wheel: "Sphinx-1.1.3-py27-none-any.whl" }
- { pkg: "sphinx_rtd_theme==0.1.5", wheel: "sphinx_rtd_theme-0.1.5-py27-none-any.whl" }
- { pkg: "Babel==1.3", wheel: "Babel-1.3-py27-none-any.whl" }
- { pkg: "transifex-client==0.10", wheel: "transifex_client-0.10-py27-none-any.whl" }
- { pkg: "django_debug_toolbar", wheel: "django_debug_toolbar-0.11.0-py2.py3-none-any.whl" }
- { pkg: "django-debug-toolbar-mongo", wheel: "django_debug_toolbar_mongo-0.1.10-py27-none-any.whl" }
- { pkg: "chrono==1.0.2", wheel: "chrono-1.0.2-py2.py3-none-any.whl" }
- { pkg: "coverage==3.6", wheel: "coverage-3.6-cp27-none-linux_x86_64.whl" }
- { pkg: "ddt==0.7.1", wheel: "ddt-0.7.1-py27-none-any.whl" }
- { pkg: "django-crum==0.5", wheel: "django_crum-0.5-py27-none-any.whl" }
- { pkg: "django_nose==1.1", wheel: "django_nose-1.1-py27-none-any.whl" }
- { pkg: "factory_boy==2.1.2", wheel: "factory_boy-2.1.2-py27-none-any.whl" }
- { pkg: "freezegun==0.1.11", wheel: "freezegun-0.1.11-py27-none-any.whl" }
- { pkg: "mock==1.0.1", wheel: "mock-1.0.1-py27-none-any.whl" }
- { pkg: "nosexcover==1.0.7", wheel: "nosexcover-1.0.7-py27-none-any.whl" }
- { pkg: "pep8==1.4.5", wheel: "pep8-1.4.5-py27-none-any.whl" }
- { pkg: "pylint==0.28", wheel: "pylint-0.28.0-py27-none-any.whl" }
- { pkg: "python-subunit==0.0.16", wheel: "python_subunit-0.0.16-py27-none-any.whl" }
- { pkg: "rednose==0.3", wheel: "rednose-0.3-py27-none-any.whl" }
- { pkg: "selenium==2.39.0", wheel: "selenium-2.39.0-py27-none-any.whl" }
- { pkg: "splinter==0.5.4", wheel: "splinter-0.5.4-py27-none-any.whl" }
- { pkg: "testtools==0.9.34", wheel: "testtools-0.9.34-py27-none-any.whl" }
- { pkg: "Paver==1.2.1", wheel: "Paver-1.2.1-py27-none-any.whl" }
- { pkg: "psutil==1.2.1", wheel: "psutil-1.2.1-cp27-none-linux_x86_64.whl" }
- { pkg: "lazy==1.1", wheel: "lazy-1.1-py27-none-any.whl" }
- { pkg: "path.py==3.0.1", wheel: "path.py-3.0.1-py27-none-any.whl" }
- { pkg: "MySQL-python==1.2.5", wheel: "MySQL_python-1.2.5-cp27-none-linux_x86_64.whl" }
playbooks/roles/jenkins_worker/tasks/python.yml
......@@ -17,45 +17,31 @@
owner=root group=root
mode=755
# Create wheelhouse to enable fast virtualenv creation
- name: Create wheel virtualenv
command: /usr/local/bin/virtualenv {{ jenkins_venv }} creates={{ jenkins_venv }}
# Create a virtualenv for edx-platform by installing the requirements
# and packaging the virtualenv.
# A shallow clone is created off of master. The depth setting
# refers to the --depth-setting of git clone. A value of 1
# will truncate all history prior to the last revision.
- name: Create shallow clone of edx-platform
git: >
repo=https://github.com/edx/edx-platform.git
dest={{ jenkins_home }}/shallow-clone
version=master
depth=1
sudo_user: "{{ jenkins_user }}"
- name: Install wheel
pip: name=wheel virtualenv={{ jenkins_venv }} virtualenv_command=/usr/local/bin/virtualenv
sudo_user: "{{ jenkins_user }}"
- name: Create wheelhouse dir
file:
path={{ jenkins_wheel_dir }} state=directory
owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
# (need to install each one in the venv to satisfy dependencies)
- name: Create wheel archives
shell:
"{{ jenkins_pip }} wheel --wheel-dir={{ jenkins_wheel_dir }} \"${item.pkg}\" &&
{{ jenkins_pip }} install --use-wheel --no-index --find-links={{ jenkins_wheel_dir }} \"${item.pkg}\"
creates={{ jenkins_wheel_dir }}/${item.wheel}"
sudo_user: "{{ jenkins_user }}"
with_items: jenkins_wheels
- name: Add wheel_venv.sh script
template:
src=wheel_venv.sh.j2 dest={{ jenkins_home }}/wheel_venv.sh
owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
# Run the wheel_venv.sh script for the first time
# This was previously done in the Jenkins global
# configuration as part of the AMI Init script.
# Moving here so that we can archive a clean snapshot
# of the virtualenv with only the defined packages
# from jenkins_wheels.
- name: Run the wheel_venv.sh script
command: >
./wheel_venv.sh edx-venv
chdir={{ jenkins_home }}
creates={{ jenkins_home }}/edx-venv
- name: Install edx-platform requirements
pip: >
requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
extra_args="--exists-action w"
virtualenv={{ jenkins_home }}/edx-venv
virtualenv_command=virtualenv-2.7
with_items:
- pre.txt
- github.txt
- base.txt
- post.txt
- paver.txt
sudo_user: "{{ jenkins_user }}"
# Archive the current state of the virtualenv
......@@ -66,5 +52,9 @@
command: >
tar -cpzf edx-venv_clean.tar.gz edx-venv
chdir={{ jenkins_home }}
creates={{ jenkins_home }}/edx-venv_clean.tar.gz
sudo_user: "{{ jenkins_user }}"
# Remove the shallow-clone directory now that we archive
# done with it
- name: Remove shallow-clone
file: path={{ jenkins_home }}/shallow-clone state=absent
playbooks/roles/jenkins_worker/templates/wheel_venv.sh.j2 deleted 100644 → 0
#! /usr/bin/env bash
if [ $# -ne 1 ]; then
echo "Usage: $0 VENV_DIR"
exit 1
fi
# Create and activate the new virtualenv
VENV=$1
mkdir -p $VENV
/usr/local/bin/virtualenv $VENV
. $VENV/bin/activate
# Install each available wheel archive
ls {{ jenkins_wheel_dir }} | cut -d- -f1 | while read line ; do
pip install --use-wheel --no-index --find-links={{ jenkins_wheel_dir }} $line ;
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment