Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
C
configuration
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
OpenEdx
configuration
Commits
4a292071
Commit
4a292071
authored
Aug 11, 2014
by
Ben Patterson
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Remove usage of wheels during jenkins worker provisioning.
parent
92994a30
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
20 additions
and
156 deletions
+20
-156
playbooks/roles/jenkins_worker/defaults/main.yml
+0
-101
playbooks/roles/jenkins_worker/tasks/python.yml
+20
-38
playbooks/roles/jenkins_worker/templates/wheel_venv.sh.j2
+0
-17
No files found.
playbooks/roles/jenkins_worker/defaults/main.yml
View file @
4a292071
...
...
@@ -29,104 +29,3 @@ jenkins_ruby_version: "1.9.3-p374"
# JSCover direct download URL
jscover_url
:
"
http://files.edx.org/testeng/JSCover-1.0.2.zip"
jscover_version
:
"
1.0.2"
# Python
jenkins_venv
:
"
{{
jenkins_home
}}/wheel_venv"
jenkins_pip
:
"
{{
jenkins_venv
}}/bin/pip"
jenkins_wheel_dir
:
"
{{
jenkins_home
}}/wheelhouse"
jenkins_wheels
:
-
{
pkg
:
"
numpy==1.6.2"
,
wheel
:
"
numpy-1.6.2-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
django-celery==3.0.17"
,
wheel
:
"
django_celery-3.0.17-py27-none-any.whl"
}
-
{
pkg
:
"
beautifulsoup4==4.1.3"
,
wheel
:
"
beautifulsoup4-4.1.3-py27-none-any.whl"
}
-
{
pkg
:
"
beautifulsoup==3.2.1"
,
wheel
:
"
BeautifulSoup-3.2.1-py27-none-any.whl"
}
-
{
pkg
:
"
bleach==1.4"
,
wheel
:
"
bleach-1.4-py27-none-any.whl"
}
-
{
pkg
:
"
html5lib==0.999"
,
wheel
:
"
html5lib-0.999-py27-none-any.whl"
}
-
{
pkg
:
"
boto==2.13.3"
,
wheel
:
"
boto-2.13.3-py27-none-any.whl"
}
-
{
pkg
:
"
celery==3.0.19"
,
wheel
:
"
celery-3.0.19-py27-none-any.whl"
}
-
{
pkg
:
"
dealer==0.2.3"
,
wheel
:
"
dealer-0.2.3-py27-none-any.whl"
}
-
{
pkg
:
"
django-countries==1.5"
,
wheel
:
"
django_countries-1.5-py27-none-any.whl"
}
-
{
pkg
:
"
django-filter==0.6.0"
,
wheel
:
"
django_filter-0.6-py27-none-any.whl"
}
-
{
pkg
:
"
django-followit==0.0.3"
,
wheel
:
"
django_followit-0.0.3-py27-none-any.whl"
}
-
{
pkg
:
"
django-kombu==0.9.4"
,
wheel
:
"
kombu-2.5.16-py27-none-any.whl"
}
-
{
pkg
:
"
django-mako==0.1.5pre"
,
wheel
:
"
django_mako-0.1.5pre-py27-none-any.whl"
}
-
{
pkg
:
"
django-model-utils==1.4.0"
,
wheel
:
"
django_model_utils-1.4.0-py27-none-any.whl"
}
-
{
pkg
:
"
django-masquerade==0.1.6"
,
wheel
:
"
django_masquerade-0.1.6-py27-none-any.whl"
}
-
{
pkg
:
"
django-mptt==0.5.5"
,
wheel
:
"
django_mptt-0.5.5-py27-none-any.whl"
}
-
{
pkg
:
"
django-openid-auth==0.4"
,
wheel
:
"
python_openid-2.2.5-py27-none-any.whl"
}
-
{
pkg
:
"
django-robots==0.9.1"
,
wheel
:
"
django_robots-0.9.1-py27-none-any.whl"
}
-
{
pkg
:
"
django-sekizai==0.6.1"
,
wheel
:
"
django_sekizai-0.6.1-py27-none-any.whl"
}
-
{
pkg
:
"
django-ses==0.4.1"
,
wheel
:
"
django_ses-0.4.1-py27-none-any.whl"
}
-
{
pkg
:
"
django-storages==1.1.5"
,
wheel
:
"
django_storages-1.1.5-py27-none-any.whl"
}
-
{
pkg
:
"
django-method-override==0.1.0"
,
wheel
:
"
django_method_override-0.1.0-py27-none-any.whl"
}
-
{
pkg
:
"
djangorestframework==2.3.5"
,
wheel
:
"
djangorestframework-2.3.5-py27-none-any.whl"
}
-
{
pkg
:
"
django==1.4.8"
,
wheel
:
"
Django-1.4.8-py27-none-any.whl"
}
-
{
pkg
:
"
feedparser==5.1.3"
,
wheel
:
"
feedparser-5.1.3-py27-none-any.whl"
}
-
{
pkg
:
"
fs==0.4.0"
,
wheel
:
"
fs-0.4.0-py27-none-any.whl"
}
-
{
pkg
:
"
GitPython==0.3.2.RC1"
,
wheel
:
"
GitPython-0.3.2.RC1-py27-none-any.whl"
}
-
{
pkg
:
"
glob2==0.3"
,
wheel
:
"
glob2-0.3-py27-none-any.whl"
}
-
{
pkg
:
"
gunicorn==0.17.4"
,
wheel
:
"
gunicorn-0.17.4-py27-none-any.whl"
}
-
{
pkg
:
"
lazy==1.1"
,
wheel
:
"
lazy-1.1-py27-none-any.whl"
}
-
{
pkg
:
"
lxml==3.0.1"
,
wheel
:
"
lxml-3.0.1-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
mako==0.9.1"
,
wheel
:
"
Mako-0.9.1-py2.py3-none-any.whl"
}
-
{
pkg
:
"
Markdown==2.2.1"
,
wheel
:
"
Markdown-2.2.1-py27-none-any.whl"
}
-
{
pkg
:
"
mongoengine==0.7.10"
,
wheel
:
"
mongoengine-0.7.10-py27-none-any.whl"
}
-
{
pkg
:
"
networkx==1.7"
,
wheel
:
"
networkx-1.7-py27-none-any.whl"
}
-
{
pkg
:
"
nltk==2.0.4"
,
wheel
:
"
nltk-2.0.4-py27-none-any.whl"
}
-
{
pkg
:
"
oauthlib==0.6.3"
,
wheel
:
"
oauthlib-0.6.3-py27-none-any.whl"
}
-
{
pkg
:
"
paramiko==1.9.0"
,
wheel
:
"
paramiko-1.9.0-py27-none-any.whl"
}
-
{
pkg
:
"
path.py==3.0.1"
,
wheel
:
"
path.py-3.0.1-py27-none-any.whl"
}
-
{
pkg
:
"
Pillow==1.7.8"
,
wheel
:
"
Pillow-1.7.8-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
polib==1.0.3"
,
wheel
:
"
polib-1.0.3-py27-none-any.whl"
}
-
{
pkg
:
"
pycrypto>=2.6"
,
wheel
:
"
pycrypto-2.6.1-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
pygments==1.6"
,
wheel
:
"
Pygments-1.6-py27-none-any.whl"
}
-
{
pkg
:
"
pygraphviz==1.1"
,
wheel
:
"
pygraphviz-1.1-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
pymongo==2.4.1"
,
wheel
:
"
pymongo-2.4.1-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
pyparsing==1.5.6"
,
wheel
:
"
pyparsing-1.5.6-py27-none-any.whl"
}
-
{
pkg
:
"
python-memcached==1.48"
,
wheel
:
"
python_memcached-1.48-py27-none-any.whl"
}
-
{
pkg
:
"
python-openid==2.2.5"
,
wheel
:
"
python_openid-2.2.5-py27-none-any.whl"
}
-
{
pkg
:
"
python-dateutil==2.1"
,
wheel
:
"
python_dateutil-2.1-py27-none-any.whl"
}
-
{
pkg
:
"
python-social-auth==0.1.21"
,
wheel
:
"
python_social_auth-0.1.21-py27-none-any.whl"
}
-
{
pkg
:
"
pytz==2012h"
,
wheel
:
"
pytz-2012h-py27-none-any.whl"
}
-
{
pkg
:
"
pysrt==0.4.7"
,
wheel
:
"
pysrt-0.4.7-py27-none-any.whl"
}
-
{
pkg
:
"
PyYAML==3.10"
,
wheel
:
"
PyYAML-3.10-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
requests==2.3.0"
,
wheel
:
"
requests-2.3.0-py27-none-any.whl"
}
-
{
pkg
:
"
scipy==0.11.0"
,
wheel
:
"
scipy-0.11.0-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
Shapely==1.2.16"
,
wheel
:
"
Shapely-1.2.16-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
singledispatch==3.4.0.2"
,
wheel
:
"
singledispatch-3.4.0.2-py27-none-any.whl"
}
-
{
pkg
:
"
sorl-thumbnail==11.12"
,
wheel
:
"
sorl_thumbnail-11.12-py27-none-any.whl"
}
-
{
pkg
:
"
South==0.7.6"
,
wheel
:
"
South-0.7.6-py27-none-any.whl"
}
-
{
pkg
:
"
sympy==0.7.1"
,
wheel
:
"
sympy-0.7.1-py27-none-any.whl"
}
-
{
pkg
:
"
xmltodict==0.4.1"
,
wheel
:
"
xmltodict-0.4.1-py27-none-any.whl"
}
-
{
pkg
:
"
django-ratelimit-backend==0.6"
,
wheel
:
"
django_ratelimit_backend-0.6-py27-none-any.whl"
}
-
{
pkg
:
"
unicodecsv==0.9.4"
,
wheel
:
"
unicodecsv-0.9.4-py27-none-any.whl"
}
-
{
pkg
:
"
ipython==0.13.1"
,
wheel
:
"
ipython-0.13.1-py27-none-any.whl"
}
-
{
pkg
:
"
watchdog==0.6.0"
,
wheel
:
"
watchdog-0.6.0-py27-none-any.whl"
}
-
{
pkg
:
"
dogapi==1.2.1"
,
wheel
:
"
dogapi-1.2.1-py27-none-any.whl"
}
-
{
pkg
:
"
newrelic==2.4.0.4"
,
wheel
:
"
newrelic-2.4.0.4-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
sphinx==1.1.3"
,
wheel
:
"
Sphinx-1.1.3-py27-none-any.whl"
}
-
{
pkg
:
"
sphinx_rtd_theme==0.1.5"
,
wheel
:
"
sphinx_rtd_theme-0.1.5-py27-none-any.whl"
}
-
{
pkg
:
"
Babel==1.3"
,
wheel
:
"
Babel-1.3-py27-none-any.whl"
}
-
{
pkg
:
"
transifex-client==0.10"
,
wheel
:
"
transifex_client-0.10-py27-none-any.whl"
}
-
{
pkg
:
"
django_debug_toolbar"
,
wheel
:
"
django_debug_toolbar-0.11.0-py2.py3-none-any.whl"
}
-
{
pkg
:
"
django-debug-toolbar-mongo"
,
wheel
:
"
django_debug_toolbar_mongo-0.1.10-py27-none-any.whl"
}
-
{
pkg
:
"
chrono==1.0.2"
,
wheel
:
"
chrono-1.0.2-py2.py3-none-any.whl"
}
-
{
pkg
:
"
coverage==3.6"
,
wheel
:
"
coverage-3.6-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
ddt==0.7.1"
,
wheel
:
"
ddt-0.7.1-py27-none-any.whl"
}
-
{
pkg
:
"
django-crum==0.5"
,
wheel
:
"
django_crum-0.5-py27-none-any.whl"
}
-
{
pkg
:
"
django_nose==1.1"
,
wheel
:
"
django_nose-1.1-py27-none-any.whl"
}
-
{
pkg
:
"
factory_boy==2.1.2"
,
wheel
:
"
factory_boy-2.1.2-py27-none-any.whl"
}
-
{
pkg
:
"
freezegun==0.1.11"
,
wheel
:
"
freezegun-0.1.11-py27-none-any.whl"
}
-
{
pkg
:
"
mock==1.0.1"
,
wheel
:
"
mock-1.0.1-py27-none-any.whl"
}
-
{
pkg
:
"
nosexcover==1.0.7"
,
wheel
:
"
nosexcover-1.0.7-py27-none-any.whl"
}
-
{
pkg
:
"
pep8==1.4.5"
,
wheel
:
"
pep8-1.4.5-py27-none-any.whl"
}
-
{
pkg
:
"
pylint==0.28"
,
wheel
:
"
pylint-0.28.0-py27-none-any.whl"
}
-
{
pkg
:
"
python-subunit==0.0.16"
,
wheel
:
"
python_subunit-0.0.16-py27-none-any.whl"
}
-
{
pkg
:
"
rednose==0.3"
,
wheel
:
"
rednose-0.3-py27-none-any.whl"
}
-
{
pkg
:
"
selenium==2.39.0"
,
wheel
:
"
selenium-2.39.0-py27-none-any.whl"
}
-
{
pkg
:
"
splinter==0.5.4"
,
wheel
:
"
splinter-0.5.4-py27-none-any.whl"
}
-
{
pkg
:
"
testtools==0.9.34"
,
wheel
:
"
testtools-0.9.34-py27-none-any.whl"
}
-
{
pkg
:
"
Paver==1.2.1"
,
wheel
:
"
Paver-1.2.1-py27-none-any.whl"
}
-
{
pkg
:
"
psutil==1.2.1"
,
wheel
:
"
psutil-1.2.1-cp27-none-linux_x86_64.whl"
}
-
{
pkg
:
"
lazy==1.1"
,
wheel
:
"
lazy-1.1-py27-none-any.whl"
}
-
{
pkg
:
"
path.py==3.0.1"
,
wheel
:
"
path.py-3.0.1-py27-none-any.whl"
}
-
{
pkg
:
"
MySQL-python==1.2.5"
,
wheel
:
"
MySQL_python-1.2.5-cp27-none-linux_x86_64.whl"
}
playbooks/roles/jenkins_worker/tasks/python.yml
View file @
4a292071
...
...
@@ -17,45 +17,28 @@
owner=root group=root
mode=755
# Create wheelhouse to enable fast virtualenv creation
-
name
:
Create wheel virtualenv
command
:
/usr/local/bin/virtualenv {{ jenkins_venv }} creates={{ jenkins_venv }}
# Create a virtualenv for edx-platform by installing the requirements
# and packaging the virtualenv.
-
name
:
Create shallow clone of edx-platform
git
:
>
repo=https://github.com/edx/edx-platform.git
dest={{ jenkins_home }}/shallow-clone
version=master
depth=1
sudo_user
:
"
{{
jenkins_user
}}"
-
name
:
Install wheel
pip
:
name=wheel virtualenv={{ jenkins_venv }} virtualenv_command=/usr/local/bin/virtualenv
sudo_user
:
"
{{
jenkins_user
}}"
-
name
:
Create wheelhouse dir
file
:
path={{ jenkins_wheel_dir }} state=directory
owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
# (need to install each one in the venv to satisfy dependencies)
-
name
:
Create wheel archives
shell
:
"
{{
jenkins_pip
}}
wheel
--wheel-dir={{
jenkins_wheel_dir
}}
\"
${item.pkg}
\"
&&
{{
jenkins_pip
}}
install
--use-wheel
--no-index
--find-links={{
jenkins_wheel_dir
}}
\"
${item.pkg}
\"
creates={{
jenkins_wheel_dir
}}/${item.wheel}"
sudo_user
:
"
{{
jenkins_user
}}"
with_items
:
jenkins_wheels
-
name
:
Add wheel_venv.sh script
template
:
src=wheel_venv.sh.j2 dest={{ jenkins_home }}/wheel_venv.sh
owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
# Run the wheel_venv.sh script for the first time
# This was previously done in the Jenkins global
# configuration as part of the AMI Init script.
# Moving here so that we can archive a clean snapshot
# of the virtualenv with only the defined packages
# from jenkins_wheels.
-
name
:
Run the wheel_venv.sh script
command
:
>
./wheel_venv.sh edx-venv
chdir={{ jenkins_home }}
creates={{ jenkins_home }}/edx-venv
-
name
:
Install edx-platform requirements
pip
:
>
requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
extra_args="--exists-action w"
virtualenv={{ jenkins_home }}/edx-venv
virtualenv_command=virtualenv-2.7
with_items
:
-
pre.txt
-
github.txt
-
base.txt
-
post.txt
-
paver.txt
sudo_user
:
"
{{
jenkins_user
}}"
# Archive the current state of the virtualenv
...
...
@@ -66,5 +49,4 @@
command
:
>
tar -cpzf edx-venv_clean.tar.gz edx-venv
chdir={{ jenkins_home }}
creates={{ jenkins_home }}/edx-venv_clean.tar.gz
sudo_user
:
"
{{
jenkins_user
}}"
playbooks/roles/jenkins_worker/templates/wheel_venv.sh.j2
deleted
100644 → 0
View file @
92994a30
#! /usr/bin/env bash
if
[
$#
-ne
1
]
;
then
echo
"Usage:
$0
VENV_DIR"
exit
1
fi
# Create and activate the new virtualenv
VENV
=
$1
mkdir
-p
$VENV
/usr/local/bin/virtualenv
$VENV
.
$VENV
/bin/activate
# Install each available wheel archive
ls
{{
jenkins_wheel_dir
}}
| cut
-d-
-f1
|
while
read
line
;
do
pip install
--use-wheel
--no-index
--find-links
={{
jenkins_wheel_dir
}}
$line
;
done
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment