Merge pull request #1397 from edx/benp/remove-wheels-jenkins-worker

Remove usage of wheels during jenkins worker provisioning.

playbooks/roles/jenkins_worker/defaults/main.yml

@@ -29,104 +29,3 @@ jenkins_ruby_version: "1.9.3-p374"
 # JSCover direct download URL
 jscover_url: "http://files.edx.org/testeng/JSCover-1.0.2.zip"
 jscover_version: "1.0.2"
-
-# Python
-jenkins_venv: "{{ jenkins_home }}/wheel_venv"
-jenkins_pip: "{{ jenkins_venv }}/bin/pip"
-jenkins_wheel_dir: "{{ jenkins_home }}/wheelhouse"
-jenkins_wheels:
-    - { pkg: "numpy==1.6.2", wheel: "numpy-1.6.2-cp27-none-linux_x86_64.whl" }
-    - { pkg: "django-celery==3.0.17", wheel: "django_celery-3.0.17-py27-none-any.whl" }
-    - { pkg: "beautifulsoup4==4.1.3", wheel: "beautifulsoup4-4.1.3-py27-none-any.whl"}
-    - { pkg: "beautifulsoup==3.2.1", wheel: "BeautifulSoup-3.2.1-py27-none-any.whl" }
-    - { pkg: "bleach==1.4", wheel: "bleach-1.4-py27-none-any.whl" }
-    - { pkg: "html5lib==0.999", wheel: "html5lib-0.999-py27-none-any.whl" }
-    - { pkg: "boto==2.13.3", wheel: "boto-2.13.3-py27-none-any.whl" }
-    - { pkg: "celery==3.0.19", wheel: "celery-3.0.19-py27-none-any.whl" }
-    - { pkg: "dealer==0.2.3", wheel: "dealer-0.2.3-py27-none-any.whl" }
-    - { pkg: "django-countries==1.5", wheel: "django_countries-1.5-py27-none-any.whl" }
-    - { pkg: "django-filter==0.6.0", wheel: "django_filter-0.6-py27-none-any.whl" }
-    - { pkg: "django-followit==0.0.3", wheel: "django_followit-0.0.3-py27-none-any.whl" }
-    - { pkg: "django-kombu==0.9.4", wheel: "kombu-2.5.16-py27-none-any.whl" }
-    - { pkg: "django-mako==0.1.5pre", wheel: "django_mako-0.1.5pre-py27-none-any.whl" }
-    - { pkg: "django-model-utils==1.4.0", wheel: "django_model_utils-1.4.0-py27-none-any.whl" }
-    - { pkg: "django-masquerade==0.1.6", wheel: "django_masquerade-0.1.6-py27-none-any.whl" }
-    - { pkg: "django-mptt==0.5.5", wheel: "django_mptt-0.5.5-py27-none-any.whl" }
-    - { pkg: "django-openid-auth==0.4", wheel: "python_openid-2.2.5-py27-none-any.whl" }
-    - { pkg: "django-robots==0.9.1", wheel: "django_robots-0.9.1-py27-none-any.whl" }
-    - { pkg: "django-sekizai==0.6.1", wheel: "django_sekizai-0.6.1-py27-none-any.whl" }
-    - { pkg: "django-ses==0.4.1", wheel: "django_ses-0.4.1-py27-none-any.whl" }
-    - { pkg: "django-storages==1.1.5", wheel: "django_storages-1.1.5-py27-none-any.whl" }
-    - { pkg: "django-method-override==0.1.0", wheel: "django_method_override-0.1.0-py27-none-any.whl" }
-    - { pkg: "djangorestframework==2.3.5", wheel: "djangorestframework-2.3.5-py27-none-any.whl" }
-    - { pkg: "django==1.4.8", wheel: "Django-1.4.8-py27-none-any.whl" }
-    - { pkg: "feedparser==5.1.3", wheel: "feedparser-5.1.3-py27-none-any.whl" }
-    - { pkg: "fs==0.4.0", wheel: "fs-0.4.0-py27-none-any.whl" }
-    - { pkg: "GitPython==0.3.2.RC1", wheel: "GitPython-0.3.2.RC1-py27-none-any.whl" }
-    - { pkg: "glob2==0.3", wheel: "glob2-0.3-py27-none-any.whl" }
-    - { pkg: "gunicorn==0.17.4", wheel: "gunicorn-0.17.4-py27-none-any.whl" }
-    - { pkg: "lazy==1.1", wheel: "lazy-1.1-py27-none-any.whl" }
-    - { pkg: "lxml==3.0.1", wheel: "lxml-3.0.1-cp27-none-linux_x86_64.whl" }
-    - { pkg: "mako==0.9.1", wheel: "Mako-0.9.1-py2.py3-none-any.whl" }
-    - { pkg: "Markdown==2.2.1", wheel: "Markdown-2.2.1-py27-none-any.whl" }
-    - { pkg: "mongoengine==0.7.10", wheel: "mongoengine-0.7.10-py27-none-any.whl" }
-    - { pkg: "networkx==1.7", wheel: "networkx-1.7-py27-none-any.whl" }
-    - { pkg: "nltk==2.0.4", wheel: "nltk-2.0.4-py27-none-any.whl" }
-    - { pkg: "oauthlib==0.6.3", wheel: "oauthlib-0.6.3-py27-none-any.whl" }
-    - { pkg: "paramiko==1.9.0", wheel: "paramiko-1.9.0-py27-none-any.whl" }
-    - { pkg: "path.py==3.0.1", wheel: "path.py-3.0.1-py27-none-any.whl" }
-    - { pkg: "Pillow==1.7.8", wheel: "Pillow-1.7.8-cp27-none-linux_x86_64.whl" }
-    - { pkg: "polib==1.0.3", wheel: "polib-1.0.3-py27-none-any.whl" }
-    - { pkg: "pycrypto>=2.6", wheel: "pycrypto-2.6.1-cp27-none-linux_x86_64.whl" }
-    - { pkg: "pygments==1.6", wheel: "Pygments-1.6-py27-none-any.whl" }
-    - { pkg: "pygraphviz==1.1", wheel: "pygraphviz-1.1-cp27-none-linux_x86_64.whl" }
-    - { pkg: "pymongo==2.4.1", wheel: "pymongo-2.4.1-cp27-none-linux_x86_64.whl" }
-    - { pkg: "pyparsing==1.5.6", wheel: "pyparsing-1.5.6-py27-none-any.whl" }
-    - { pkg: "python-memcached==1.48", wheel: "python_memcached-1.48-py27-none-any.whl" }
-    - { pkg: "python-openid==2.2.5", wheel: "python_openid-2.2.5-py27-none-any.whl" }
-    - { pkg: "python-dateutil==2.1", wheel: "python_dateutil-2.1-py27-none-any.whl" }
-    - { pkg: "python-social-auth==0.1.21", wheel: "python_social_auth-0.1.21-py27-none-any.whl" }
-    - { pkg: "pytz==2012h", wheel: "pytz-2012h-py27-none-any.whl" }
-    - { pkg: "pysrt==0.4.7", wheel: "pysrt-0.4.7-py27-none-any.whl" }
-    - { pkg: "PyYAML==3.10", wheel: "PyYAML-3.10-cp27-none-linux_x86_64.whl" }
-    - { pkg: "requests==2.3.0", wheel: "requests-2.3.0-py27-none-any.whl" }
-    - { pkg: "scipy==0.11.0", wheel: "scipy-0.11.0-cp27-none-linux_x86_64.whl" }
-    - { pkg: "Shapely==1.2.16", wheel: "Shapely-1.2.16-cp27-none-linux_x86_64.whl" }
-    - { pkg: "singledispatch==3.4.0.2", wheel: "singledispatch-3.4.0.2-py27-none-any.whl" }
-    - { pkg: "sorl-thumbnail==11.12", wheel: "sorl_thumbnail-11.12-py27-none-any.whl" }
-    - { pkg: "South==0.7.6", wheel: "South-0.7.6-py27-none-any.whl" }
-    - { pkg: "sympy==0.7.1", wheel: "sympy-0.7.1-py27-none-any.whl" }
-    - { pkg: "xmltodict==0.4.1", wheel: "xmltodict-0.4.1-py27-none-any.whl" }
-    - { pkg: "django-ratelimit-backend==0.6", wheel: "django_ratelimit_backend-0.6-py27-none-any.whl" }
-    - { pkg: "unicodecsv==0.9.4", wheel: "unicodecsv-0.9.4-py27-none-any.whl" }
-    - { pkg: "ipython==0.13.1", wheel: "ipython-0.13.1-py27-none-any.whl" }
-    - { pkg: "watchdog==0.6.0", wheel: "watchdog-0.6.0-py27-none-any.whl" }
-    - { pkg: "dogapi==1.2.1", wheel: "dogapi-1.2.1-py27-none-any.whl" }
-    - { pkg: "newrelic==2.4.0.4", wheel: "newrelic-2.4.0.4-cp27-none-linux_x86_64.whl" }
-    - { pkg: "sphinx==1.1.3", wheel: "Sphinx-1.1.3-py27-none-any.whl" }
-    - { pkg: "sphinx_rtd_theme==0.1.5", wheel: "sphinx_rtd_theme-0.1.5-py27-none-any.whl" }
-    - { pkg: "Babel==1.3", wheel: "Babel-1.3-py27-none-any.whl" }
-    - { pkg: "transifex-client==0.10", wheel: "transifex_client-0.10-py27-none-any.whl" }
-    - { pkg: "django_debug_toolbar", wheel: "django_debug_toolbar-0.11.0-py2.py3-none-any.whl" }
-    - { pkg: "django-debug-toolbar-mongo", wheel: "django_debug_toolbar_mongo-0.1.10-py27-none-any.whl" }
-    - { pkg: "chrono==1.0.2", wheel: "chrono-1.0.2-py2.py3-none-any.whl" }
-    - { pkg: "coverage==3.6", wheel: "coverage-3.6-cp27-none-linux_x86_64.whl" }
-    - { pkg: "ddt==0.7.1", wheel: "ddt-0.7.1-py27-none-any.whl" }
-    - { pkg: "django-crum==0.5", wheel: "django_crum-0.5-py27-none-any.whl" }
-    - { pkg: "django_nose==1.1", wheel: "django_nose-1.1-py27-none-any.whl" }
-    - { pkg: "factory_boy==2.1.2", wheel: "factory_boy-2.1.2-py27-none-any.whl" }
-    - { pkg: "freezegun==0.1.11", wheel: "freezegun-0.1.11-py27-none-any.whl" }
-    - { pkg: "mock==1.0.1", wheel: "mock-1.0.1-py27-none-any.whl" }
-    - { pkg: "nosexcover==1.0.7", wheel: "nosexcover-1.0.7-py27-none-any.whl" }
-    - { pkg: "pep8==1.4.5", wheel: "pep8-1.4.5-py27-none-any.whl" }
-    - { pkg: "pylint==0.28", wheel: "pylint-0.28.0-py27-none-any.whl" }
-    - { pkg: "python-subunit==0.0.16", wheel: "python_subunit-0.0.16-py27-none-any.whl" }
-    - { pkg: "rednose==0.3", wheel: "rednose-0.3-py27-none-any.whl" }
-    - { pkg: "selenium==2.39.0", wheel: "selenium-2.39.0-py27-none-any.whl" }
-    - { pkg: "splinter==0.5.4", wheel: "splinter-0.5.4-py27-none-any.whl" }
-    - { pkg: "testtools==0.9.34", wheel: "testtools-0.9.34-py27-none-any.whl" }
-    - { pkg: "Paver==1.2.1", wheel: "Paver-1.2.1-py27-none-any.whl" }
-    - { pkg: "psutil==1.2.1", wheel: "psutil-1.2.1-cp27-none-linux_x86_64.whl" }
-    - { pkg: "lazy==1.1", wheel: "lazy-1.1-py27-none-any.whl" }
-    - { pkg: "path.py==3.0.1", wheel: "path.py-3.0.1-py27-none-any.whl" }
-    - { pkg: "MySQL-python==1.2.5", wheel: "MySQL_python-1.2.5-cp27-none-linux_x86_64.whl" }

playbooks/roles/jenkins_worker/tasks/python.yml

@@ -17,45 +17,31 @@
             owner=root group=root
             mode=755
 
-# Create wheelhouse to enable fast virtualenv creation
-- name: Create wheel virtualenv
-  command: /usr/local/bin/virtualenv {{ jenkins_venv }} creates={{ jenkins_venv }}
+# Create a virtualenv for edx-platform by installing the requirements
+# and packaging the virtualenv.
+# A shallow clone is created off of master. The depth setting
+# refers to the --depth-setting of git clone. A value of 1
+# will truncate all history prior to the last revision.
+- name: Create shallow clone of edx-platform
+  git: >
+    repo=https://github.com/edx/edx-platform.git
+    dest={{ jenkins_home }}/shallow-clone
+    version=master
+    depth=1
   sudo_user: "{{ jenkins_user }}"
 
-- name: Install wheel
-  pip: name=wheel virtualenv={{ jenkins_venv }} virtualenv_command=/usr/local/bin/virtualenv
-  sudo_user: "{{ jenkins_user }}"
-
-- name: Create wheelhouse dir
-  file:
-    path={{ jenkins_wheel_dir }} state=directory
-    owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
-
-# (need to install each one in the venv to satisfy dependencies)
-- name: Create wheel archives
-  shell:
-    "{{ jenkins_pip }} wheel --wheel-dir={{ jenkins_wheel_dir }} \"${item.pkg}\" &&
-    {{ jenkins_pip }} install --use-wheel --no-index --find-links={{ jenkins_wheel_dir }} \"${item.pkg}\"
-    creates={{ jenkins_wheel_dir }}/${item.wheel}"
-  sudo_user: "{{ jenkins_user }}"
-  with_items: jenkins_wheels
-
-- name: Add wheel_venv.sh script
-  template:
-    src=wheel_venv.sh.j2 dest={{ jenkins_home }}/wheel_venv.sh
-    owner={{ jenkins_user }} group={{ jenkins_group }} mode=700
-
-# Run the wheel_venv.sh script for the first time
-# This was previously done in the Jenkins global
-# configuration as part of the AMI Init script.
-# Moving here so that we can archive a clean snapshot
-# of the virtualenv with only the defined packages
-# from jenkins_wheels.
-- name: Run the wheel_venv.sh script
-  command: >
-    ./wheel_venv.sh edx-venv
-    chdir={{ jenkins_home }}
-    creates={{ jenkins_home }}/edx-venv
+- name: Install edx-platform requirements
+  pip: >
+    requirements={{ jenkins_home }}/shallow-clone/requirements/edx/{{ item }}
+    extra_args="--exists-action w"
+    virtualenv={{ jenkins_home }}/edx-venv
+    virtualenv_command=virtualenv-2.7
+  with_items:
+    - pre.txt
+    - github.txt
+    - base.txt
+    - post.txt
+    - paver.txt
   sudo_user: "{{ jenkins_user }}"
 
 # Archive the current state of the virtualenv
@@ -66,5 +52,9 @@
   command: >
     tar -cpzf edx-venv_clean.tar.gz edx-venv
     chdir={{ jenkins_home }}
-    creates={{ jenkins_home }}/edx-venv_clean.tar.gz
   sudo_user: "{{ jenkins_user }}"
+
+# Remove the shallow-clone directory now that we archive
+# done with it
+- name: Remove shallow-clone
+  file: path={{ jenkins_home }}/shallow-clone state=absent

playbooks/roles/jenkins_worker/templates/wheel_venv.sh.j2

-#! /usr/bin/env bash
-
-if [ $# -ne 1 ]; then
-    echo "Usage: $0 VENV_DIR"
-    exit 1
-fi
-
-# Create and activate the new virtualenv
-VENV=$1
-mkdir -p $VENV
-/usr/local/bin/virtualenv $VENV
-. $VENV/bin/activate
-
-# Install each available wheel archive
-ls {{ jenkins_wheel_dir }} | cut -d- -f1 | while read line ; do
-    pip install --use-wheel --no-index --find-links={{ jenkins_wheel_dir }} $line ;
-done