Download packages as wheels

This dramatically reduces the time it takes to initialize the virtualenv by using pre-compiled versions of numpy and pandas etc.

Change-Id: I050b90a06b7521eefb9b6c822f46e5c6fcc310ff

Makefile

+
+PIP_INSTALL = pip install --use-wheel --find-links=$$WHEEL_URL/Python-$$WHEEL_PYVER
+
 .PHONY:	requirements test test-requirements .tox
 
-install: requirements
+uninstall:
+	while pip uninstall -y edx.analytics.tasks; do true; done
+	python setup.py clean
+
+install: requirements uninstall
 	python setup.py install
 
 develop: requirements
@@ -11,23 +18,22 @@ system-requirements:
 	sudo apt-get install -y -q libmysqlclient-dev
 
 requirements:
-	pip install -r requirements/default.txt
+	$(PIP_INSTALL) -r requirements/default.txt
 
 test-requirements: requirements
-	pip install -r requirements/test.txt
+	$(PIP_INSTALL) -r requirements/test.txt
 
 test: test-requirements
 	# TODO: when we have better coverage, modify this to actually fail when coverage is too low.
 	rm -rf .coverage
-	python -m coverage run --rcfile=./.coveragerc `which nosetests` -A 'not acceptance'
+	python -m coverage run --rcfile=./.coveragerc -m nose -A 'not acceptance'
 
 test-acceptance: test-requirements
-	rm -rf .coverage
-	python -m coverage run --rcfile=./.coveragerc `which nosetests` --nocapture -A acceptance
+	python -m coverage run --rcfile=./.coveragerc -m nose --nocapture -A acceptance
 
 coverage: test
-	coverage html
-	coverage xml -o coverage.xml
+	python -m coverage html
+	python -m coverage xml -o coverage.xml
 	diff-cover coverage.xml --html-report diff_cover.html
 
 	# Compute quality
@@ -41,7 +47,7 @@ coverage: test
 jenkins: .tox
 	virtualenv ./venv
 	./venv/bin/pip install -U tox
-	./venv/bin/tox
+	./venv/bin/tox -v --recreate
 
 get_config = $(shell echo "$$ACCEPTANCE_TEST_CONFIG" | python -c 'import sys, json; print json.load(sys.stdin)[sys.argv[1]]' $(1))
 VENV_ROOT = $(shell echo "$$WORKSPACE/build/venvs")
@@ -61,7 +67,7 @@ jenkins-acceptance:
 	$(META_BIN)/pip install awscli
 	$(META_BIN)/aws s3 rm --recursive $(call get_config,tasks_output_url)$(call get_config,identifier) || true
 
-	$(EXPORTER_BIN)/pip install -r $$WORKSPACE/analytics-exporter/requirements.txt
-	$(EXPORTER_BIN)/pip install -e $$WORKSPACE/analytics-exporter/
+	$(EXPORTER_BIN)/$(PIP_INSTALL) -r $$WORKSPACE/analytics-exporter/requirements.txt
+	$(EXPORTER_BIN)/$(PIP_INSTALL) -e $$WORKSPACE/analytics-exporter/
 
 	. $(TASKS_BIN)/activate && $(MAKE) test-acceptance

requirements/default.txt

@@ -11,4 +11,4 @@ pygeoip==0.3.1
 python-cjson==1.0.5
 stevedore==0.14.1
 tornado==3.1.1
--e git+https://github.com/spotify/luigi.git@a33756c781b9bf7e51384f0eb19d6a25050ef136#egg=luigi
+git+https://github.com/spotify/luigi.git@a33756c781b9bf7e51384f0eb19d6a25050ef136#egg=luigi

share/task.yml

@@ -26,6 +26,11 @@
     - git_server_public_key: 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=='
     - wait_for_task: False
     - local_log_dir: build/logs
+    - install_env:
+        # EMR runs a modified version of Debian 6 (squeeze)
+        WHEEL_URL: http://edx-wheelhouse.s3-website-us-east-1.amazonaws.com/Debian/squeeze
+        # EMR ships with python 2.6 (unfortunately)
+        WHEEL_PYVER: 2.6
 
   tasks:
     - name: known_hosts file exists
@@ -87,6 +92,7 @@
       shell: >
         . {{ working_venv_dir }}/bin/activate && make install
         chdir={{ working_repo_dir }}
+      environment: install_env
 
     - name: logging configured
       template: src=logging.cfg.j2 dest={{ working_repo_dir }}/logging.cfg
@@ -121,3 +127,4 @@
         - stdout
         - stderr
         - edx_analytics.log
+      when: wait_for_task

tox.ini

 [tox]
 envlist=py26,py27
-toxworkdir=/tmp/analytics-tox-cache
 
 # tox does not update the virtualenv after changes to requirement files.
 # http://bitbucket.org/hpk42/tox/issue/149/virtualenv-is-not-recreated-when-deps
 
 [testenv]
-install_command=pip install --pre --allow-external argparse -U {opts} {packages}
 deps=-r{toxinidir}/requirements/default.txt
      -r{toxinidir}/requirements/test.txt
-commands=nosetests -A 'not acceptance'
+commands=nosetests -A 'not acceptance'
\ No newline at end of file
+
+[testenv:py26]
+install_command=pip install --use-wheel --download-cache=/tmp/tox/cache --find-links={env:WHEEL_URL}/Python-2.6 --pre --allow-external argparse {opts} {packages}
+
+[testenv:py27]
+install_command=pip install --use-wheel --download-cache=/tmp/tox/cache --find-links={env:WHEEL_URL}/Python-2.7 --pre --allow-external argparse {opts} {packages}