Skip to content

E
edx-analytics-pipeline
Unverified Commit 2b74c865 by brianhw Committed by GitHub
Options

Merge pull request #464 from edx/brian/pip-compile 

Introduce pip-compile tooling
parents c6c09807 32bf21a0
Showing with 370 additions and 135 deletions
Makefile
.PHONY: requirements test test-requirements .tox .PHONY: requirements test test-requirements .tox upgrade
uninstall: uninstall:
while pip uninstall -y edx.analytics.tasks; do true; done while pip uninstall -y edx.analytics.tasks; do true; done
...@@ -35,11 +35,17 @@ endif ...@@ -35,11 +35,17 @@ endif
requirements: requirements:
pip install -U -r requirements/pre.txt pip install -U -r requirements/pre.txt
pip install -U -r requirements/default.txt --no-cache-dir pip install -U -r requirements/default.txt --no-cache-dir --upgrade-strategy only-if-needed
pip install -U -r requirements/extra.txt --no-cache-dir pip install -U -r requirements/extra.txt --no-cache-dir
test-requirements: requirements test-requirements: requirements
pip install -U -r requirements/test.txt --no-cache-dir pip install -U -r requirements/test.txt --no-cache-dir --upgrade-strategy only-if-needed
upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
pip-compile --upgrade -o requirements/base.txt requirements/base.in
pip-compile --upgrade -o requirements/default.txt requirements/default.in requirements/base.in
pip-compile --upgrade -o requirements/docs.txt requirements/docs.in requirements/default.in requirements/base.in
pip-compile --upgrade -o requirements/test.txt requirements/test.in requirements/default.in requirements/base.in
test-docker: test-docker:
docker run -v `(pwd)`:/edx/app/analytics-pipeline -it edxops/analytics-pipeline:latest make develop-local test-local docker run -v `(pwd)`:/edx/app/analytics-pipeline -it edxops/analytics-pipeline:latest make develop-local test-local
...@@ -81,7 +87,9 @@ coverage-local: test-local ...@@ -81,7 +87,9 @@ coverage-local: test-local
coverage: test coverage-local coverage: test coverage-local
docs-requirements: docs-requirements:
pip install -U -r requirements/docs.txt --no-cache-dir pip install -U -r requirements/pre.txt
pip install -U -r requirements/docs.txt --no-cache-dir --upgrade-strategy only-if-needed
pip install -U -r requirements/extra.txt --no-cache-dir
python setup.py install --force python setup.py install --force
docs-local: docs-local:
......
edx/analytics/tasks/launchers/local.py
...@@ -15,9 +15,12 @@ from contextlib import contextmanager ...@@ -15,9 +15,12 @@ from contextlib import contextmanager
import boto import boto
import bson import bson
import certifi
import chardet
import ciso8601 import ciso8601
import cjson import cjson
import filechunkio import filechunkio
import idna
import luigi import luigi
import luigi.configuration import luigi.configuration
import luigi.hadoop import luigi.hadoop
...@@ -25,12 +28,12 @@ import opaque_keys ...@@ -25,12 +28,12 @@ import opaque_keys
import pyinstrument import pyinstrument
import requests import requests
import stevedore import stevedore
# Tell urllib3 to switch the ssl backend to PyOpenSSL. import urllib3
# see https://urllib3.readthedocs.org/en/latest/security.html#pyopenssl
import urllib3.contrib.pyopenssl
import edx.analytics.tasks import edx.analytics.tasks
# Tell urllib3 to switch the ssl backend to PyOpenSSL.
# see https://urllib3.readthedocs.org/en/latest/security.html#pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3() urllib3.contrib.pyopenssl.inject_into_urllib3()
...@@ -82,8 +85,10 @@ def main(): ...@@ -82,8 +85,10 @@ def main():
# - opaque_keys is used to interpret serialized course_ids # - opaque_keys is used to interpret serialized course_ids
# - opaque_keys extensions: ccx_keys # - opaque_keys extensions: ccx_keys
# - dependencies of opaque_keys: bson, stevedore # - dependencies of opaque_keys: bson, stevedore
# - requests has several dependencies:
# - chardet, urllib3, certifi, idna
luigi.hadoop.attach(edx.analytics.tasks) luigi.hadoop.attach(edx.analytics.tasks)
luigi.hadoop.attach(boto, cjson, filechunkio, opaque_keys, bson, stevedore, ciso8601, requests) luigi.hadoop.attach(boto, cjson, filechunkio, opaque_keys, bson, stevedore, ciso8601, chardet, urllib3, certifi, idna, requests)
if configuration.getboolean('ccx', 'enabled', default=False): if configuration.getboolean('ccx', 'enabled', default=False):
import ccx_keys import ccx_keys
......
requirements/base.in 0 → 100644
# These packages are required for bootstrapping.
--no-binary ansible
ansible==1.4.5 # GPL v3 License
boto==2.48.0 # MIT
ecdsa==0.13 # MIT
Jinja2==2.8.1 # BSD
pycrypto==2.6.1 # public domain
wheel==0.30.0
# Add this here, not because it's needed here, but because
# 1.11.0 will collide with a dependency in default.in.
six==1.10.0
requirements/base.txt
# These packages are required for bootstrapping. #
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements/base.txt requirements/base.in
#
--no-binary ansible
ansible==1.4.5 --no-binary ansible # GPL v3 License ansible==1.4.5
boto==2.48.0 # MIT asn1crypto==0.23.0 # via cryptography
ecdsa==0.13 # MIT bcrypt==3.1.4 # via paramiko
Jinja2==2.8.1 # BSD boto==2.48.0
pycrypto==2.6.1 # public domain cffi==1.11.2 # via bcrypt, cryptography, pynacl
cryptography==2.1.4 # via paramiko
# The following are dependencies pulled in by the above, ecdsa==0.13
# but are pinned here. enum34==1.1.6 # via cryptography
httplib2==0.10.3 # via ansible
asn1crypto==0.22.0 idna==2.6 # via cryptography
bcrypt==3.1.3 ipaddress==1.0.18 # via cryptography
cffi==1.10.0 jinja2==2.8.1
cryptography==2.0.3 markupsafe==1.0 # via jinja2
enum34==1.1.6 paramiko==2.4.0 # via ansible
httplib2==0.10.3 pyasn1==0.4.2 # via paramiko
idna==2.6 pycparser==2.18 # via cffi
ipaddress==1.0.18 pycrypto==2.6.1
MarkupSafe==1.0 pynacl==1.2.1 # via paramiko
paramiko==2.2.1 pyyaml==3.12 # via ansible
pyasn1==0.3.4
pycparser==2.18
PyNaCl==1.1.2
PyYAML==3.12
six==1.10.0 six==1.10.0
# Already present, but not affected by the above, is setuptools and wheel.
# Pin these here as well.
setuptools==36.4.0
wheel==0.30.0 wheel==0.30.0
requirements/default.in 0 → 100644
# These packages are required for production
# (along with extra.txt, which is manually maintained).
argparse==1.2.1 # Python Software Foundation License
boto3==1.4.8 # Apache 2.0
ciso8601==1.0.3 # MIT
edx-ccx-keys==0.2.1 # AGPL
edx-opaque-keys==0.4 # AGPL
elasticsearch==1.7.0 # Apache
filechunkio==1.8 # MIT
google-cloud-bigquery==0.27.0
graphitesend==0.10.0 # Apache
html5lib==1.0b3 # MIT
isoweek==1.3.3 # BSD
numpy==1.11.3 # BSD
paypalrestsdk==1.9.0 # Paypal SDK License
pygeoip==0.3.2 # LGPL
python-cjson==1.1.0 # LGPL
python-dateutil==2.6.1 # BSD
python-gnupg==0.3.9 # BSD
pytz==2017.3 # ZPL
requests==2.18.4 # Apache 2.0
six==1.10.0 # MIT
stevedore==1.19.1 # Apache 2.0
ua-parser==0.3.6 # Apache
urllib3==1.22 # MIT
user-agents==0.3.2 # MIT
vertica-python==0.6.11 # MIT
yarn-api-client==0.2.3 # BSD
requirements/default.txt
-r base.txt #
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements/default.txt requirements/default.in requirements/base.in
#
--no-binary ansible
# These dependencies are explicitly included in code. ansible==1.4.5
argparse==1.2.1
argparse==1.2.1 # Python Software Foundation License asn1crypto==0.23.0 # via cryptography
boto3==1.4.4 # Apache 2.0 bcrypt==3.1.4 # via paramiko
ciso8601==1.0.3 # MIT boto3==1.4.8
edx-ccx-keys==0.2.1 # AGPL boto==2.48.0
edx-opaque-keys==0.4 # AGPL botocore==1.8.13 # via boto3, s3transfer
elasticsearch==1.7.0 # Apache cachetools==2.0.1 # via google-auth
filechunkio==1.8 # MIT certifi==2017.11.5 # via requests
graphitesend==0.10.0 # Apache cffi==1.11.2 # via bcrypt, cryptography, pynacl
html5lib==1.0b3 # MIT chardet==3.0.4 # via requests
isoweek==1.3.1 # BSD ciso8601==1.0.3
http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-1.2.2.zip # GPL v2 with FOSS License Exception cryptography==2.1.4 # via paramiko, pyopenssl
numpy==1.11.3 # BSD docutils==0.14 # via botocore
pandas==0.13.0 # BSD ecdsa==0.13
paypalrestsdk==1.9.0 # Paypal SDK License edx-ccx-keys==0.2.1
pygeoip==0.3.2 # LGPL edx-opaque-keys==0.4
python-cjson==1.1.0 # LGPL elasticsearch==1.7.0
python-dateutil==2.6.0 # BSD enum34==1.1.6 # via cryptography
python-gnupg==0.3.9 # BSD filechunkio==1.8
pytz==2016.10 # ZPL future==0.16.0 # via vertica-python
requests==2.12.4 # Apache 2.0 futures==3.2.0 # via google-cloud-core, s3transfer
six==1.10.0 # MIT google-auth==1.2.1 # via google-cloud-bigquery, google-cloud-core
stevedore==1.19.1 # Apache 2.0
ua-parser==0.3.6 # Apache
urllib3==1.19.1 # MIT
user-agents==0.3.2 # MIT
vertica-python==0.6.11 # MIT
yarn-api-client==0.2.3 # BSD
google-cloud-bigquery==0.27.0 google-cloud-bigquery==0.27.0
google-cloud-core==0.27.1 # via google-cloud-bigquery
git+https://github.com/edx/luigi.git@a73700ca51685974220ef6069d2f078312055444#egg=luigi # Apache License 2.0 google-resumable-media==0.3.1 # via google-cloud-bigquery
git+https://github.com/edx/pyinstrument.git@a35ff76df4c3d5ff9a2876d859303e33d895e78f#egg=pyinstrument # BSD googleapis-common-protos==1.5.3 # via google-cloud-core
graphitesend==0.10.0
# These dependencies are pulled in by the above requirements, html5lib==1.0b3
# and are pinned here. httplib2==0.10.3 # via ansible
idna==2.6 # via cryptography, requests
botocore==1.5.95 ipaddress==1.0.18 # via cryptography
distlib==0.2.2 isoweek==1.3.3
docutils==0.14 jinja2==2.8.1
google-auth==1.2.0 jmespath==0.9.3 # via boto3, botocore
google-cloud-core==0.27.1 markupsafe==1.0 # via jinja2
google-resumable-media==0.3.1 numpy==1.11.3
googleapis-common-protos==1.5.3 paramiko==2.4.0 # via ansible
future==0.16.0 paypalrestsdk==1.9.0
futures==3.1.1 pbr==3.1.1 # via stevedore
jmespath==0.9.3 protobuf==3.5.0.post1 # via google-cloud-core, googleapis-common-protos
pbr==3.1.1 pyasn1-modules==0.2.1 # via google-auth
protobuf==3.4.0 pyasn1==0.4.2 # via google-auth, paramiko, pyasn1-modules, rsa
pymongo==3.5.1 pycparser==2.18 # via cffi
pyOpenSSL==17.2.0 pycrypto==2.6.1
pyasn1-modules==0.1.5 pygeoip==0.3.2
s3transfer==0.1.11 pymongo==3.6.0 # via edx-opaque-keys
pynacl==1.2.1 # via paramiko
pyopenssl==17.5.0 # via paypalrestsdk
# These dependencies are already pulled in by earlier requirements (e.g. base.txt) python-cjson==1.1.0
# and are NOT pinned here. python-dateutil==2.6.1
python-gnupg==0.3.9
# cffi==1.9.1 # MIT pytz==2017.3
# cryptography==1.7.1 # BSD or Apache 2.0 pyyaml==3.12 # via ansible
# enum34==1.1.6 # BSD requests==2.18.4
# idna==2.2 # BSD-like rsa==3.4.2 # via google-auth
# pip==9.0.1 s3transfer==0.1.12 # via boto3
# pyasn1==0.1.9 # BSD six==1.10.0
# pycparser==2.17 # BSD stevedore==1.19.1
# setuptools==32.3.1 ua-parser==0.3.6
urllib3==1.22
user-agents==0.3.2
vertica-python==0.6.11
wheel==0.30.0
yarn-api-client==0.2.3
requirements/docs.in 0 → 100644
# Requirements for documentation validation
Sphinx==1.5.1
requirements/docs.txt
-r default.txt #
Sphinx==1.5.1 # This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements/docs.txt requirements/docs.in requirements/default.in requirements/base.in
#
--no-binary ansible
alabaster==0.7.10 # via sphinx
ansible==1.4.5
argparse==1.2.1
asn1crypto==0.23.0 # via cryptography
babel==2.5.1 # via sphinx
bcrypt==3.1.4 # via paramiko
boto3==1.4.8
boto==2.48.0
botocore==1.8.13 # via boto3, s3transfer
cachetools==2.0.1 # via google-auth
certifi==2017.11.5 # via requests
cffi==1.11.2 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
ciso8601==1.0.3
cryptography==2.1.4 # via paramiko, pyopenssl
docutils==0.14 # via botocore, sphinx
ecdsa==0.13
edx-ccx-keys==0.2.1
edx-opaque-keys==0.4
elasticsearch==1.7.0
enum34==1.1.6 # via cryptography
filechunkio==1.8
future==0.16.0 # via vertica-python
futures==3.2.0 # via google-cloud-core, s3transfer
google-auth==1.2.1 # via google-cloud-bigquery, google-cloud-core
google-cloud-bigquery==0.27.0
google-cloud-core==0.27.1 # via google-cloud-bigquery
google-resumable-media==0.3.1 # via google-cloud-bigquery
googleapis-common-protos==1.5.3 # via google-cloud-core
graphitesend==0.10.0
html5lib==1.0b3
httplib2==0.10.3 # via ansible
idna==2.6 # via cryptography, requests
imagesize==0.7.1 # via sphinx
ipaddress==1.0.18 # via cryptography
isoweek==1.3.3
jinja2==2.8.1
jmespath==0.9.3 # via boto3, botocore
markupsafe==1.0 # via jinja2
numpy==1.11.3
paramiko==2.4.0 # via ansible
paypalrestsdk==1.9.0
pbr==3.1.1 # via stevedore
protobuf==3.5.0.post1 # via google-cloud-core, googleapis-common-protos
pyasn1-modules==0.2.1 # via google-auth
pyasn1==0.4.2 # via google-auth, paramiko, pyasn1-modules, rsa
pycparser==2.18 # via cffi
pycrypto==2.6.1
pygeoip==0.3.2
pygments==2.2.0 # via sphinx
pymongo==3.6.0 # via edx-opaque-keys
pynacl==1.2.1 # via paramiko
pyopenssl==17.5.0 # via paypalrestsdk
python-cjson==1.1.0
python-dateutil==2.6.1
python-gnupg==0.3.9
pytz==2017.3
pyyaml==3.12 # via ansible
requests==2.18.4
rsa==3.4.2 # via google-auth
s3transfer==0.1.12 # via boto3
six==1.10.0
snowballstemmer==1.2.1 # via sphinx
sphinx==1.5.1
stevedore==1.19.1
ua-parser==0.3.6
urllib3==1.22
user-agents==0.3.2
vertica-python==0.6.11
wheel==0.30.0
yarn-api-client==0.2.3
requirements/extra.txt
psycopg2==2.6.2 # LGPL # These are "extra" dependencies, for use in production but not in doc-generation.
# Generic URLs are not yet supported by pip-compile.
http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-1.2.2.zip # GPL v2 with FOSS License Exception
# Putting these here rather than in default.in allows us to avoid using editable mode that pip-compile requires.
git+https://github.com/edx/luigi.git@a73700ca51685974220ef6069d2f078312055444#egg=luigi # Apache License 2.0
git+https://github.com/edx/pyinstrument.git@a35ff76df4c3d5ff9a2876d859303e33d895e78f#egg=pyinstrument # BSD
# This was originally separated out from default.in so that it was not included in docs.txt. Not clear why.
psycopg2==2.6.2
requirements/pre.txt
pip==9.0.1 # MIT // AN-4322 pip==9.0.1
setuptools==36.4.0
requirements/test.in 0 → 100644
# Add the previous dependencies, so that there is agreement.
# For example, six==1.10.0 would be replaced with six==1.11.0 if we didn't somehow pin it.
# Downside is that the resulting file has all of those duplicate dependencies, rather than
# only the increment.
# -r default.txt
# Stuff for tests
coverage==4.3.1
ddt==1.1.1
diff-cover >= 0.2.1
freezegun==0.3.9
httpretty==0.8.14
mock==2.0.0
nose-ignore-docstring==0.2
nose==1.3.7
pandas==0.13.0
# Stuff for development
# edx-lint
# pip-tools
# Stuff for quality
isort==4.2.15
pycodestyle==2.3.1
pylint==1.6.4
requirements/test.txt
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements/test.txt requirements/test.in requirements/default.in requirements/base.in
#
--no-binary ansible
ansible==1.4.5
argparse==1.2.1
asn1crypto==0.23.0 # via cryptography
astroid==1.4.9 # via pylint
backports.functools-lru-cache==1.4 # via pylint
bcrypt==3.1.4 # via paramiko
boto3==1.4.8
boto==2.48.0
botocore==1.8.13 # via boto3, s3transfer
cachetools==2.0.1 # via google-auth
certifi==2017.11.5 # via requests
cffi==1.11.2 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
ciso8601==1.0.3
configparser==3.5.0 # via pylint
coverage==4.3.1 coverage==4.3.1
cryptography==2.1.4 # via paramiko, pyopenssl
ddt==1.1.1 ddt==1.1.1
diff-cover >= 0.2.1 diff-cover==1.0.0
freezegun==0.3.8 docutils==0.14 # via botocore
ecdsa==0.13
edx-ccx-keys==0.2.1
edx-opaque-keys==0.4
elasticsearch==1.7.0
enum34==1.1.6 # via cryptography
filechunkio==1.8
freezegun==0.3.9
funcsigs==1.0.2 # via mock
future==0.16.0 # via vertica-python
futures==3.2.0 # via google-cloud-core, s3transfer
google-auth==1.2.1 # via google-cloud-bigquery, google-cloud-core
google-cloud-bigquery==0.27.0
google-cloud-core==0.27.1 # via google-cloud-bigquery
google-resumable-media==0.3.1 # via google-cloud-bigquery
googleapis-common-protos==1.5.3 # via google-cloud-core
graphitesend==0.10.0
html5lib==1.0b3
httplib2==0.10.3 # via ansible
httpretty==0.8.14 httpretty==0.8.14
idna==2.6 # via cryptography, requests
inflect==0.2.5 # via jinja2-pluralize
ipaddress==1.0.18 # via cryptography
isort==4.2.15 isort==4.2.15
isoweek==1.3.3
jinja2-pluralize==0.3.0 # via diff-cover
jinja2==2.8.1
jmespath==0.9.3 # via boto3, botocore
lazy-object-proxy==1.3.1 # via astroid
markupsafe==1.0 # via jinja2
mccabe==0.6.1 # via pylint
mock==2.0.0 mock==2.0.0
nose-ignore-docstring==0.2 nose-ignore-docstring==0.2
nose==1.3.7 nose==1.3.7
numpy==1.11.3
pandas==0.13.0
paramiko==2.4.0 # via ansible
paypalrestsdk==1.9.0
pbr==3.1.1 # via mock, stevedore
protobuf==3.5.0.post1 # via google-cloud-core, googleapis-common-protos
pyasn1-modules==0.2.1 # via google-auth
pyasn1==0.4.2 # via google-auth, paramiko, pyasn1-modules, rsa
pycodestyle==2.3.1 pycodestyle==2.3.1
pycparser==2.18 # via cffi
pycrypto==2.6.1
pygeoip==0.3.2
pygments==2.2.0 # via diff-cover
pylint==1.6.4 pylint==1.6.4
pymongo==3.6.0 # via edx-opaque-keys
# dependencies pulled in by the above, that pynacl==1.2.1 # via paramiko
# are being pinned. pyopenssl==17.5.0 # via paypalrestsdk
python-cjson==1.1.0
astroid==1.4.9 python-dateutil==2.6.1
backports.functools-lru-cache==1.4 python-gnupg==0.3.9
configparser==3.5.0 pytz==2017.3
funcsigs==1.0.2 pyyaml==3.12 # via ansible
inflect==0.2.5 requests==2.18.4
isort==4.2.15 rsa==3.4.2 # via google-auth
jinja2-pluralize==0.3.0 s3transfer==0.1.12 # via boto3
lazy-object-proxy==1.3.1
mccabe==0.6.1
Pygments==2.2.0
wrapt==1.10.11
# dependencies that are already pinned earlier.
Jinja2==2.8.1
MarkupSafe==1.0
pbr==3.1.1
python-dateutil==2.6.0
six==1.10.0 six==1.10.0
stevedore==1.19.1
ua-parser==0.3.6
urllib3==1.22
user-agents==0.3.2
vertica-python==0.6.11
wheel==0.30.0
wrapt==1.10.11 # via astroid
yarn-api-client==0.2.3
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment