TE-2565 Improve requirements file management

Makefile

@@ -9,6 +9,7 @@ main.help:
 	@echo ''
 	@echo 'Usage:'
 	@echo '    make requirements              install requirements'
+	@echo '    make upgrade                   upgrade dependencies in requirements files'
 	@echo '    make test                      run all tests'
 	@echo '    make build                     build everything'
 	@echo '    make pkg                       package everything'
@@ -19,4 +20,29 @@ requirements:
 	pip install -qr pre-requirements.txt --exists-action w
 	pip install -qr requirements.txt --exists-action w
 
+upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints
+	pip install -qr pre-requirements.txt --exists-action w
+	pip install -qr requirements/pip-tools.txt
+	# Make sure to compile files after any other files they include!
+	pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip-compile --upgrade -o requirements.txt requirements/base.in
+	pip-compile --upgrade -o playbooks/roles/aws/templates/requirements.txt.j2 requirements/aws.in
+	pip-compile --upgrade -o requirements3.txt requirements/ses-limits.in
+	pip-compile --upgrade -o util/elasticsearch/requirements.txt requirements/elasticsearch.in
+	pip-compile --upgrade -o util/jenkins/requirements-celery.txt requirements/celery.in
+	pip-compile --upgrade -o util/jenkins/requirements-cloudflare.txt requirements/cloudflare.in
+	pip-compile --upgrade -o util/pingdom/requirements.txt requirements/pingdom.in
+	pip-compile --upgrade -o util/vpc-tools/requirements.txt requirements/vpc-tools.in
+	# Post process all of the files generated above to work around open pip-tools issues
+	util/post-pip-compile.sh \
+ 	    requirements/pip-tools.txt \
+	    requirements.txt \
+	    playbooks/roles/aws/templates/requirements.txt.j2 \
+	    requirements3.txt \
+	    util/elasticsearch/requirements.txt \
+	    util/jenkins/requirements-celery.txt \
+	    util/jenkins/requirements-cloudflare.txt \
+	    util/pingdom/requirements.txt \
+	    util/vpc-tools/requirements.txt
+
 include *.mk

github.txt

-# Dependencies install directly from github.  We typically segregate them in other projects,
-# and they cannot pass the VersionEye licence checks
-git+https://github.com/edx/ansible.git@stable-1.9.3-rc1-edx#egg=ansible==1.9.3-edx

playbooks/roles/aws/defaults/main.yml

@@ -35,6 +35,7 @@ aws_region: "us-east-1"
 # default path to the aws binary
 aws_s3cmd: "/usr/local/bin/s3cmd"
 aws_cmd: "/usr/local/bin/aws"
+aws_requirements: "{{ vhost_dirs.home.path }}/requirements.txt"
 #
 # OS packages
 #
@@ -42,10 +43,4 @@ aws_cmd: "/usr/local/bin/aws"
 aws_debian_pkgs:
   - python-setuptools
 
-aws_pip_pkgs:
-  - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
-  - awscli==1.11.58
-  - boto=="{{ common_boto_version }}"
-  - s3cmd==1.6.1
-
 aws_redhat_pkgs: []

playbooks/roles/aws/tasks/main.yml

@@ -54,12 +54,19 @@
   with_items: "{{ aws_debian_pkgs }}"
   when: ansible_distribution in common_debian_variants
 
+- name: Copy the Python requirements file
+  template:
+    src: "requirements.txt.j2"
+    dest: "{{ aws_requirements }}"
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Install aws python packages
   pip:
-    name: "{{ item }}"
+    requirements: "{{ aws_requirements }}"
     state: present
     extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-  with_items: "{{ aws_pip_pkgs }}"
 
 - name: Copy the boto global config file
   template:
@@ -105,4 +112,4 @@
     path: "{{ item.item }}"
     mode: "0644"
   when: vagrant_home_dir.stat.exists == False and ansible_distribution in common_debian_variants and item.stat.exists
-  with_items: "{{ motd_files_exist.results }}" 
+  with_items: "{{ motd_files_exist.results }}"

playbooks/roles/aws/templates/requirements.txt.j2

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
+awscli==1.11.58
+boto==2.48.0
+botocore==1.5.21          # via awscli, s3transfer
+colorama==0.3.7           # via awscli
+docutils==0.14            # via awscli, botocore
+futures==3.2.0            # via s3transfer
+jmespath==0.9.3           # via botocore
+pyasn1==0.4.2             # via rsa
+python-dateutil==2.7.3    # via botocore, s3cmd
+python-magic==0.4.15      # via s3cmd
+pyyaml==3.12              # via awscli
+rsa==3.4.2                # via awscli
+s3cmd==1.6.1
+s3transfer==0.1.13        # via awscli
+six==1.11.0               # via python-dateutil

requirements.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
 ansible==2.2.0.0
-PyYAML==3.12
-Jinja2==2.8
-MarkupSafe==1.0
+asn1crypto==0.24.0        # via cryptography
+awscli==1.15.19
+bcrypt==3.1.4             # via paramiko
+boto3==1.7.14
 boto==2.48.0
+botocore==1.10.19         # via awscli, boto3, s3transfer
+certifi==2018.4.16        # via requests
+cffi==1.11.5              # via bcrypt, cryptography, pynacl
+chardet==3.0.4            # via requests
+colorama==0.3.7           # via awscli
+cryptography==2.2.2       # via paramiko
+datadog==0.8.0
+decorator==4.3.0          # via datadog, networkx
+docopt==0.6.2
+docutils==0.14            # via awscli, botocore
 ecdsa==0.13
+enum34==1.1.6             # via cryptography
+futures==3.2.0            # via s3transfer
+idna==2.6                 # via cryptography, requests
+ipaddress==1.0.22         # via cryptography
+jinja2==2.8
+jmespath==0.9.3           # via boto3, botocore
+markupsafe==1.0
+mysql-python==1.2.5
+networkx==1.11
 paramiko==2.4.1
+pathlib2==2.3.0
+prettytable==0.7.2
+pyasn1==0.4.2             # via paramiko, rsa
+pycparser==2.18           # via cffi
 pycrypto==2.6.1
-wsgiref==0.1.2
-docopt==0.6.2
+pymongo==3.2.2
+pynacl==1.2.1             # via paramiko
+python-dateutil==2.7.3    # via botocore
 python-simple-hipchat==0.2
-prettytable==0.7.2
-# Don't forget to update the version of this installed in
-# the aws role as well.
-awscli==1.15.19
+pyyaml==3.12
 requests==2.18.4
-datadog==0.8.0
-networkx==1.11
-pathlib2==2.3.0
-boto3==1.7.14
-
-# Needed for the mongo_* modules (playbooks/library/mongo_*)
-pymongo==3.2.2
-
-# Needed for the mysql_db module
-MySQL-python==1.2.5
+rsa==3.4.2                # via awscli
+s3transfer==0.1.13        # via awscli, boto3
+scandir==1.7              # via pathlib2
+simplejson==3.15.0        # via datadog
+six==1.11.0               # via bcrypt, cryptography, pathlib2, pynacl, python-dateutil
+urllib3==1.22             # via requests
+wsgiref==0.1.2

requirements/aws.in

+# Python dependencies for the aws role
+
+awscli==1.11.58
+boto==2.48.0
+s3cmd==1.6.1

requirements/base.in

+# Standard dependencies for Ansible runs
+
+ansible==2.2.0.0
+awscli==1.15.19
+boto==2.48.0
+boto3==1.7.14
+datadog==0.8.0
+docopt==0.6.2
+ecdsa==0.13
+Jinja2==2.8
+MarkupSafe==1.0
+MySQL-python==1.2.5                 # Needed for the mysql_db module
+networkx==1.11
+paramiko==2.4.1
+pathlib2==2.3.0
+prettytable==0.7.2
+pycrypto==2.6.1
+pymongo==3.2.2                      # Needed for the mongo_* modules (playbooks/library/mongo_*)
+python-simple-hipchat==0.2
+PyYAML==3.12
+requests==2.18.4
+wsgiref==0.1.2

requirements/celery.in

+# Requirements for the celery/redis monitoring script in util/jenkins
+
+awscli==1.14.32
+backoff==1.4.3
+boto3==1.5.4
+click==6.7
+redis==2.10.6

requirements/cloudflare.in

+# Needed for CloudFlare cache hit rate job in util/jenkins
+
+requests==2.9.1

requirements/elasticsearch.in

+# Requirements for util/elasticsearch/verify-index-copy.py
+
+deepdiff==3.1.0
+elasticsearch==0.4.5

requirements/pingdom.in

+# Requirements for util/pingdom/create_pingdom_alerts.py
+
+click==6.7
+PyYAML==3.12
+requests==2.18.4

requirements/pip-tools.in

+# Just the dependencies to run pip-tools, mainly for the "upgrade" make target
+
+pip-tools                 # Contains pip-compile, used to generate pip requirements files

requirements/pip-tools.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
+click==6.7                # via pip-tools
+first==2.0.1              # via pip-tools
+pip-tools==2.0.2
+six==1.11.0               # via pip-tools

requirements/ses-limits.in

+# Requirements for the SES limits check job
+
+boto3==1.7.14
+awscli==1.15.19

requirements/vpc-tools.in

+# Requirements for the scripts in util/vpc-tools
+
+boto
+docopt
+python-simple-hipchat==0.2
+requests

requirements3.txt

-# Needed for SES limits check job
-boto3==1.7.14
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
 awscli==1.15.19
+boto3==1.7.14
+botocore==1.10.19         # via awscli, boto3, s3transfer
+colorama==0.3.7           # via awscli
+docutils==0.14            # via awscli, botocore
+futures==3.2.0            # via s3transfer
+jmespath==0.9.3           # via boto3, botocore
+pyasn1==0.4.2             # via rsa
+python-dateutil==2.7.3    # via botocore
+pyyaml==3.12              # via awscli
+rsa==3.4.2                # via awscli
+s3transfer==0.1.13        # via awscli, boto3
+six==1.11.0               # via python-dateutil

util/elasticsearch/requirements.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
 deepdiff==3.1.0
 elasticsearch==0.4.5
+jsonpickle==0.9.6         # via deepdiff
+urllib3==1.22             # via elasticsearch

util/jenkins/requirements-celery.txt

-# Requirements for celery/redis monitoring script
-redis==2.10.6
-click==6.7
-boto3==1.5.4
-backoff==1.4.3
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
 awscli==1.14.32
+backoff==1.4.3
+boto3==1.5.4
+botocore==1.8.36          # via awscli, boto3, s3transfer
+click==6.7
+colorama==0.3.7           # via awscli
+docutils==0.14            # via awscli, botocore
+futures==3.2.0            # via s3transfer
+jmespath==0.9.3           # via boto3, botocore
+pyasn1==0.4.2             # via rsa
+python-dateutil==2.7.3    # via botocore
+pyyaml==3.12              # via awscli
+redis==2.10.6
+rsa==3.4.2                # via awscli
+s3transfer==0.1.13        # via awscli, boto3
+six==1.11.0               # via python-dateutil

util/jenkins/requirements-cloudflare.txt

-# Needed for CloudFlare cache hit rate job
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
 requests==2.9.1

util/pingdom/requirements.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
+certifi==2018.4.16        # via requests
+chardet==3.0.4            # via requests
 click==6.7
-PyYAML==3.12
+idna==2.6                 # via requests
+pyyaml==3.12
 requests==2.18.4
+urllib3==1.22             # via requests

util/post-pip-compile.sh

+#!/usr/bin/env bash
+set -e
+
+# Remove any cruft from a requirements file generated by pip-tools which we don't want to keep
+
+function show_help {
+    echo "Usage: post-pip-compile.sh file ..."
+    echo "Remove any cruft left behind by pip-compile in the given requirements file(s)."
+    echo ""
+    echo "Updates the instructions for re-generating each requirements file, and removes"
+    echo "\"-e\" prefixes which were added to GitHub URLs only so that pip-compile could"
+    echo "process them correctly."
+}
+
+function clean_file {
+    FILE_PATH=$1
+    TEMP_FILE=${FILE_PATH}.tmp
+    # Replace the instructions for regenerating the output file.
+    sed "s/pip-compile --output-file.*/make upgrade/" ${FILE_PATH} > ${TEMP_FILE}
+    mv ${TEMP_FILE} ${FILE_PATH}
+}
+
+for i in "$@"; do
+    case ${i} in
+        -h|--help)
+            # help or unknown option
+            show_help
+            exit 0
+            ;;
+        *)
+            clean_file ${i}
+            ;;
+    esac
+done

util/vpc-tools/requirements.txt

-boto
-docopt
-requests
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
+boto==2.48.0
+certifi==2018.4.16        # via requests
+chardet==3.0.4            # via requests
+docopt==0.6.2
+idna==2.6                 # via requests
 python-simple-hipchat==0.2
+requests==2.18.4
+urllib3==1.22             # via requests