Merge pull request #131 from edx/jmbowman/PLAT-1985

PLAT-1985 Better dependency management

.travis.yml

@@ -6,8 +6,7 @@ env:
 - TOXENV=django110
 - TOXENV=django111
 install:
-- pip install tox
-- pip install coveralls
+- pip install -r requirements/travis.txt
 script:
 - tox
 after_success: coveralls

MANIFEST.in

-include requirements.txt
-include test-requirements.txt
-include django-requirements.txt
+include requirements/base.in
+include requirements/test.in
 include LICENSE
 include AUTHORS
 include README.rst

Makefile

+.PHONY: clean coverage docs help quality requirements selfcheck test test-all upgrade validate
+
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+try:
+	from urllib import pathname2url
+except:
+	from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+help: ## display this help message
+	@echo "Please use \`make <target>' where <target> is one of"
+	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+
+clean: ## remove generated byte code, coverage reports, and build artifacts
+	find . -name '__pycache__' -exec rm -rf {} +
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	coverage erase
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+coverage: clean ## generate and view HTML coverage report
+	coverage report html
+	$(BROWSER) htmlcov/index.html
+
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	pip install -q pip-tools
+	pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in requirements/test.in requirements/travis.in
+	pip-compile --upgrade -o requirements/quality.txt requirements/base.in requirements/quality.in requirements/test.in
+	pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in
+	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
+	# Let tox control the Django version for tests
+	sed '/django==/d' requirements/test.txt > requirements/test.tmp
+	mv requirements/test.tmp requirements/test.txt
+
+quality: ## check coding style with pycodestyle and pylint
+	tox -e quality
+
+requirements: ## install development environment requirements
+	pip install -qr requirements/dev.txt --exists-action w
+	pip-sync requirements/dev.txt requirements/private.*
+
+test: clean ## run tests in the current virtualenv
+	python manage.py test
+
+diff_cover: test ## find diff lines that need test coverage
+	diff-cover coverage.xml
+
+test-all: ## run tests on every supported Python/Django combination
+	tox -e quality
+	tox
+
+validate: quality test ## run tests and quality checks
+
+selfcheck: ## check that the Makefile is well-formed
+	@echo "The Makefile is well-formed."

README.md → README.rst

@@ -5,41 +5,45 @@ edx-val is a django app that creates and retrieves metadata for videos and subti
  
 Example:
 Retrieve all profiles for a video with `edx_video_id`="example"
-```
->>> get_video_info("example")
-Returns (dict):
-{
-    'url' : '/edxval/videos/example',
-    'edx_video_id': u'example',
-    'duration': 111.0,
-    'client_video_id': u'The example video',
-    'encoded_videos': [
-        {
-            'url': u'http://www.example.com/example_mobile_video.mp4',
-            'file_size': 25556,
-            'bitrate': 9600,
-            'profile': u'mobile'
-        },
-        {
-            'url': u'http://www.example.com/example_desktop_video.mp4',
-            'file_size': 43096734,
-            'bitrate': 64000,
-            'profile': u'desktop'
-        }
-    ]
-}
-```
+
+.. code-block:: python
+
+    >>> get_video_info("example")
+    {
+        'url' : '/edxval/videos/example',
+        'edx_video_id': u'example',
+        'duration': 111.0,
+        'client_video_id': u'The example video',
+        'encoded_videos': [
+            {
+                'url': u'http://www.example.com/example_mobile_video.mp4',
+                'file_size': 25556,
+                'bitrate': 9600,
+                'profile': u'mobile'
+            },
+            {
+                'url': u'http://www.example.com/example_desktop_video.mp4',
+                'file_size': 43096734,
+                'bitrate': 64000,
+                'profile': u'desktop'
+            }
+        ]
+    }
+
  
 Developing
------------------
+----------
+
 First, create a virtual environment:
-```
-virtualenv venvs/val
-source venvs/val/bin/activate
-```
+
+.. code-block:: bash
+
+    virtualenv venvs/val
+    source venvs/val/bin/activate
+
 
 To run tests:
-```
-pip install -r test-requirements.txt
-tox  # to run only a single environment, do e.g. tox -e django18
-```
+
+.. code-block:: bash
+
+    make test

django-requirements.txt

-Django>=1.8,<2

requirements.txt

--e git+https://github.com/edx/django-oauth2-provider.git@0.2.7-fork-edx-6a#egg=django-oauth2-provider==0.2.7-fork-edx-6
--e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
-pillow==4.2.1
-boto==2.46.1
-django-model-utils==2.3.1
-django-storages==1.5.2
-enum34==1.1.6
-git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
-lxml==3.8.0

requirements/base.in

+# Core requirements for using this application
+
+boto
+Django>=1.8,<2
+django-model-utils
+-e git+https://github.com/edx/django-oauth2-provider.git@0.2.7-fork-edx-6a#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+-e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
+django-storages
+enum34
+lxml
+pillow

requirements/dev.in

+# Additional requirements for development of this application
+
+diff-cover                # Changeset diff test coverage
+pip-tools                 # Requirements file management

requirements/dev.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in requirements/test.in requirements/travis.in
+#
+-e git+https://github.com/edx/django-oauth2-provider.git@0.2.7-fork-edx-6a#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+appdirs==1.4.3            # via fs
+argparse==1.4.0           # via caniusepython3
+asn1crypto==0.24.0        # via cryptography
+astroid==1.5.2            # via edx-lint, pylint, pylint-celery, pylint-plugin-utils
+backports.functools-lru-cache==1.5  # via astroid, caniusepython3, pylint
+boto==2.48.0
+caniusepython3==6.0.0
+certifi==2018.1.18        # via requests, urllib3
+cffi==1.11.5              # via cryptography
+chardet==3.0.4            # via requests
+click-log==0.1.8          # via edx-lint
+click==6.7                # via click-log, edx-lint, pip-tools
+configparser==3.5.0       # via pylint
+coverage==4.5.1
+coveralls==1.2.0
+cryptography==2.1.4       # via pyopenssl, urllib3
+ddt==1.1.1
+diff-cover==1.0.2
+distlib==0.2.6            # via caniusepython3
+django-model-utils==3.1.1
+django-nose==1.4.5
+django-oauth-plus==2.2.9
+django-storages==1.6.5
+django==1.11.10
+docopt==0.6.2             # via coveralls
+edx-lint==0.5.5
+enum34==1.1.6
+first==2.0.1              # via pip-tools
+fs==2.0.18
+funcsigs==1.0.2           # via mock
+futures==3.2.0            # via caniusepython3, isort
+httplib2==0.10.3          # via oauth2
+idna==2.6                 # via cryptography, requests, urllib3
+inflect==0.2.5            # via jinja2-pluralize
+ipaddress==1.0.19         # via cryptography, urllib3
+isort==4.3.4
+jinja2-pluralize==0.3.0   # via diff-cover
+jinja2==2.10              # via diff-cover, jinja2-pluralize
+lazy-object-proxy==1.3.1  # via astroid
+lxml==4.1.1
+markupsafe==1.0           # via jinja2
+mccabe==0.6.1             # via pylint
+mock==2.0.0
+nose==1.3.7               # via django-nose
+oauth2==1.9.0.post1       # via django-oauth-plus
+packaging==17.1           # via caniusepython3
+pbr==3.1.1                # via mock
+pillow==5.0.0
+pip-tools==1.11.0
+pluggy==0.6.0             # via tox
+py==1.5.2                 # via tox
+pycodestyle==2.3.1
+pycparser==2.18           # via cffi
+pydocstyle==2.1.1
+pygments==2.2.0           # via diff-cover
+pylint-celery==0.3        # via edx-lint
+pylint-django==0.7.2      # via edx-lint
+pylint-plugin-utils==0.2.6  # via pylint-celery, pylint-django
+pylint==1.7.1             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
+pyopenssl==17.5.0         # via urllib3
+pyparsing==2.2.0          # via packaging
+pytz==2018.3              # via django, fs
+requests==2.18.4          # via caniusepython3, coveralls
+shortuuid==0.3
+singledispatch==3.4.0.3   # via astroid, pylint
+six==1.11.0               # via astroid, cryptography, diff-cover, edx-lint, fs, mock, packaging, pip-tools, pydocstyle, pylint, pyopenssl, singledispatch, tox
+snowballstemmer==1.2.1    # via pydocstyle
+tox-battery==0.5
+tox==2.9.1
+urllib3[secure]==1.22     # via coveralls, requests
+virtualenv==15.1.0        # via tox
+wrapt==1.10.11            # via astroid

requirements/private.readme

+# If there are any Python packages you want to keep in your virtualenv beyond
+# those listed in the official requirements files, create a "private.in" file
+# and list them there.  Generate the corresponding "private.txt" file pinning
+# all of their indirect dependencies to specific versions as follows:
+
+#     pip-compile private.in
+
+# This allows you to use "pip-sync" without removing these packages:
+
+#     pip-sync requirements/*.txt
+
+# "private.in" and "private.txt" aren't checked into git to avoid merge
+# conflicts, and the presence of this file allows "private.*" to be
+# included in scripted pip-sync usage without requiring that those files be
+# created first.

requirements/quality.in

+# Requirements for code quality checks
+
+caniusepython3            # Additional Python 3 compatibility pylint checks
+edx-lint                  # edX pylint rules and plugins
+isort                     # to standardize order of imports
+pycodestyle               # PEP 8 compliance validation
+pydocstyle                # PEP 257 compliance validation

requirements/quality.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/quality.txt requirements/base.in requirements/quality.in requirements/test.in
+#
+-e git+https://github.com/edx/django-oauth2-provider.git@0.2.7-fork-edx-6a#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+appdirs==1.4.3            # via fs
+argparse==1.4.0           # via caniusepython3
+astroid==1.5.2            # via edx-lint, pylint, pylint-celery, pylint-plugin-utils
+backports.functools-lru-cache==1.5  # via astroid, caniusepython3, pylint
+boto==2.48.0
+caniusepython3==6.0.0
+certifi==2018.1.18        # via requests
+chardet==3.0.4            # via requests
+click-log==0.1.8          # via edx-lint
+click==6.7                # via click-log, edx-lint
+configparser==3.5.0       # via pylint
+coverage==4.5.1
+ddt==1.1.1
+distlib==0.2.6            # via caniusepython3
+django-model-utils==3.1.1
+django-nose==1.4.5
+django-oauth-plus==2.2.9
+django-storages==1.6.5
+django==1.11.10
+edx-lint==0.5.5
+enum34==1.1.6
+fs==2.0.18
+funcsigs==1.0.2           # via mock
+futures==3.2.0            # via caniusepython3, isort
+httplib2==0.10.3          # via oauth2
+idna==2.6                 # via requests
+isort==4.3.4
+lazy-object-proxy==1.3.1  # via astroid
+lxml==4.1.1
+mccabe==0.6.1             # via pylint
+mock==2.0.0
+nose==1.3.7               # via django-nose
+oauth2==1.9.0.post1       # via django-oauth-plus
+packaging==17.1           # via caniusepython3
+pbr==3.1.1                # via mock
+pillow==5.0.0
+pycodestyle==2.3.1
+pydocstyle==2.1.1
+pylint-celery==0.3        # via edx-lint
+pylint-django==0.7.2      # via edx-lint
+pylint-plugin-utils==0.2.6  # via pylint-celery, pylint-django
+pylint==1.7.1             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
+pyparsing==2.2.0          # via packaging
+pytz==2018.3              # via django, fs
+requests==2.18.4          # via caniusepython3
+shortuuid==0.3
+singledispatch==3.4.0.3   # via astroid, pylint
+six==1.11.0               # via astroid, edx-lint, fs, mock, packaging, pydocstyle, pylint, singledispatch
+snowballstemmer==1.2.1    # via pydocstyle
+urllib3==1.22             # via requests
+wrapt==1.10.11            # via astroid

requirements/test.in

+# Requirements for test runs.
+
+coverage
+ddt
+django-nose
+fs
+mock

requirements/test.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/test.txt requirements/base.in requirements/test.in
+#
+-e git+https://github.com/edx/django-oauth2-provider.git@0.2.7-fork-edx-6a#egg=django-oauth2-provider==0.2.7-fork-edx-6
+-e git+https://github.com/edx/django-rest-framework-oauth.git@f0b503fda8c254a38f97fef802ded4f5fe367f7a#egg=djangorestframework-oauth
+-e git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+appdirs==1.4.3            # via fs
+boto==2.48.0
+coverage==4.5.1
+ddt==1.1.1
+django-model-utils==3.1.1
+django-nose==1.4.5
+django-oauth-plus==2.2.9
+django-storages==1.6.5
+enum34==1.1.6
+fs==2.0.18
+funcsigs==1.0.2           # via mock
+httplib2==0.10.3          # via oauth2
+lxml==4.1.1
+mock==2.0.0
+nose==1.3.7               # via django-nose
+oauth2==1.9.0.post1       # via django-oauth-plus
+pbr==3.1.1                # via mock
+pillow==5.0.0
+pytz==2018.3              # via django, fs
+shortuuid==0.3
+six==1.11.0               # via fs, mock

requirements/travis.in

+# Requirements for running tests in Travis
+
+coveralls                 # Code coverage reporting
+tox                       # Virtualenv management for tests
+tox-battery               # Makes tox aware of requirements file changes

requirements/travis.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/travis.txt requirements/travis.in
+#
+asn1crypto==0.24.0        # via cryptography
+certifi==2018.1.18        # via requests, urllib3
+cffi==1.11.5              # via cryptography
+chardet==3.0.4            # via requests
+coverage==4.5.1           # via coveralls
+coveralls==1.2.0
+cryptography==2.1.4       # via pyopenssl, urllib3
+docopt==0.6.2             # via coveralls
+enum34==1.1.6             # via cryptography
+idna==2.6                 # via cryptography, requests, urllib3
+ipaddress==1.0.19         # via cryptography, urllib3
+pluggy==0.6.0             # via tox
+py==1.5.2                 # via tox
+pycparser==2.18           # via cffi
+pyopenssl==17.5.0         # via urllib3
+requests==2.18.4          # via coveralls
+six==1.11.0               # via cryptography, pyopenssl, tox
+tox-battery==0.5
+tox==2.9.1
+urllib3[secure]==1.22     # via coveralls, requests
+virtualenv==15.1.0        # via tox

setup.py

@@ -34,7 +34,7 @@ def load_requirements(*requirements_paths):
     requirements = set()
     for path in requirements_paths:
         requirements.update(
-            line.strip() for line in open(path).readlines()
+            line.split('#')[0].strip() for line in open(path).readlines()
             if is_requirement(line)
         )
     return list(requirements)
@@ -54,6 +54,6 @@ setup(
         'Programming Language :: Python',
     ],
     packages=PACKAGES,
-    install_requires=load_requirements('requirements.txt', 'django-requirements.txt'),
-    tests_require=load_requirements('test-requirements.txt'),
+    install_requires=load_requirements('requirements/base.in'),
+    tests_require=load_requirements('requirements/test.in'),
 )

test-requirements.txt

-coverage==3.7.1
-ddt==0.8.0
-django-nose==1.4.4
-fs==2.0.17
-mock==1.0.1
-pylint==1.3.0

tox.ini

@@ -6,7 +6,19 @@ deps =
     django18: Django>=1.8,<1.9
     django110: Django>=1.10,<1.11
     django111: Django>=1.11,<2
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/test-requirements.txt
+    -r{toxinidir}/requirements/test.txt
 commands =
     python manage.py test {posargs}
+
+[testenv:quality]
+whitelist_externals =
+    make
+deps =
+    -r{toxinidir}/requirements/quality.txt
+commands =
+    pylint edxval
+    pylint --py3k edxval
+    pycodestyle edxval
+    pydocstyle edxval
+    isort --check-only --recursive edxval manage.py setup.py
+    make selfcheck