PLAT-1861 Upgrade to pyfilesystem2

.travis.yml

@@ -13,7 +13,7 @@ env:
 sudo: false
 
 install:
-  - pip install -r requirements/test_requirements.txt
+  - pip install -r requirements/travis.txt
 
 script:
   - tox

Makefile

+.PHONY: clean help requirements test test-all upgrade
+
+.DEFAULT_GOAL := help
+
+help: ## display this help message
+	@echo "Please use \`make <target>' where <target> is one of"
+	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+
+clean: ## remove generated byte code, coverage reports, and build artifacts
+	find . -name '__pycache__' -exec rm -rf {} +
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	coverage erase
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+requirements: ## install development environment requirements
+	pip install -qr requirements/dev.txt --exists-action w
+	pip-sync requirements/dev.txt requirements/private.* requirements/test.txt
+
+test: clean ## run tests in the current virtualenv
+	pytest
+
+test-all: ## run tests on every supported Python/Django combination
+	tox
+
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	pip install -q pip-tools
+	pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in
+	pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in
+	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
+	# Let tox control the django version for tests
+	sed '/^django==/d' requirements/test.txt > requirements/test.tmp
+	mv requirements/test.tmp requirements/test.txt

README.md

 django-pyfs
 ===========
 
-A Django module which extends pyfilesystem with several methods to
-make it convenient for web use. Specifically, it extends pyfilesystem
+A Django module which extends pyfilesystem2 with several methods to
+make it convenient for web use. Specifically, it extends pyfilesystem2
 with two methods:
 
     fs.get_url(filename, timeout=0)
@@ -28,8 +28,8 @@ To configure a django-pyfs to use static files, set a parameter in
 Django settings: 
 
     DJFS = {'type' : 'osfs',
-                     'directory_root' : 'djpyfs/static/djpyfs', 
-                     'url_root' : '/static/djpyfs'}
+            'directory_root' : 'djpyfs/static/djpyfs',
+            'url_root' : '/static/djpyfs'}
 
 Here, `directory_root` is where the files go. `url_root` is the URL
 base of where your web server is configured to serve them from.
@@ -51,17 +51,14 @@ Each module should pass a unique namespace. These will typically
 correspond to subdirectories within the filesystem. 
 
 The django-pyfs interface is designed as a generic (non-Django
-specific) extension to pyfilesystem. However, the specific
+specific) extension to pyfilesystem2. However, the specific
 implementation is very Django-specific. 
 
 Good next steps would be to:
 
-* Upgrade to use PyFilesystem2 when it supports S3
 * Allow Django storages to act as a back-end for pyfilesystem
 * Allow django-pyfs to act as a back-end for Django storages
 * Support more types of pyfilesystems (esp. in-memory would be nice)
 
 State: This code is tested and has worked well in a range of settings,
-and is currently deployed on edx.org. However, it doesn't have test
-cases and similar, so can't be considered truly production-ready. The
-expiration functionality, in particular, we are not using right now.
+and is currently deployed on edx.org.

djpyfs/djpyfs.py

@@ -18,7 +18,7 @@ import types
 from boto.s3.connection import S3Connection
 from django.conf import settings
 from fs.osfs import OSFS
-from fs.s3fs import S3FS
+from fs_s3fs import S3FS
 
 
 if hasattr(settings, 'DJFS'):
@@ -133,7 +133,7 @@ def get_s3fs(namespace):
 
     if 'prefix' in DJFS_SETTINGS:
         fullpath = os.path.join(DJFS_SETTINGS['prefix'], fullpath)
-    s3fs = S3FS(DJFS_SETTINGS['bucket'], fullpath, aws_access_key=key_id, aws_secret_key=key_secret)
+    s3fs = S3FS(DJFS_SETTINGS['bucket'], fullpath, aws_secret_access_key=key_id, aws_access_key_id=key_secret)
 
     def get_s3_url(self, filename, timeout=60):
         """

djpyfs/tests.py

@@ -166,9 +166,9 @@ class _BaseFs(TestCase):
         for curr_fs in (fs1, fs2):
             curr_fs.makedir(self.test_dir_name)
 
-            foo = StringIO("foo")
-            curr_fs.setcontents(self.relative_path_to_test_file, foo, 'utf-8', 'strict')
-            curr_fs.setcontents(self.relative_path_to_secondary_test_file, foo, 'utf-8', 'strict')
+            foo = 'foo'
+            curr_fs.settext(self.relative_path_to_test_file, foo, 'utf-8', 'strict')
+            curr_fs.settext(self.relative_path_to_secondary_test_file, foo, 'utf-8', 'strict')
 
             self.assertTrue(curr_fs.exists(self.relative_path_to_test_file))
             self.assertTrue(curr_fs.exists(self.relative_path_to_secondary_test_file))

requirements/base.in

+# Core requirements for using this package
+
+django>=1.8,<2.0                  # The web application framework this package enhances
+fs>=2.0.1                         # Filesystem API for Python
+fs-s3fs                           # Amazon S3 support for the fs package
+futures; python_version == "2.7"  # Transitive dependency, listed here to preserve the environment marker
+six==1.10.0                       # Utilities to enable Python 2 & 3 support

requirements/dev.in

+# Additional requirements for development of this application
+
+pip-tools                 # Requirements file management
+tox                       # virtualenv management for tests
+tox-battery               # Makes tox aware of requirements file changes

requirements/dev.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/dev.txt requirements/base.in requirements/dev.in
+#
+appdirs==1.4.3            # via fs
+boto3==1.4.8              # via fs-s3fs
+botocore==1.8.17          # via boto3, s3transfer
+click==6.7                # via pip-tools
+django==1.11.8
+docutils==0.14            # via botocore
+enum34==1.1.6             # via fs
+first==2.0.1              # via pip-tools
+fs-s3fs==0.1.5
+fs==2.0.17
+futures==3.2.0 ; python_version == "2.7"
+jmespath==0.9.3           # via boto3, botocore
+pip-tools==1.11.0
+pluggy==0.6.0             # via tox
+py==1.5.2                 # via tox
+python-dateutil==2.6.1    # via botocore
+pytz==2017.3              # via django, fs
+s3transfer==0.1.12        # via boto3
+six==1.10.0
+tox-battery==0.5
+tox==2.9.1
+virtualenv==15.1.0        # via tox

requirements/github.txt

-# This requirement breaks both pip-tools and versioneye. The requirements are
-# unchanged from master. It is possible to temporarily change this to fs for
-# updating pinned versions using pip-tools, then change it back after, but you
-# will have to hand edit the resulting requirements.txt to replace this url
-# since it contains changes necessary for Python 3 support.
-
-# The pip-tools issue should be cleared up with this PR (or when we move to a
-# full release of pyfilesystem or pyfilsystem2):
-# https://github.com/nvie/pip-tools/pull/405
-git+https://github.com/edx/pyfilesystem.git@bmedx/s3fs-py3-support#egg=fs==0.5.5a1

requirements/requirements.in

--r github.txt
-boto==2.45.0
-six==1.10.0

requirements/requirements.txt

-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file requirements/requirements.txt requirements/requirements.in
-#
-appdirs==1.4.0            # via fs
-boto==2.45.0
-enum34==1.1.6             # via fs
-
-# This line is NOT autogenerated, see requirements/requirements.in for details.
--r github.txt
-
-pytz==2016.10             # via fs
-scandir==1.4              # via fs
-six==1.10.0
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools                # via fs

requirements/test.in

+# Additional requirements for unit tests
+
+mock           # For setup of testing corner cases
+moto==0.4.30   # For mocking the boto3 Amazon S3 API
+pypng==0.0.18  # For the sample app
+pytest         # Test runner
+pytest-django  # pytest extension for better Django support
+pytest-cov     # pytest extension for code coverage statistics

requirements/test.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/test.txt requirements/base.in requirements/test.in
+#
+appdirs==1.4.3            # via fs
+attrs==17.3.0             # via pytest
+boto3==1.4.8              # via fs-s3fs
+boto==2.48.0              # via moto
+botocore==1.8.17          # via boto3, s3transfer
+certifi==2017.11.5        # via requests
+chardet==3.0.4            # via requests
+coverage==4.4.2           # via pytest-cov
+docutils==0.14            # via botocore
+enum34==1.1.6             # via fs
+fs-s3fs==0.1.5
+fs==2.0.17
+funcsigs==1.0.2           # via mock, pytest
+futures==3.2.0 ; python_version == "2.7"
+httpretty==0.8.10         # via moto
+idna==2.6                 # via requests
+jinja2==2.10              # via moto
+jmespath==0.9.3           # via boto3, botocore
+markupsafe==1.0           # via jinja2
+mock==2.0.0
+moto==0.4.30
+pbr==3.1.1                # via mock
+pluggy==0.6.0             # via pytest
+py==1.5.2                 # via pytest
+pypng==0.0.18
+pytest-cov==2.5.1
+pytest-django==3.1.2
+pytest==3.3.1
+python-dateutil==2.6.1    # via botocore, moto
+pytz==2017.3              # via django, fs, moto
+requests==2.18.4          # via moto
+s3transfer==0.1.12        # via boto3
+six==1.10.0
+urllib3==1.22             # via requests
+werkzeug==0.13            # via moto
+xmltodict==0.11.0         # via moto

requirements/test_requirements.in

-# Additional requirements for unit tests, you must also run the main requirements.txt.
-codecov==2.0.5
-mock==2.0.0
-moto==0.4.30
-pytest==3.0.5
-pytest-django==3.1.2
-pytest-cov==2.4.0
-tox==2.5.0
-
-# For the sample app
-pypng==0.0.18
-
-# Uncomment if you want to run the tests standalone. Tox will test against multiple versions by default.
-# Django>=1.8

requirements/test_requirements.txt

-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --output-file requirements/test_requirements.txt requirements/test_requirements.in
-#
-argparse==1.4.0           # via codecov
-boto==2.45.0              # via moto
-codecov==2.0.5
-coverage==4.3.1           # via codecov, pytest-cov
-funcsigs==1.0.2           # via mock
-httpretty==0.8.10         # via moto
-jinja2==2.8.1             # via moto
-markupsafe==0.23          # via jinja2
-mock==2.0.0
-moto==0.4.30
-pbr==1.10.0               # via mock
-pluggy==0.4.0             # via tox
-py==1.4.32                # via pytest, tox
-pypng==0.0.18
-pytest-cov==2.4.0
-pytest-django==3.1.2
-pytest==3.0.5
-python-dateutil==2.6.0    # via moto
-pytz==2016.10             # via moto
-requests==2.12.4          # via codecov, moto
-six==1.10.0               # via mock, moto, python-dateutil
-tox==2.5.0
-virtualenv==15.1.0        # via tox
-werkzeug==0.11.15         # via moto
-xmltodict==0.10.2         # via moto

requirements/travis.in

+# Requirements for running tests in Travis
+
+codecov                   # Code coverage reporting
+tox                       # Virtualenv management for tests
+tox-battery               # Makes tox aware of requirements file changes

requirements/travis.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/travis.txt requirements/travis.in
+#
+certifi==2017.11.5        # via requests
+chardet==3.0.4            # via requests
+codecov==2.0.10
+coverage==4.4.2           # via codecov
+idna==2.6                 # via requests
+pluggy==0.6.0             # via tox
+py==1.5.2                 # via tox
+requests==2.18.4          # via codecov
+six==1.11.0               # via tox
+tox-battery==0.5
+tox==2.9.1
+urllib3==1.22             # via requests
+virtualenv==15.1.0        # via tox

setup.py

@@ -12,7 +12,7 @@ else:
 
 setup(
     name='django-pyfs',
-    version='1.0.7',
+    version='2.0',
     description='Django pyfilesystem integration',
     author='Piotr Mitros',
     author_email='pmitros@edx.org',
@@ -35,5 +35,5 @@ setup(
         "Topic :: Software Development :: Libraries :: Python Modules",
         "License :: OSI Approved :: Apache Software License",
     ],
-    install_requires=['fs', 'boto', 'six', 'django'],
+    install_requires=['fs', 'fs-s3fs', 'six', 'django'],
 )

tox.ini

@@ -10,8 +10,7 @@ envlist = {py27,py35}-{django18,django19,django110,django111}
 passenv = CI TRAVIS TRAVIS_*
 commands = pytest {posargs}
 deps =
-    -r{toxinidir}/requirements/requirements.txt
-    -r{toxinidir}/requirements/test_requirements.txt
+    -r{toxinidir}/requirements/test.txt
     django18: django==1.8
     django19: django==1.9
     django110: django==1.10