Testing and Py3 compatibility changes (#5)

* Made Py3 compatible

* Added unit tests using pytest

* Travis CI, versioneye, codecov, and tox integrations.  

* Pointed to new `edx/pyfilesystem` dependency (needed to support Py3)

* Updated license to Apache 2.0

.coveragerc

+[run]
+branch = True
+source = djpyfs
+data_file = .coverage
+omit = djpyfs/tests.py
+
+[report]
+exclude_lines =
+   pragma: no cover
+   raise NotImplementedError

.gitignore

@@ -53,4 +53,12 @@ coverage.xml
 docs/_build/
 
 # Emacs backups
-*~
\ No newline at end of file
+*~
+
+# PyCharm config
+.idea/
+
+# Paths generated in testing
+django-pyfs/
+example/db.sql
+example/sample/static/

.travis.yml

+language: python
+
+python:
+ - 2.7
+ - 3.5
+
+env:
+  - TOXENV=django18
+  - TOXENV=django19
+  - TOXENV=django110
+
+sudo: false
+
+branches:
+  only:
+  - master
+
+install:
+  - pip install -r requirements/test_requirements.txt
+
+script:
+  - tox
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

conftest.py

+from django.conf import settings
+
+
+def pytest_configure():
+    settings.configure(
+        DEBUG=True,
+        USE_TZ=True,
+        DATABASES={
+            "default": {
+                "ENGINE": "django.db.backends.sqlite3",
+            }
+        },
+        ROOT_URLCONF="test_urls",
+        INSTALLED_APPS=[
+            "django.contrib.auth",
+            "django.contrib.contenttypes",
+            "django.contrib.sites",
+            "django.contrib.sessions",
+            "djpyfs",
+        ],
+        MIDDLEWARE_CLASSES=[
+            'django.contrib.sessions.middleware.SessionMiddleware',
+            'restrictedsessions.middleware.RestrictedSessionsMiddleware',
+            'django.middleware.common.CommonMiddleware',
+            'django.middleware.csrf.CsrfViewMiddleware',
+            'django.contrib.auth.middleware.AuthenticationMiddleware',
+            'django.contrib.messages.middleware.MessageMiddleware',
+            'django.middleware.clickjacking.XFrameOptionsMiddleware',
+        ],
+        SITE_ID=1,
+    )

djpyfs/djpyfs.py

@@ -10,35 +10,27 @@ filesystem.
 task can garbage-collect those objects. 
 
 '''
+from __future__ import absolute_import
 
-import json
 import os
 import os.path
 import types
 
 from django.conf import settings
+from fs.osfs import OSFS
 
-from models import FSExpirations
+from .models import FSExpirations
 
 
 if hasattr(settings, 'DJFS'):
-    djfs_settings = settings.DJFS
+    djfs_settings = settings.DJFS  #pragma: no cover
 else:
     djfs_settings = {'type' : 'osfs',
                      'directory_root' : 'django-pyfs/static/django-pyfs', 
                      'url_root' : '/static/django-pyfs'}
 
-if djfs_settings['type'] == 'osfs':
-    from fs.osfs import OSFS
-elif djfs_settings['type'] == 's3fs':
-    from fs.s3fs import S3FS
-    from boto.s3.connection import S3Connection
-    from boto.s3.key import Key
-    key_id = djfs_settings.get('aws_access_key_id', None)
-    key_secret = djfs_settings.get('aws_secret_access_key', None)
-    s3conn = None
-else: 
-    raise AttributeError("Bad filesystem: "+str(djfs_settings['type']))
+s3conn = None
+
 
 def get_filesystem(namespace):
     ''' Returns a pyfilesystem for static module storage. 
@@ -56,7 +48,7 @@ def get_filesystem(namespace):
 
 def expire_objects():
     ''' Remove all obsolete objects from the file systems. Untested. '''
-    objects = sorted(FSExpirations.expired(), key=lambda x:x.module)
+    objects = sorted(FSExpirations.expired(), key=lambda x: x.module)
     fs = None
     module = None
     for o in objects:
@@ -98,12 +90,20 @@ def get_osfs(namespace):
     if not os.path.exists(full_path):
         os.makedirs(full_path)
     osfs = OSFS(full_path)
-    osfs = patch_fs(osfs, namespace, lambda self, filename, timeout=0:os.path.join(djfs_settings['url_root'], namespace, filename))
+    osfs = patch_fs(osfs, namespace, lambda self, filename, timeout=0: os.path.join(djfs_settings['url_root'], namespace, filename))
     return osfs
 
 def get_s3fs(namespace):
     ''' Helper method to get_filesystem for a file system on S3 '''
-    global key_id, key_secret
+    # Our test suite does not presume Amazon S3, and we would prefer not to have a global import so that we can run
+    # tests without requiring boto. These will become global when and if we include S3/boto in our test suite.
+    from fs.s3fs import S3FS
+    from boto.s3.connection import S3Connection
+
+    key_id = djfs_settings.get('aws_access_key_id', None)
+    key_secret = djfs_settings.get('aws_secret_access_key', None)
+    s3conn = None
+
     fullpath = namespace
     if 'prefix' in djfs_settings: 
         fullpath = os.path.join(djfs_settings['prefix'], fullpath)
@@ -121,4 +121,3 @@ def get_s3fs(namespace):
 
     s3fs = patch_fs(s3fs, namespace, get_s3_url)
     return s3fs
-

djpyfs/models.py

-import django
-from django.db import models
-import datetime
+import os
 
+from django.db import models
 from django.utils import timezone
 
-## Create your models here.
-#class StudentBookAccesses(models.Model):
-#    username = models.CharField(max_length=500, unique=True) # TODO: Should not have max_length
-#    count = models.IntegerField()
 
 class FSExpirations(models.Model):
-    ''' The modules have access to a pyfilesystem object where they
+    '''
+    Model to handle expiring temporary files.
+
+    The modules have access to a pyfilesystem object where they
     can store big data, images, etc. In most cases, we would like
     those objects to expire (e.g. if a view generates a .PNG analytic
     to show to a user). This model keeps track of files stored, as
-    well as the expirations of those models. 
+    well as the expirations of those models.
     '''
+    module = models.CharField(max_length=382)  # Defines the namespace
+    filename = models.CharField(max_length=382)  # Filename within the namespace
+    expires = models.BooleanField() # Does it expire?
+    expiration = models.DateTimeField(db_index=True)
 
     @classmethod
     def create_expiration(cls, module, filename, seconds, days=0, expires = True):
-        ''' May be used instead of the constructor to create a new expiration. 
+        '''
+        May be used instead of the constructor to create a new expiration.
         Automatically applies timedelta and saves to DB. 
         '''
         expiration_time = timezone.now() + timezone.timedelta(days, seconds)
 
         # If object exists, update it
-        objects = cls.objects.filter(module = module, filename = filename)
+        objects = cls.objects.filter(module=module, filename=filename)
         if objects:
             exp = objects[0]
             exp.expires = expires
@@ -41,30 +44,26 @@ class FSExpirations(models.Model):
         f.expiration = expiration_time
         f.save()
 
-    
-    module = models.CharField(max_length=382)  # Defines the namespace
-    filename = models.CharField(max_length=382)  # Filename within the namespace
-    expires = models.BooleanField() # Does it expire? 
-    expiration = models.DateTimeField(db_index = True) 
-
     @classmethod
     def expired(cls):
-        ''' Returns a list of expired objects '''
+        '''
+        Returns a list of expired objects
+        '''
 
         expiration_lte = timezone.now()
         return cls.objects.filter(expires=True, expiration__lte = expiration_lte)
 
-    class Meta:
+    class Meta(object):
         app_label = 'djpyfs'
-        unique_together = (("module","filename"))
+        unique_together = (("module", "filename"),)
         # We'd like to create an index first on expiration than on expires (so we can
         # search for objects where expires=True and expiration is before now).
         index_together = [
             ["expiration", "expires"],
-            ]
+        ]
 
     def __str__(self):
         if self.expires: 
-            return self.module+'/'+self.filename+" Expires "+str(self.expiration)
+            return "{} Expires {}".format(os.path.join(self.module, self.filename), str(self.expiration))
         else:
-            return self.module+'/'+self.filename+" Permanent ("+str(self.expiration)+")"
+            return "{} Permanent ({})".format(os.path.join(self.module, self.filename), str(self.expiration))

example/sample/views.py

+from future.builtins import map
 import png
 
 from django.http import HttpResponse
@@ -10,14 +11,17 @@ arrow = ["11011",
          "11011", 
          "11011", 
          "10001"]
-arrow = [map(int, x) for x in arrow]
+arrow = [list(map(int, x)) for x in arrow]
 
-def index(request):
+
+def index(_):
     fs = djpyfs.get_filesystem("sample")
     f = fs.open("uparrow.png", "wb")
-    png.Writer(len(arrow[0]), len(arrow), greyscale = True, bitdepth = 1).write(f, arrow)
+    png.Writer(len(arrow[0]), len(arrow), greyscale=True, bitdepth=1).write(f, arrow)
     f.close()
 
     url = fs.get_url("uparrow.png")
 
-    return HttpResponse("<html><body>Hello, world. You're at the polls index. <img src=\"{source}\"> </body></html>".format(source=url))
+    return HttpResponse(
+        "<html><body>Hello, world. You're at the test index. <img src=\"{source}\"> </body></html>".format(source=url)
+    )

pytest.ini

+[pytest]
+python_files = djpyfs/tests.py
+addopts = --cov djpyfs --cov-report term-missing

requirements/github.txt

+# This requirement breaks both pip-tools and versioneye. The requirements are unchanged from master. It is possible to
+# temporarily change this to fs for updating pinned versions using pip-tools, then change it back after, but you will
+# have to hand edit the resulting requirements.txt to replace this url since it contains changes necessary for Python 3.
+git+https://github.com/edx/pyfilesystem.git@bmedx/s3fs-py3-support#egg=fs==0.5.5a1

requirements/requirements.in

+-r github.txt
+six==1.10.0

requirements/requirements.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/requirements.txt requirements/requirements.in
+#
+appdirs==1.4.0            # via fs
+enum34==1.1.6             # via fs
+
+# This line is NOT autogenerated, see requirements/requirements.in for details.
+-r github.txt
+
+pytz==2016.10             # via fs
+scandir==1.4              # via fs
+six==1.10.0

requirements/test_requirements.in

+# Additional requirements for unit tests, you must also run the main requirements.txt.
+boto==2.45.0
+codecov==2.0.5
+moto==0.4.30
+pytest==3.0.5
+pytest-django==3.1.2
+pytest-cov==2.4.0
+tox==2.5.0
+
+# For the sample app
+pypng==0.0.18
+
+# Uncomment if you want to run the tests standalone. Tox will test against multiple versions by default.
+# Django>=1.8

requirements/test_requirements.txt

+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/test_requirements.txt requirements/test_requirements.in
+#
+argparse==1.4.0           # via codecov
+boto==2.45.0
+codecov==2.0.5
+coverage==4.3.1           # via codecov, pytest-cov
+httpretty==0.8.10         # via moto
+jinja2==2.8.1             # via moto
+markupsafe==0.23          # via jinja2
+moto==0.4.30
+pluggy==0.4.0             # via tox
+py==1.4.32                # via pytest, tox
+pypng==0.0.18
+pytest-cov==2.4.0
+pytest-django==3.1.2
+pytest==3.0.5
+python-dateutil==2.6.0    # via moto
+pytz==2016.10             # via moto
+requests==2.12.4          # via codecov, moto
+six==1.10.0               # via moto, python-dateutil
+tox==2.5.0
+virtualenv==15.1.0        # via tox
+werkzeug==0.11.15         # via moto
+xmltodict==0.10.2         # via moto

setup.py

@@ -17,13 +17,22 @@ setup(
     author='Piotr Mitros',
     author_email='pmitros@edx.org',
     packages=['djpyfs'],
-    license = "AGPLv3",
-    url = "https://github.com/edx/django-pyfs",
-    long_description = ld,
-    classifiers = [
+    license="Apache 2.0",
+    url="https://github.com/edx/django-pyfs",
+    long_description=ld,
+    classifiers=[
         "Development Status :: 4 - Beta",
+        "Framework :: Django",
+        "Framework :: Django :: 1.8",
+        "Framework :: Django :: 1.9",
+        "Framework :: Django :: 1.10",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.5",
         "Topic :: Software Development :: Libraries :: Python Modules",
-        "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+        "License :: OSI Approved :: Apache Software License",
     ],
-    install_requires=['fs'],
+    install_requires=['fs', 'six'],
 )

test_urls.py

+from django.conf.urls import patterns, url
+from django.http import HttpResponse
+
+
+urlpatterns = patterns(
+    '',
+    url(
+        regex=r'test_view/',
+        view=lambda r: HttpResponse(content="For unittests", status=200),
+        name='test_view'
+    ),
+)
\ No newline at end of file

tox.ini

+# Tox (http://tox.testrun.org/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = {py27,py35}-{django18,django19,django110}
+
+[testenv]
+passenv = CI TRAVIS TRAVIS_*
+commands = pytest
+deps =
+    -r{toxinidir}/requirements/requirements.txt
+    -r{toxinidir}/requirements/test_requirements.txt
+    django18: django==1.8
+    django19: django==1.9
+    django110: django==1.10