experiment with a collector process

The idea is to have a process similar to collecstatic, and
copy files in a temporary directory.

This avoid storage that use finders, streamline dev process with
production process, and make compilers easier to deal with.

.travis.yml

 language: python
 env:
-  - TOXENV=py26-1.5.X
-  - TOXENV=py27-1.5.X
-  - TOXENV=pypy-1.5.X
-  - TOXENV=py26
+  - TOXENV=py27-1.6.X
+  - TOXENV=pypy-1.6.X
+  - TOXENV=py33-1.6.X
   - TOXENV=py27
   - TOXENV=pypy
   - TOXENV=py33
-docsinstall: pip install -q --use-mirrors tox
+docsinstall: pip install -q tox
 script: tox

docs/conf.py

@@ -49,9 +49,9 @@ copyright = u'2011-2014, Timothée Peignier'
 # built documents.
 #
 # The short X.Y version.
-version = '1.3'
+version = '1.4'
 # The full version, including alpha/beta/rc tags.
-release = '1.3.27'
+release = '1.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pipeline/collector.py

+from __future__ import unicode_literals
+
+import os
+
+from collections import OrderedDict
+
+from django.contrib.staticfiles import finders
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+from pipeline.finders import PipelineFinder
+
+
+class Collector(object):
+    def __init__(self, storage=None):
+        if storage is None:
+            storage = staticfiles_storage
+        self.storage = storage
+
+    def clear(self, path=""):
+        dirs, files = self.storage.listdir(path)
+        for f in files:
+            fpath = os.path.join(path, f)
+            self.storage.delete(fpath)
+        for d in dirs:
+            self.clear(os.path.join(path, d))
+
+    def collect(self):
+        found_files = OrderedDict()
+        for finder in finders.get_finders():
+            # Ignore our finder to avoid looping
+            if isinstance(finder, PipelineFinder):
+                continue
+            for path, storage in finder.list(['CVS', '.*', '*~']):
+                # Prefix the relative path if the source storage contains it
+                if getattr(storage, 'prefix', None):
+                    prefixed_path = os.path.join(storage.prefix, path)
+                else:
+                    prefixed_path = path
+                if prefixed_path not in found_files:
+                    found_files[prefixed_path] = (storage, path)
+                    self.copy_file(path, prefixed_path, storage)
+
+    def copy_file(self, path, prefixed_path, source_storage):
+        # Delete the target file if needed or break
+        if not self.delete_file(path, prefixed_path, source_storage):
+            return
+        # Finally start copying
+        with source_storage.open(path) as source_file:
+            self.storage.save(prefixed_path, source_file)
+
+    def delete_file(self, path, prefixed_path, source_storage):
+        if self.storage.exists(prefixed_path):
+            try:
+                # When was the target file modified last time?
+                target_last_modified = self.storage.modified_time(prefixed_path)
+            except (OSError, NotImplementedError, AttributeError):
+                # The storage doesn't support ``modified_time`` or failed
+                pass
+            else:
+                try:
+                    # When was the source file modified last time?
+                    source_last_modified = source_storage.modified_time(path)
+                except (OSError, NotImplementedError, AttributeError):
+                    pass
+                else:
+                    # Skip the file if the source file is younger
+                    # Avoid sub-second precision
+                    if (target_last_modified.replace(microsecond=0)
+                            >= source_last_modified.replace(microsecond=0)):
+                            return False
+            # Then delete the existing file if really needed
+            self.storage.delete(prefixed_path)
+        return True
+
+default_collector = Collector()

pipeline/compilers/__init__.py

@@ -7,18 +7,19 @@ try:
 except ImportError:
     from pipes import quote
 
-from django.contrib.staticfiles import finders
+from django.contrib.staticfiles.storage import staticfiles_storage
 from django.core.files.base import ContentFile
 from django.utils.encoding import smart_str, smart_bytes
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompilerError
-from pipeline.storage import default_storage
 from pipeline.utils import to_class
 
 
 class Compiler(object):
-    def __init__(self, storage=default_storage, verbose=False):
+    def __init__(self, storage=None, verbose=False):
+        if storage is None:
+            storage = staticfiles_storage
         self.storage = storage
         self.verbose = verbose
 
@@ -32,7 +33,7 @@ class Compiler(object):
                 compiler = compiler(verbose=self.verbose, storage=self.storage)
                 if compiler.match_file(input_path):
                     output_path = self.output_path(input_path, compiler.output_extension)
-                    infile = finders.find(input_path)
+                    infile = self.storage.path(input_path)
                     outfile = self.output_path(infile, compiler.output_extension)
                     outdated = compiler.is_outdated(input_path, output_path)
                     try:

pipeline/compressors/__init__.py

@@ -7,10 +7,10 @@ import re
 
 from itertools import takewhile
 
+from django.contrib.staticfiles.storage import staticfiles_storage
 from django.utils.encoding import smart_bytes, force_text
 
 from pipeline.conf import settings
-from pipeline.storage import default_storage
 from pipeline.utils import to_class, relpath
 from pipeline.exceptions import CompressorError
 
@@ -39,7 +39,9 @@ FONT_EXTS = ['.ttf', '.otf', '.woff']
 class Compressor(object):
     asset_contents = {}
 
-    def __init__(self, storage=default_storage, verbose=False):
+    def __init__(self, storage=None, verbose=False):
+        if storage is None:
+            storage = staticfiles_storage
         self.storage = storage
         self.verbose = verbose
 
@@ -191,7 +193,7 @@ class Compressor(object):
         given the path of the stylesheet that contains it.
         """
         if posixpath.isabs(path):
-            path = posixpath.join(default_storage.location, path)
+            path = posixpath.join(staticfiles_storage.location, path)
         else:
             path = posixpath.join(start, path)
         return posixpath.normpath(path)
@@ -204,7 +206,7 @@ class Compressor(object):
 
     def read_bytes(self, path):
         """Read file content in binary mode"""
-        file = default_storage.open(path)
+        file = staticfiles_storage.open(path)
         content = file.read()
         file.close()
         return content

pipeline/conf.py

@@ -11,8 +11,6 @@ DEFAULTS = {
     'PIPELINE_ROOT': _settings.STATIC_ROOT,
     'PIPELINE_URL': _settings.STATIC_URL,
 
-    'PIPELINE_STORAGE': 'pipeline.storage.PipelineFinderStorage',
-
     'PIPELINE_CSS_COMPRESSOR': 'pipeline.compressors.yuglify.YuglifyCompressor',
     'PIPELINE_JS_COMPRESSOR': 'pipeline.compressors.yuglify.YuglifyCompressor',
     'PIPELINE_COMPILERS': [],

pipeline/finders.py

 from itertools import chain
 
-from django.contrib.staticfiles.finders import BaseFinder, AppDirectoriesFinder, FileSystemFinder, find
+from django.contrib.staticfiles.storage import staticfiles_storage
+from django.contrib.staticfiles.finders import BaseFinder, BaseStorageFinder, AppDirectoriesFinder, FileSystemFinder, find
 from django.utils._os import safe_join
 
 from pipeline.conf import settings
 
 
-class PipelineFinder(BaseFinder):
+class PipelineFinder(BaseStorageFinder):
+    storage = staticfiles_storage
+
+
+class ManifestFinder(BaseFinder):
     def find(self, path, all=False):
         """
         Looks for files in PIPELINE_CSS and PIPELINE_JS

pipeline/glob.py

@@ -4,7 +4,7 @@ import os
 import re
 import fnmatch
 
-from pipeline.storage import default_storage
+from django.contrib.staticfiles.storage import staticfiles_storage
 
 __all__ = ["glob", "iglob"]
 
@@ -26,7 +26,7 @@ def iglob(pathname):
     """
     if not has_magic(pathname):
         try:
-            if default_storage.exists(pathname):
+            if staticfiles_storage.exists(pathname):
                 yield pathname
         except NotImplementedError:
             # Being optimistic
@@ -56,7 +56,7 @@ def iglob(pathname):
 
 def glob1(dirname, pattern):
     try:
-        directories, files = default_storage.listdir(dirname)
+        directories, files = staticfiles_storage.listdir(dirname)
         names = directories + files
     except Exception:
         # We are not sure that dirname is a real directory
@@ -68,7 +68,7 @@ def glob1(dirname, pattern):
 
 
 def glob0(dirname, basename):
-    if default_storage.exists(os.path.join(dirname, basename)):
+    if staticfiles_storage.exists(os.path.join(dirname, basename)):
         return [basename]
     return []
 

pipeline/packager.py

 from __future__ import unicode_literals
 
+from django.contrib.staticfiles.storage import staticfiles_storage
 from django.contrib.staticfiles.finders import find
 from django.core.files.base import ContentFile
 from django.utils.encoding import smart_str
@@ -10,7 +11,6 @@ from pipeline.conf import settings
 from pipeline.exceptions import PackageNotFound
 from pipeline.glob import glob
 from pipeline.signals import css_compressed, js_compressed
-from pipeline.storage import default_storage
 
 
 class Package(object):
@@ -61,11 +61,13 @@ class Package(object):
 
 
 class Packager(object):
-    def __init__(self, storage=default_storage, verbose=False, css_packages=None, js_packages=None):
+    def __init__(self, storage=None, verbose=False, css_packages=None, js_packages=None):
+        if storage is None:
+            storage = staticfiles_storage
         self.storage = storage
         self.verbose = verbose
         self.compressor = Compressor(storage=storage, verbose=verbose)
-        self.compiler = Compiler(verbose=verbose)
+        self.compiler = Compiler(storage=storage, verbose=verbose)
         if css_packages is None:
             css_packages = settings.PIPELINE_CSS
         if js_packages is None:

pipeline/storage.py

 from __future__ import unicode_literals
 
 import gzip
+import tempfile
 
 from io import BytesIO
 
-from django.contrib.staticfiles import finders
 from django.contrib.staticfiles.storage import CachedStaticFilesStorage, StaticFilesStorage
 from django.contrib.staticfiles.utils import matches_patterns
 
-from django.core.exceptions import ImproperlyConfigured
 from django.core.files.base import File
-from django.core.files.storage import get_storage_class
-from django.utils.functional import LazyObject
 
 from pipeline.conf import settings
 
@@ -19,6 +16,11 @@ from pipeline.conf import settings
 class PipelineMixin(object):
     packing = True
 
+    def __init__(self, location=None, *args, **kwargs):
+        if settings.PIPELINE_ENABLED and location is None:
+            location = tempfile.mkdtemp()
+        super(PipelineMixin, self).__init__(location, *args, **kwargs)
+
     def post_process(self, paths, dry_run=False, **options):
         if dry_run:
             return
@@ -105,76 +107,3 @@ class PipelineCachedStorage(PipelineMixin, CachedStaticFilesStorage):
 class NonPackagingPipelineCachedStorage(NonPackagingMixin, PipelineCachedStorage):
     pass
 
-
-class BaseFinderStorage(PipelineStorage):
-    finders = None
-
-    def __init__(self, finders=None, *args, **kwargs):
-        if finders is not None:
-            self.finders = finders
-        if self.finders is None:
-            raise ImproperlyConfigured("The storage %r doesn't have a finders class assigned." % self.__class__)
-        super(BaseFinderStorage, self).__init__(*args, **kwargs)
-
-    def path(self, name):
-        path = self.finders.find(name)
-        if not path:
-            path = super(BaseFinderStorage, self).path(name)
-        return path
-
-    def exists(self, name):
-        exists = self.finders.find(name) is not None
-        if not exists:
-            return super(BaseFinderStorage, self).exists(name)
-        return exists
-
-    def listdir(self, path):
-        directories, files = [], []
-        for finder in self.finders.get_finders():
-            try:
-                storages = finder.storages.values()
-            except AttributeError:
-                continue
-            else:
-                for storage in storages:
-                    try:
-                        new_directories, new_files = storage.listdir(path)
-                    except OSError:
-                        pass
-                    else:
-                        directories.extend(new_directories)
-                        files.extend(new_files)
-        return directories, files
-
-    def find_storage(self, name):
-        for finder in self.finders.get_finders():
-            path = finder.find(name)
-            if path:
-                for storage in finder.storages.values():
-                    if path.startswith(storage.location):
-                        return path, storage
-
-        raise ValueError("The file '%s' could not be found with %r." % (name, self))
-
-    def _open(self, name, mode="rb"):
-        name, storage = self.find_storage(name)
-        return storage._open(name, mode)
-
-    def _save(self, name, content):
-        name, storage = self.find_storage(name)
-        # Ensure we overwrite file, since we have no control on external storage
-        if storage.exists(name):
-            storage.delete(name)
-        return storage._save(name, content)
-
-
-class PipelineFinderStorage(BaseFinderStorage):
-    finders = finders
-
-
-class DefaultStorage(LazyObject):
-    def _setup(self):
-        self._wrapped = get_storage_class(settings.PIPELINE_STORAGE)()
-
-
-default_storage = DefaultStorage()

pipeline/templatetags/pipeline.py

@@ -6,10 +6,12 @@ from django import template
 from django.template.loader import render_to_string
 from django.utils.safestring import mark_safe
 
+from ..collector import default_collector
 from ..conf import settings
 from ..packager import Packager, PackageNotFound
 from ..utils import guess_type
 
+
 register = template.Library()
 
 
@@ -35,6 +37,8 @@ class PipelineMixin(object):
             method = getattr(self, "render_{0}".format(package_type))
             return method(package, package.output_filename)
         else:
+            default_collector.collect()
+
             packager = Packager()
             method = getattr(self, "render_individual_{0}".format(package_type))
             paths = packager.compile(package.paths)

setup.py

@@ -6,7 +6,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='django-pipeline',
-    version='1.3.27',
+    version='1.4.0',
     description='Pipeline is an asset packaging library for Django.',
     long_description=io.open('README.rst', encoding='utf-8').read() + '\n\n' +
         io.open('HISTORY.rst', encoding='utf-8').read(),

tests/assets2/css/third.css

-.third {
-  display: none;
-}

tests/settings.py

@@ -8,6 +8,8 @@ DATABASES = {
     }
 }
 
+DEBUG = False
+
 SITE_ID = 1
 
 INSTALLED_APPS = [
@@ -37,11 +39,11 @@ STATIC_ROOT = local_path('static/')
 STATIC_URL = '/static/'
 STATICFILES_DIRS = (
     ('pipeline', local_path('assets/')),
-    local_path('assets2/'),
 )
 STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.FileSystemFinder',
-    'django.contrib.staticfiles.finders.AppDirectoriesFinder'
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'pipeline.finders.PipelineFinder',
 )
 
 SECRET_KEY = "django-pipeline"

tests/tests/static/app.css → tests/templates/empty.html

+ 
\ No newline at end of file

tests/templates/index.html

+{% load pipeline %}
+<html>
+ <head>
+  <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+  <title>Pipeline</title>
+  {% stylesheet 'screen' %}
+  {% javascript 'scripts' %}
+ </head>
+</html>

tests/tests/test_compiler.py

@@ -4,6 +4,8 @@ from django.test import TestCase
 
 from pipeline.conf import settings
 from pipeline.compilers import Compiler, CompilerBase
+from pipeline.collector import default_collector
+
 
 from tests.utils import _
 
@@ -20,6 +22,7 @@ class DummyCompiler(CompilerBase):
 
 class CompilerTest(TestCase):
     def setUp(self):
+        default_collector.collect()
         self.compiler = Compiler()
         self.old_compilers = settings.PIPELINE_COMPILERS
         settings.PIPELINE_COMPILERS = ['tests.tests.test_compiler.DummyCompiler']
@@ -40,4 +43,5 @@ class CompilerTest(TestCase):
         self.assertEqual([_('pipeline/js/dummy.js'), _('pipeline/js/application.js')], list(paths))
 
     def tearDown(self):
+        default_collector.clear()
         settings.PIPELINE_COMPILERS = self.old_compilers

tests/tests/test_compressor.py

@@ -15,6 +15,8 @@ from django.test.utils import override_settings
 from pipeline.compressors import Compressor, TEMPLATE_FUNC, \
     SubProcessCompressor
 from pipeline.compressors.yuglify import YuglifyCompressor
+from pipeline.collector import default_collector
+
 
 from tests.utils import _
 
@@ -23,6 +25,7 @@ class CompressorTest(TestCase):
     def setUp(self):
         self.maxDiff = None
         self.compressor = Compressor()
+        default_collector.collect()
 
     def test_js_compressor_class(self):
         self.assertEqual(self.compressor.js_compressor, YuglifyCompressor)
@@ -172,3 +175,6 @@ class CompressorTest(TestCase):
   content: "áéíóú";
 }
 """, output)
+
+    def tearDown(self):
+        default_collector.clear()

tests/tests/test_glob.py

@@ -29,8 +29,8 @@ class GlobTest(TestCase):
 
     def setUp(self):
         self.storage = FileSystemStorage(local_path('glob_dir'))
-        self.old_storage = glob.default_storage
-        glob.default_storage = self.storage
+        self.old_storage = glob.staticfiles_storage
+        glob.staticfiles_storage = self.storage
         self.mktemp('a', 'D')
         self.mktemp('aab', 'F')
         self.mktemp('aaa', 'zzzF')
@@ -47,7 +47,7 @@ class GlobTest(TestCase):
 
     def tearDown(self):
         shutil.rmtree(self.storage.location)
-        glob.default_storage = self.old_storage
+        glob.staticfiles_storage = self.old_storage
 
     def test_glob_literal(self):
         self.assertSequenceEqual(self.glob('a'),

tests/tests/test_packager.py

@@ -2,12 +2,16 @@ from __future__ import unicode_literals
 
 from django.test import TestCase
 
+from pipeline.collector import default_collector
 from pipeline.packager import Packager, PackageNotFound
 
 from tests.utils import _
 
 
 class PackagerTest(TestCase):
+    def setUp(self):
+        default_collector.collect()
+
     def test_package_for(self):
         packager = Packager()
         packager.packages['js'] = packager.create_packages({
@@ -39,3 +43,6 @@ class PackagerTest(TestCase):
             }
         })
         self.assertEqual(packages['templates'].templates, [_('pipeline/templates/photo/list.jst')])
+
+    def tearDown(self):
+        default_collector.clear()

tests/tests/test_storage.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from django.test import TestCase
 
-from pipeline.storage import PipelineStorage, PipelineFinderStorage
+from pipeline.storage import PipelineStorage
 
 from tests.utils import pipeline_settings
 
@@ -19,10 +19,3 @@ class StorageTest(TestCase):
             processed_files = storage.post_process({})
             self.assertTrue(('screen.css', 'screen.css', True) in processed_files)
             self.assertTrue(('scripts.js', 'scripts.js', True) in processed_files)
-
-    def test_find_storage(self):
-        try:
-            storage = PipelineFinderStorage()
-            storage.find_storage('app.css')
-        except ValueError:
-            self.fail()

tests/urls.py

-from django.conf.urls import patterns, include
+from django.conf.urls import patterns, include, url
 from django.contrib import admin
+from django.views.generic import TemplateView
+
 
 urlpatterns = patterns(
     '',
+    url(r'^$', TemplateView.as_view(template_name="index.html"), name="index"),
+    url(r'^empty/$', TemplateView.as_view(template_name="empty.html"), name="empty"),
     (r'^admin/', include(admin.site.urls)),
 )

tox.ini

 [tox]
 envlist =
-    py26-1.5.X, py27-1.5.X, pypy-1.5.X,
-    py26, py27, pypy, py33, py34, docs
+    py27-1.6.X, pypy-1.6.X, py33-1.6.X, py34-1.6.X,
+    py27, pypy, py33, py34,
+    docs
 
 [testenv]
 downloadcache = {toxworkdir}/_download/
@@ -11,40 +12,30 @@ setenv =
 commands =
     {envbindir}/django-admin.py test {posargs:tests}
 
-[testenv:py26-1.5.X]
-basepython = python2.6
-deps =
-    Django==1.5.5
-    mock
-    jinja2
-    futures
-
-[testenv:py27-1.5.X]
+[testenv:py27-1.6.X]
 basepython = python2.7
 deps =
-    Django==1.5.5
+    Django>=1.6.0, <1.7
     mock
     jinja2
     futures
 
-[testenv:pypy-1.5.X]
-basepython = pypy
+[testenv:py34-1.6.X]
+basepython = python3.4
 deps =
-    Django==1.5.5
-    mock
+    Django>=1.6.0, <1.7
     jinja2
-    futures
 
-[testenv:py33-1.5.X]
+[testenv:py33-1.6.X]
 basepython = python3.3
 deps =
+    Django>=1.6.0, <1.7
     jinja2
-    Django==1.5.5
 
-[testenv:py26]
-basepython = python2.6
+[testenv:pypy-1.6.X]
+basepython = pypy
 deps =
-    Django==1.6
+    Django>=1.6.0, <1.7
     mock
     jinja2
     futures
@@ -52,7 +43,7 @@ deps =
 [testenv:py27]
 basepython = python2.7
 deps =
-    Django==1.6
+    https://www.djangoproject.com/m/releases/1.7/Django-1.7b1.tar.gz#egg=Django
     mock
     jinja2
     futures
@@ -60,7 +51,7 @@ deps =
 [testenv:pypy]
 basepython = pypy
 deps =
-    Django==1.6
+    https://www.djangoproject.com/m/releases/1.7/Django-1.7b1.tar.gz#egg=Django
     mock
     jinja2
     futures
@@ -69,7 +60,13 @@ deps =
 basepython = python3.3
 deps =
     jinja2
-    Django==1.6
+    https://www.djangoproject.com/m/releases/1.7/Django-1.7b1.tar.gz#egg=Django
+
+[testenv:py34]
+basepython = python3.4
+deps =
+    jinja2
+    https://www.djangoproject.com/m/releases/1.7/Django-1.7b1.tar.gz#egg=Django
 
 [testenv:py34]
 basepython = python3.4