improve post processing

Use generators and add a gzip support.

Signed-off-by: Timothée Peignier <timothee.peignier@tryphon.org>

docs/storages.rst

@@ -52,6 +52,25 @@ tool like Bower. ::
       'pipeline.finders.CachedFileFinder',
   )
 
+GZIP compression
+================
+
+Pipeline can also creates a gzipped version of your collected static files,
+so that you can avoid compressing them on the fly. ::
+
+  STATICFILES_STORAGE = 'your.app.GZIPCachedStorage'
+
+The storage need to inherit from ``GzipMixin``: ::
+
+  from staticfiles.storage import CachedStaticFilesStorage
+
+  from pipeline.storage import GZIPMixin
+
+
+  class GZIPCachedStorage(GZIPMixin, CachedStaticFilesStorage):
+      pass
+
+
 Using with other storages
 =========================
 
@@ -69,7 +88,7 @@ Your storage only need to inherit from ``PipelineMixin`` and/or ``CachedFilesMix
 
 
   class S3PipelineStorage(PipelineMixin, CachedFilesMixin, S3BotoStorage):
-       pass
+      pass
 
 Using Pipeline with Bower
 =========================

pipeline/conf.py

 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import sys
-
 from django.conf import settings as _settings
 
 DEFAULTS = {

pipeline/storage.py

 from __future__ import unicode_literals
 
-import os
+import gzip
+
+from io import BytesIO
 
 from django.contrib.staticfiles import finders
-from django.contrib.staticfiles.storage import CachedFilesMixin, StaticFilesStorage
+from django.contrib.staticfiles.storage import CachedStaticFilesStorage, StaticFilesStorage
+from django.contrib.staticfiles.utils import matches_patterns
 
 from django.core.exceptions import ImproperlyConfigured
+from django.core.files.base import File
 from django.core.files.storage import get_storage_class
 from django.utils.functional import LazyObject
 
@@ -17,7 +21,7 @@ class PipelineMixin(object):
 
     def post_process(self, paths, dry_run=False, **options):
         if dry_run:
-            return []
+            return
 
         from pipeline.packager import Packager
         packager = Packager(storage=self)
@@ -27,21 +31,19 @@ class PipelineMixin(object):
             if self.packing:
                 packager.pack_stylesheets(package)
             paths[output_file] = (self, output_file)
+            yield output_file, output_file, True
         for package_name in packager.packages['js']:
             package = packager.package_for('js', package_name)
             output_file = package.output_filename
             if self.packing:
                 packager.pack_javascripts(package)
             paths[output_file] = (self, output_file)
+            yield output_file, output_file, True
 
         super_class = super(PipelineMixin, self)
         if hasattr(super_class, 'post_process'):
-            return super_class.post_process(paths, dry_run, **options)
-
-        return [
-            (path, path, True)
-            for path in paths
-        ]
+            for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options):
+                yield name, hashed_name, processed
 
     def get_available_name(self, name):
         if self.exists(name):
@@ -49,6 +51,40 @@ class PipelineMixin(object):
         return name
 
 
+class GZIPMixin(object):
+    gzip_patterns = ("*.css", "*.js")
+
+    def _compress(self, original_file):
+        content = BytesIO()
+        gzip_file = gzip.GzipFile(mode='wb', fileobj=content)
+        gzip_file.write(original_file.read())
+        gzip_file.close()
+        content.seek(0)
+        return File(content)
+
+    def post_process(self, paths, dry_run=False, **options):
+        super_class = super(GZIPMixin, self)
+        if hasattr(super_class, 'post_process'):
+            for name, hashed_name, processed in super_class.post_process(paths.copy(), dry_run, **options):
+                if hashed_name != name:
+                    paths[hashed_name] = (self, hashed_name)
+                yield name, hashed_name, processed
+
+        if dry_run:
+            return
+
+        for path in paths:
+            if not matches_patterns(path, self.gzip_patterns):
+                continue
+            original_file = self.open(path)
+            gzipped_path = "{0}.gz".format(path)
+            if self.exists(gzipped_path):
+                self.delete(gzipped_path)
+            gzipped_file = self._compress(original_file)
+            gzipped_path = self.save(gzipped_path, gzipped_file)
+            yield gzipped_path, gzipped_path, True
+
+
 class NonPackagingMixin(object):
     packing = False
 
@@ -61,7 +97,7 @@ class NonPackagingPipelineStorage(NonPackagingMixin, PipelineStorage):
     pass
 
 
-class PipelineCachedStorage(PipelineMixin, CachedFilesMixin, StaticFilesStorage):
+class PipelineCachedStorage(PipelineMixin, CachedStaticFilesStorage):
     pass
 
 

tests/assets/js/application.js

-function()  {
+function test() {
   alert('this is a test');
 }

tests/settings.py

@@ -64,6 +64,6 @@ PIPELINE_JS = {
             'pipeline/js/application.js',
             'pipeline/templates/**/*.jst'
         ),
-        'output_filename': 'scripts.css'
+        'output_filename': 'scripts.js'
     }
 }

tests/tests/test_storage.py

 from __future__ import unicode_literals
 
 from django.test import TestCase
-from django.utils.datastructures import SortedDict
 
 from pipeline.storage import PipelineStorage, PipelineFinderStorage
 
@@ -11,18 +10,15 @@ from tests.utils import pipeline_settings
 class StorageTest(TestCase):
     def test_post_process_dry_run(self):
         with pipeline_settings(PIPELINE_JS_COMPRESSOR=None, PIPELINE_CSS_COMPRESSOR=None):
-            processed_files = PipelineStorage().post_process([], True)
-            self.assertEqual(processed_files, [])
+            processed_files = PipelineStorage().post_process({}, True)
+            self.assertEqual(list(processed_files), [])
 
     def test_post_process(self):
         storage = PipelineStorage()
         with pipeline_settings(PIPELINE_JS_COMPRESSOR=None, PIPELINE_CSS_COMPRESSOR=None):
-            processed_files = storage.post_process(SortedDict({
-                'css/first.css': (storage, 'css/first.css'),
-                'images/arrow.png': (storage, 'images/arrow.png')
-            }))
-            self.assertTrue(('css/first.css', 'css/first.css', True) in processed_files)
-            self.assertTrue(('images/arrow.png', 'images/arrow.png', True) in processed_files)
+            processed_files = storage.post_process({})
+            self.assertTrue(('screen.css', 'screen.css', True) in processed_files)
+            self.assertTrue(('scripts.js', 'scripts.js', True) in processed_files)
 
     def test_find_storage(self):
         try:

tests/tests/test_template.py

@@ -35,7 +35,7 @@ class JinjaTest(TestCase):
 
     def test_package_js(self):
         template = self.env.from_string(u"""{% compressed_js "scripts" %}""")
-        self.assertEqual(u'<script   type="text/css" src="/static/scripts.css" charset="utf-8"></script>', template.render())
+        self.assertEqual(u'<script   type="text/javascript" src="/static/scripts.js" charset="utf-8"></script>', template.render())
 
 
 class DjangoTest(TestCase):
@@ -52,4 +52,4 @@ class DjangoTest(TestCase):
 
     def test_compressed_js(self):
         rendered = self.render_template(u"""{% load compressed %}{% compressed_js "scripts" %}""")
-        self.assertEqual(u'<script   type="text/css" src="/static/scripts.css" charset="utf-8"></script>', rendered)
+        self.assertEqual(u'<script   type="text/javascript" src="/static/scripts.js" charset="utf-8"></script>', rendered)

tests/utils.py

@@ -8,18 +8,15 @@ def _(path):
     # Make sure the path contains only the correct separator
     return path.replace('/', os.sep).replace('\\', os.sep)
 
+
 @contextlib.contextmanager
 def pipeline_settings(**kwargs):
     try:
         saved = {}
-
         for name, value in kwargs.items():
             saved[name] = getattr(settings, name)
             setattr(settings, name, value)
-
         yield
-
     finally:
-
         for name, value in saved.items():
             setattr(settings, name, value)