Merge pull request #531 from theatlantic/pr/compiler-compressor-unit-tests

Add unit tests for the bundled compiler and compressor classes

.gitignore

@@ -13,6 +13,7 @@ docs/_build/
 coverage/
 tests/static/
 tests/assets/js/dummy.js
+tests/node_modules/
 .tox/
 .DS_Store
 .idea
@@ -21,3 +22,6 @@ tests/assets/js/dummy.js
 .pydevproject
 .ropeproject
 __pycache__
+npm-debug.log
+tests/npm-cache
+django-pipeline-*/

MANIFEST.in

@@ -3,5 +3,8 @@ recursive-include pipeline/jinja2 *.html *.jinja
 include AUTHORS LICENSE README.rst HISTORY.rst
 recursive-include tests *
 recursive-exclude tests *.pyc *.pyo
+recursive-exclude tests/node_modules *
+recursive-exclude tests/npm-cache *
+recursive-exclude tests/npm *
 include docs/Makefile docs/make.bat docs/conf.py
 recursive-include docs *.rst

pipeline/compilers/__init__.py

@@ -12,7 +12,7 @@ from django.utils.six import string_types
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompilerError
-from pipeline.utils import to_class
+from pipeline.utils import to_class, set_std_streams_blocking
 
 
 class Compiler(object):
@@ -120,6 +120,7 @@ class SubProcessCompiler(CompilerBase):
                                              stdout=stdout,
                                              stderr=subprocess.PIPE)
                 _, stderr = compiling.communicate()
+                set_std_streams_blocking()
 
             if compiling.returncode != 0:
                 stdout_captured = None  # Don't save erroneous result.

pipeline/compressors/__init__.py

@@ -14,7 +14,7 @@ from django.utils.six import string_types
 
 from pipeline.conf import settings
 from pipeline.exceptions import CompressorError
-from pipeline.utils import to_class, relpath
+from pipeline.utils import to_class, relpath, set_std_streams_blocking
 
 URL_DETECTOR = r"""url\((['"]){0,1}\s*(.*?)["']{0,1}\)"""
 URL_REPLACER = r"""url\(__EMBED__(.+?)(\?\d+)?\)"""
@@ -248,6 +248,7 @@ class SubProcessCompressor(CompressorBase):
         if content:
             content = smart_bytes(content)
         stdout, stderr = pipe.communicate(content)
+        set_std_streams_blocking()
         if stderr.strip() and pipe.returncode != 0:
             raise CompressorError(stderr)
         elif self.verbose:

pipeline/conf.py

 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import os
 import collections
 import shlex
 
@@ -101,7 +102,7 @@ class PipelineSettings(collections.MutableMapping):
         value = self.settings[key]
         if key.endswith(("_BINARY", "_ARGUMENTS")):
             if isinstance(value, string_types):
-                return tuple(shlex.split(value))
+                return tuple(shlex.split(value, posix=(os.name == 'posix')))
             return tuple(value)
         return value
 

pipeline/utils.py

 from __future__ import unicode_literals
 
+try:
+    import fcntl
+except ImportError:
+    # windows
+    fcntl = None
+
 import importlib
 import mimetypes
 import posixpath
+import os
+import sys
 
 try:
     from urllib.parse import quote
@@ -54,3 +62,19 @@ def relpath(path, start=posixpath.curdir):
     if not rel_list:
         return posixpath.curdir
     return posixpath.join(*rel_list)
+
+
+def set_std_streams_blocking():
+    """
+    Set stdout and stderr to be blocking.
+
+    This is called after Popen.communicate() to revert stdout and stderr back
+    to be blocking (the default) in the event that the process to which they
+    were passed manipulated one or both file descriptors to be non-blocking.
+    """
+    if not fcntl:
+        return
+    for f in (sys.__stdout__, sys.__stderr__):
+        fileno = f.fileno()
+        flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
+        fcntl.fcntl(fileno, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)

tests/assets/compilers/coffee/expected.js

+(function() {
+  var cube, square;
+
+  square = function(x) {
+    return x * x;
+  };
+
+  cube = function(x) {
+    return square(x) * x;
+  };
+
+}).call(this);

tests/assets/compilers/coffee/input.coffee

+square = (x) -> x * x
+cube   = (x) -> square(x) * x

tests/assets/compilers/es6/expected.js

+"use strict";
+
+// Expression bodies
+var odds = evens.map(function (v) {
+  return v + 1;
+});
+var nums = evens.map(function (v, i) {
+  return v + i;
+});
+
+// Statement bodies
+nums.forEach(function (v) {
+  if (v % 5 === 0) fives.push(v);
+});
+
+// Lexical this
+var bob = {
+  _name: "Bob",
+  _friends: [],
+  printFriends: function printFriends() {
+    var _this = this;
+
+    this._friends.forEach(function (f) {
+      return console.log(_this._name + " knows " + f);
+    });
+  }
+};

tests/assets/compilers/es6/input.es6

+// Expression bodies
+var odds = evens.map(v => v + 1);
+var nums = evens.map((v, i) => v + i);
+
+// Statement bodies
+nums.forEach(v => {
+  if (v % 5 === 0)
+    fives.push(v);
+});
+
+// Lexical this
+var bob = {
+  _name: "Bob",
+  _friends: [],
+  printFriends() {
+    this._friends.forEach(f =>
+      console.log(this._name + " knows " + f));
+  }
+};

tests/assets/compilers/less/expected.css

+.a {
+  width: 1px;
+}

tests/assets/compilers/less/input.less

+@a: 1;
+
+.a {
+    width: (@a + 0px);
+}

tests/assets/compilers/livescript/expected.js

+(function(){
+  var times;
+  times = function(x, y){
+    return x * y;
+  };
+}).call(this);

tests/assets/compilers/livescript/input.ls

+times = (x, y) ->
+  x * y

tests/assets/compilers/scss/expected.css

+.a .b {
+  display: none; }
+
+.c .d {
+  display: block; }

tests/assets/compilers/scss/input.scss

+.a {
+    .b {
+        display: none;
+    }
+}
+.c {
+    .d {
+        display: block;
+    }
+}

tests/assets/compilers/stylus/expected.css

+.a {
+  color: #000;
+}

tests/assets/compilers/stylus/input.styl

+.a
+    color: black
\ No newline at end of file

tests/assets/compressors/closure.js

+(function(){(function(){window.concat=function(){console.log(arguments)}})();(function(){window.cat=function(){console.log("hello world")}})()}).call(this);

tests/assets/compressors/cssmin.css

+.concat{display:none}.concatenate{display:block}
\ No newline at end of file

tests/assets/compressors/csstidy.css

+.concat{display:none;}.concatenate{display:block;}
\ No newline at end of file

tests/assets/compressors/jsmin.js

+(function(){(function(){window.concat=function(){console.log(arguments);}}());(function(){window.cat=function(){console.log("hello world");}}());}).call(this);
\ No newline at end of file

tests/assets/compressors/slimit.js

+(function(){(function(){window.concat=function(){console.log(arguments);};}());(function(){window.cat=function(){console.log("hello world");};}());}).call(this);
\ No newline at end of file

tests/assets/compressors/uglifyjs.js

+(function(){(function(){window.concat=function(){console.log(arguments)}})();(function(){window.cat=function(){console.log("hello world")}})()}).call(this);

tests/assets/compressors/yuglify.css

+.concat{display:none}.concatenate{display:block}

tests/assets/compressors/yuglify.js

+(function(){(function(){window.concat=function(){console.log(arguments)}})(),function(){window.cat=function(){console.log("hello world")}}()}).call(this);

tests/assets/compressors/yui.css

+.concat{display:none}.concatenate{display:block}
\ No newline at end of file

tests/assets/compressors/yui.js

+(function(){(function(){window.concat=function(){console.log(arguments)}}());(function(){window.cat=function(){console.log("hello world")}}())}).call(this);
\ No newline at end of file

tests/package.json

+{
+  "name": "django-pipeline-tests",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Pipeline is an asset packaging library for Django.",
+  "author": "Timothée Peignier <timothee.peignier@tryphon.org>",
+  "license": "MIT",
+  "readmeFilename": "../README.rst",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/jazzband/django-pipeline.git"
+  },
+  "dependencies": {
+    "babel-cli": "^6.4.5",
+    "babel-preset-es2015": "^6.3.13",
+    "coffee-script": "^1.10.0",
+    "less": "^2.5.3",
+    "livescript": "^1.4.0",
+    "node-sass": "^3.4.2",
+    "stylus": "^0.53.0",
+    "cssmin": "^0.4.3",
+    "google-closure-compiler": "^20151216.2.0",
+    "uglify-js": "^2.6.1",
+    "yuglify": "^0.1.4",
+    "yuicompressor": "^2.4.8"
+  }
+}

tests/scripts/npm_install.py

+#!/usr/bin/env python
+"""
+A cross-platform compatible `npm install` call, checking whether npm is
+in fact installed on the system first (and on windows, checking that the
+npm version is at least 3.0 because of a bug in 2.x with MAX_PATH)
+"""
+import distutils.spawn
+import os
+from pkg_resources import parse_version
+import re
+import subprocess
+import sys
+
+
+def main():
+    tests_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+    if os.name == 'nt':
+        try:
+            npm_paths = subprocess.check_output(['where', 'npm.cmd'])
+        except subprocess.CalledProcessError:
+            return
+        else:
+            npm_bin = re.split(r'\r?\n', npm_paths)[0]
+    else:
+        npm_bin = distutils.spawn.find_executable('npm')
+    if not npm_bin:
+        return
+    if os.name == 'nt':
+        os.environ.setdefault('APPDATA', '.')
+        npm_version = subprocess.check_output([npm_bin, '--version']).strip()
+        # Skip on windows if npm version is less than 3 because of
+        # MAX_PATH issues in version 2
+        if parse_version(npm_version) < parse_version('3.0'):
+            return
+    pipe = subprocess.Popen([npm_bin, 'install'],
+        cwd=tests_dir, stdout=sys.stdout, stderr=sys.stderr)
+    pipe.communicate()
+    sys.exit(pipe.returncode)
+
+
+if __name__ == '__main__':
+    main()

tests/settings.py

+import glob
 import os
+import distutils.spawn
 
 
 def local_path(path):
@@ -58,6 +60,8 @@ SECRET_KEY = "django-pipeline"
 
 PIPELINE = {
     'PIPELINE_ENABLED': True,
+    'JS_COMPRESSOR': None,
+    'CSS_COMPRESSOR': None,
     'STYLESHEETS': {
         'screen': {
             'source_filenames': (
@@ -118,6 +122,48 @@ PIPELINE = {
     }
 }
 
+NODE_MODULES_PATH = local_path('node_modules')
+NODE_BIN_PATH = os.path.join(NODE_MODULES_PATH, '.bin')
+NODE_EXE_PATH = distutils.spawn.find_executable('node')
+JAVA_EXE_PATH = distutils.spawn.find_executable('java')
+CSSTIDY_EXE_PATH = distutils.spawn.find_executable('csstidy')
+HAS_NODE = os.path.exists(NODE_BIN_PATH) and NODE_EXE_PATH
+HAS_JAVA = bool(JAVA_EXE_PATH)
+HAS_CSSTIDY = bool(CSSTIDY_EXE_PATH)
+
+if HAS_NODE:
+    def node_exe_path(command):
+        exe_ext = '.cmd' if os.name == 'nt' else ''
+        return os.path.join(NODE_BIN_PATH, "%s%s" % (command, exe_ext))
+
+    PIPELINE.update({
+        'SASS_BINARY': node_exe_path('node-sass'),
+        'COFFEE_SCRIPT_BINARY': node_exe_path('coffee'),
+        'COFFEE_SCRIPT_ARGUMENTS': ['--no-header'],
+        'LESS_BINARY': node_exe_path('lessc'),
+        'BABEL_BINARY': node_exe_path('babel'),
+        'BABEL_ARGUMENTS': ['--presets', 'es2015'],
+        'STYLUS_BINARY': node_exe_path('stylus'),
+        'LIVE_SCRIPT_BINARY': node_exe_path('lsc'),
+        'LIVE_SCRIPT_ARGUMENTS': ['--no-header'],
+        'YUGLIFY_BINARY': node_exe_path('yuglify'),
+        'UGLIFYJS_BINARY': node_exe_path('uglifyjs'),
+        'CSSMIN_BINARY': node_exe_path('cssmin'),
+    })
+
+if HAS_NODE and HAS_JAVA:
+    PIPELINE.update({
+        'CLOSURE_BINARY': [
+            JAVA_EXE_PATH, '-jar',
+            os.path.join(NODE_MODULES_PATH, 'google-closure-compiler', 'compiler.jar')],
+        'YUI_BINARY': [
+            JAVA_EXE_PATH, '-jar',
+            glob.glob(os.path.join(NODE_MODULES_PATH, 'yuicompressor', 'build', '*.jar'))[0]]
+    })
+
+if HAS_CSSTIDY:
+    PIPELINE.update({'CSSTIDY_BINARY': CSSTIDY_EXE_PATH})
+
 TEMPLATE_DIRS = (
     local_path('templates'),
 )

tests/tests/__init__.py

 # -*- coding: utf-8 flake8: noqa -*-
+import os
+import sys
+
+
+if sys.platform.startswith('win'):
+    os.environ.setdefault('NUMBER_OF_PROCESSORS', '1')
+
+
 from .test_compiler import *
 from .test_compressor import *
 from .test_template import *

tests/tests/test_compiler.py

 from __future__ import unicode_literals
 
 import sys
-from unittest import skipIf
+from unittest import skipIf, skipUnless
 
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
 from django.test import TestCase
+from django.test.client import RequestFactory
+from django.utils.encoding import smart_bytes
 
 from pipeline.collector import default_collector
 from pipeline.compilers import Compiler, CompilerBase, SubProcessCompiler
 from pipeline.exceptions import CompilerError
+from pipeline.utils import to_class
 
 from tests.utils import _, pipeline_settings
 
@@ -169,3 +174,56 @@ class FailingCompilerTest(TestCase):
 
     def tearDown(self):
         default_collector.clear()
+
+
+@skipUnless(settings.HAS_NODE, "requires node")
+class CompilerImplementation(TestCase):
+
+    def setUp(self):
+        self.compiler = Compiler()
+        default_collector.collect(RequestFactory().get('/'))
+
+    def tearDown(self):
+        default_collector.clear()
+
+    def _test_compiler(self, compiler_cls_str, infile, expected):
+        compiler_cls = to_class(compiler_cls_str)
+        compiler = compiler_cls(verbose=False, storage=staticfiles_storage)
+        infile_path = staticfiles_storage.path(infile)
+        outfile_path = compiler.output_path(infile_path, compiler.output_extension)
+        compiler.compile_file(_(infile_path), _(outfile_path), force=True)
+        with open(outfile_path) as f:
+            result = f.read()
+        with staticfiles_storage.open(expected) as f:
+            expected = f.read()
+        self.assertEqual(smart_bytes(result), expected)
+
+    def test_sass(self):
+        self._test_compiler('pipeline.compilers.sass.SASSCompiler',
+            'pipeline/compilers/scss/input.scss',
+            'pipeline/compilers/scss/expected.css')
+
+    def test_coffeescript(self):
+        self._test_compiler('pipeline.compilers.coffee.CoffeeScriptCompiler',
+            'pipeline/compilers/coffee/input.coffee',
+            'pipeline/compilers/coffee/expected.js')
+
+    def test_less(self):
+        self._test_compiler('pipeline.compilers.less.LessCompiler',
+            'pipeline/compilers/less/input.less',
+            'pipeline/compilers/less/expected.css')
+
+    def test_es6(self):
+        self._test_compiler('pipeline.compilers.es6.ES6Compiler',
+            'pipeline/compilers/es6/input.es6',
+            'pipeline/compilers/es6/expected.js')
+
+    def test_stylus(self):
+        self._test_compiler('pipeline.compilers.stylus.StylusCompiler',
+            'pipeline/compilers/stylus/input.styl',
+            'pipeline/compilers/stylus/expected.css')
+
+    def test_livescript(self):
+        self._test_compiler('pipeline.compilers.livescript.LiveScriptCompiler',
+            'pipeline/compilers/livescript/input.ls',
+            'pipeline/compilers/livescript/expected.js')

tests/tests/test_compressor.py

@@ -11,18 +11,24 @@ try:
 except ImportError:
     from unittest.mock import patch  # noqa
 
-from unittest import skipIf
+from unittest import skipIf, skipUnless
 
+from django.conf import settings
 from django.test import TestCase
+from django.test.client import RequestFactory
+from django.utils.encoding import smart_bytes
 
-from pipeline.compressors import Compressor, TEMPLATE_FUNC, \
-    SubProcessCompressor
+from pipeline.compressors import (
+    Compressor, TEMPLATE_FUNC, SubProcessCompressor)
 from pipeline.compressors.yuglify import YuglifyCompressor
 from pipeline.collector import default_collector
 
 from tests.utils import _, pipeline_settings
 
 
+@pipeline_settings(
+    CSS_COMPRESSOR='pipeline.compressors.yuglify.YuglifyCompressor',
+    JS_COMPRESSOR='pipeline.compressors.yuglify.YuglifyCompressor')
 class CompressorTest(TestCase):
     def setUp(self):
         self.maxDiff = None
@@ -186,3 +192,84 @@ class CompressorTest(TestCase):
 
     def tearDown(self):
         default_collector.clear()
+
+
+class CompressorImplementationTest(TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.compressor = Compressor()
+        default_collector.collect(RequestFactory().get('/'))
+
+    def tearDown(self):
+        default_collector.clear()
+
+    def _test_compressor(self, compressor_cls, compress_type, expected_file):
+        override_settings = {
+            ("%s_COMPRESSOR" % compress_type.upper()): compressor_cls,
+        }
+        with pipeline_settings(**override_settings):
+            if compress_type == 'js':
+                result = self.compressor.compress_js(
+                    [_('pipeline/js/first.js'), _('pipeline/js/second.js')])
+            else:
+                result = self.compressor.compress_css(
+                    [_('pipeline/css/first.css'), _('pipeline/css/second.css')],
+                    os.path.join('pipeline', 'css', os.path.basename(expected_file)))
+        with self.compressor.storage.open(expected_file) as f:
+            expected = f.read()
+        self.assertEqual(smart_bytes(result), expected)
+
+    def test_jsmin(self):
+        self._test_compressor('pipeline.compressors.jsmin.JSMinCompressor',
+            'js', 'pipeline/compressors/jsmin.js')
+
+    def test_slimit(self):
+        self._test_compressor('pipeline.compressors.slimit.SlimItCompressor',
+            'js', 'pipeline/compressors/slimit.js')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    def test_uglifyjs(self):
+        self._test_compressor('pipeline.compressors.uglifyjs.UglifyJSCompressor',
+            'js', 'pipeline/compressors/uglifyjs.js')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    def test_yuglify_js(self):
+        self._test_compressor('pipeline.compressors.yuglify.YuglifyCompressor',
+            'js', 'pipeline/compressors/yuglify.js')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    def test_yuglify_css(self):
+        self._test_compressor('pipeline.compressors.yuglify.YuglifyCompressor',
+            'css', 'pipeline/compressors/yuglify.css')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    def test_cssmin(self):
+        self._test_compressor('pipeline.compressors.cssmin.CSSMinCompressor',
+            'css', 'pipeline/compressors/cssmin.css')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    @skipUnless(settings.HAS_JAVA, "requires java")
+    def test_closure(self):
+        self._test_compressor('pipeline.compressors.closure.ClosureCompressor',
+            'js', 'pipeline/compressors/closure.js')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    @skipUnless(settings.HAS_JAVA, "requires java")
+    def test_yui_js(self):
+        self._test_compressor('pipeline.compressors.yui.YUICompressor',
+            'js', 'pipeline/compressors/yui.js')
+
+    @skipUnless(settings.HAS_NODE, "requires node")
+    @skipUnless(settings.HAS_JAVA, "requires java")
+    def test_yui_css(self):
+        self._test_compressor('pipeline.compressors.yui.YUICompressor',
+            'css', 'pipeline/compressors/yui.css')
+
+    @skipUnless(settings.HAS_CSSTIDY, "requires csstidy")
+    def test_csstidy(self):
+        self._test_compressor('pipeline.compressors.csstidy.CSSTidyCompressor',
+            'css', 'pipeline/compressors/csstidy.css')
+
+

tests/tests/test_conf.py

 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import sys
+from unittest import skipIf, skipUnless
+
 from django.test import TestCase
 
 from pipeline.conf import PipelineSettings
@@ -23,10 +26,16 @@ class TestSettings(TestCase):
         s = PipelineSettings({"FOO_BINARY": "env actualprogram"})
         self.assertEqual(s.FOO_BINARY, ('env', 'actualprogram'))
 
+    @skipIf(sys.platform.startswith("win"), "requires posix platform")
     def test_expected_preservation(self):
         s = PipelineSettings({"FOO_BINARY": r"actual\ program"})
         self.assertEqual(s.FOO_BINARY, ('actual program',))
 
+    @skipUnless(sys.platform.startswith("win"), "requires windows")
+    def test_win_path_preservation(self):
+        s = PipelineSettings({"FOO_BINARY": "C:\\Test\\ActualProgram.exe argument"})
+        self.assertEqual(s.FOO_BINARY, ('C:\\Test\\ActualProgram.exe', 'argument'))
+
     def test_tuples_are_normal(self):
         s = PipelineSettings({"FOO_ARGUMENTS": ("explicit", "with", "args")})
         self.assertEqual(s.FOO_ARGUMENTS, ('explicit', 'with', 'args'))

tox.ini

 [tox]
 envlist =
-    {py27,pypy,py34}-django{18,19}
-    py35-django19
-    docs
+    {py27,pypy,py34}-django{18,19},py35-django19,docs
 
 [testenv]
 basepython =
@@ -16,10 +14,14 @@ deps =
   django18: Django>=1.8,<1.9
   django19: Django>=1.9,<1.10
   jinja2
+  jsmin==2.2.0
+  ply==3.4
+  slimit==0.8.1
 setenv =
     DJANGO_SETTINGS_MODULE = tests.settings
     PYTHONPATH = {toxinidir}
 commands =
+    {toxinidir}/tests/scripts/npm_install.py
     {envbindir}/django-admin.py test {posargs:tests}
 
 [testenv:docs]