ensure we access file using storage API

pipeline/compilers/__init__.py

 import os
 import subprocess
 
-from django.core.files.base import ContentFile
-
 from pipeline.conf import settings
 from pipeline.storage import storage
 from pipeline.utils import to_class
@@ -43,7 +41,9 @@ class Compiler(object):
         return content
 
     def save_file(self, path, content):
-        return storage.save(path, ContentFile(content))
+        file = storage.open(path, 'wb')
+        file.write(content)
+        file.close()
 
 
 class CompilerBase(object):

pipeline/compressors/__init__.py

@@ -196,7 +196,7 @@ class Compressor(object):
 
     def read_file(self, path):
         """Read file content in binary mode"""
-        file = storage.open(path, mode='rb')
+        file = storage.open(path, 'rb')
         content = file.read()
         file.close()
         return content

pipeline/glob.py

+import os
+import re
+import fnmatch
+
+from pipeline.storage import storage
+
+__all__ = ["glob", "iglob"]
+
+
+def glob(pathname):
+    """Return a list of paths matching a pathname pattern.
+
+    The pattern may contain simple shell-style wildcards a la fnmatch.
+
+    """
+    return list(iglob(pathname))
+
+
+def iglob(pathname):
+    """Return an iterator which yields the paths matching a pathname pattern.
+
+    The pattern may contain simple shell-style wildcards a la fnmatch.
+
+    """
+    if not has_magic(pathname):
+        if storage.exists(pathname):
+            yield pathname
+        return
+    dirname, basename = os.path.split(pathname)
+    if not dirname:
+        for name in glob1(dirname, basename):
+            yield name
+        return
+    if has_magic(dirname):
+        dirs = iglob(dirname)
+    else:
+        dirs = [dirname]
+    if has_magic(basename):
+        glob_in_dir = glob1
+    else:
+        glob_in_dir = glob0
+    for dirname in dirs:
+        for name in glob_in_dir(dirname, basename):
+            yield os.path.join(dirname, name)
+
+# These 2 helper functions non-recursively glob inside a literal directory.
+# They return a list of basenames. `glob1` accepts a pattern while `glob0`
+# takes a literal basename (so it only has to check for its existence).
+
+
+def glob1(dirname, pattern):
+    try:
+        directories, files = storage.listdir(dirname)
+        names = directories + files
+    except NotImplementedError:
+        return []
+    if pattern[0] != '.':
+        names = filter(lambda x: x[0] != '.', names)
+    return fnmatch.filter(names, pattern)
+
+
+def glob0(dirname, basename):
+    if storage.exists(os.path.join(dirname, basename)):
+        return [basename]
+    return []
+
+
+magic_check = re.compile('[*?[]')
+
+
+def has_magic(s):
+    return magic_check.search(s) is not None

pipeline/packager.py

-import glob
 import os
 import urlparse
 
-from django.core.files.base import ContentFile
-
 from pipeline.conf import settings
 from pipeline.compilers import Compiler
 from pipeline.compressors import Compressor
+from pipeline.glob import glob
 from pipeline.signals import css_compressed, js_compressed
 from pipeline.storage import storage
 from pipeline.versioning import Versioning
@@ -73,8 +71,11 @@ class Packager(object):
     def pack_templates(self, package):
         return self.compressor.compile_templates(package['templates'])
 
-    def save_file(self, filename, content):
-        return storage.save(filename, ContentFile(content))
+    def save_file(self, path, content):
+        file = storage.open(path, 'wb')
+        file.write(content)
+        file.close()
+        return path
 
     def create_packages(self, config):
         packages = {}
@@ -86,10 +87,8 @@ class Packager(object):
                 packages[name]['externals'] = config[name]['external_urls']
                 continue
             paths = []
-            for path in config[name]['source_filenames']:
-                full_path = os.path.join(settings.PIPELINE_ROOT, path)
-                for path in glob.glob(full_path):
-                    path = os.path.relpath(path, settings.PIPELINE_ROOT)
+            for pattern in config[name]['source_filenames']:
+                for path in glob(pattern):
                     if not path in paths:
                         paths.append(path)
             packages[name]['paths'] = [path for path in paths if not path.endswith(settings.PIPELINE_TEMPLATE_EXT)]

pipeline/storage.py

@@ -16,11 +16,6 @@ class PipelineStorage(FileSystemStorage):
             base_url = settings.PIPELINE_URL
         super(PipelineStorage, self).__init__(location, base_url, *args, **kwargs)
 
-    def get_available_name(self, name):
-        if self.exists(name):
-            self.delete(name)
-        return name
-
     def accessed_time(self, name):
         return datetime.fromtimestamp(os.path.getatime(self.path(name)))