Commit a0ea33ac by Timothée Peignier

what if we use thread ?

parent dd08558f
from __future__ import unicode_literals from __future__ import unicode_literals
import multiprocessing
import os import os
import subprocess import subprocess
from multiprocessing.pool import ThreadPool
from django.contrib.staticfiles import finders from django.contrib.staticfiles import finders
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
from django.utils.encoding import smart_str, smart_bytes from django.utils.encoding import smart_str, smart_bytes
...@@ -17,30 +20,36 @@ class Compiler(object): ...@@ -17,30 +20,36 @@ class Compiler(object):
def __init__(self, storage=default_storage, verbose=False): def __init__(self, storage=default_storage, verbose=False):
self.storage = storage self.storage = storage
self.verbose = verbose self.verbose = verbose
self.pool = ThreadPool(processes=multiprocessing.cpu_count())
@property @property
def compilers(self): def compilers(self):
return [to_class(compiler) for compiler in settings.PIPELINE_COMPILERS] return [to_class(compiler) for compiler in settings.PIPELINE_COMPILERS]
def compile(self, paths, force=False): def compile(self, paths, force=False):
def _compile(args):
try:
compiler = args[0]
compiler.compile_file(*args[1:])
except CompilerError:
if not self.storage.exists(output_path) or settings.DEBUG:
raise
files = []
for index, input_path in enumerate(paths): for index, input_path in enumerate(paths):
for compiler in self.compilers: for compiler in self.compilers:
compiler = compiler(verbose=self.verbose, storage=self.storage) compiler = compiler(verbose=self.verbose, storage=self.storage)
if compiler.match_file(input_path): if compiler.match_file(input_path):
output_path = self.output_path(input_path, compiler.output_extension) output_path = self.output_path(input_path, compiler.output_extension)
paths[index] = output_path paths[index] = output_path
try: infile = finders.find(input_path)
infile = finders.find(input_path) outfile = finders.find(output_path)
outfile = finders.find(output_path) if outfile is None:
if outfile is None: outfile = self.output_path(infile, compiler.output_extension)
outfile = self.output_path(infile, compiler.output_extension) outdated = True
outdated = True else:
else: outdated = self.is_outdated(input_path, output_path)
outdated = self.is_outdated(input_path, output_path) files.append((compiler, infile, outfile, outdated, force))
compiler.compile_file(infile, outfile, outdated=outdated, force=force) self.pool.map(_compile, files)
except CompilerError:
if not self.storage.exists(output_path) or settings.DEBUG:
raise
return paths return paths
def output_path(self, path, extension): def output_path(self, path, extension):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment