Merge pull request #11110 from edx/ormsbee/libsass

Speed up Sass compilation with libsass

Gemfile

 source 'https://rubygems.org'
-gem 'sass', '3.3.5'
 gem 'bourbon', '~> 4.0.2'
 gem 'neat', '~> 1.6.0'

Gemfile.lock

@@ -7,7 +7,7 @@ GEM
     neat (1.6.0)
       bourbon (>= 3.1)
       sass (>= 3.3)
-    sass (3.3.5)
+    sass (3.4.21)
     thor (0.19.1)
 
 PLATFORMS
@@ -16,4 +16,3 @@ PLATFORMS
 DEPENDENCIES
   bourbon (~> 4.0.2)
   neat (~> 1.6.0)
-  sass (= 3.3.5)

circle.yml

@@ -29,6 +29,7 @@ dependencies:
     # Install a version which falls within that range.
     - pip install  --exists-action w pbr==0.9.0
     - pip install --exists-action w -r requirements/edx/base.txt
+    - pip install --exists-action w -r requirements/edx/paver.txt
     - if [ -e requirements/edx/post.txt ]; then pip install --exists-action w -r requirements/edx/post.txt ; fi
 
     - pip install coveralls==1.0

common/static/sass/bourbon/addons/_directional-values.scss

@@ -49,7 +49,7 @@
   @return false;
 }
 
-@mixin directional-property($pre, $suf, $vals) {
+@mixin directional-property($pre, $suf, $vals...) {
   // Property Names
   $top:    $pre + "-top"    + if($suf, "-#{$suf}", "");
   $bottom: $pre + "-bottom" + if($suf, "-#{$suf}", "");

pavelib/acceptance_test.py

@@ -30,7 +30,7 @@ __test__ = False  # do not collect
 ])
 def test_acceptance(options):
     """
-    Run the acceptance tests for the either lms or cms
+    Run the acceptance tests for either lms or cms
     """
     opts = {
         'fasttest': getattr(options, 'fasttest', False),

pavelib/assets.py

@@ -3,7 +3,7 @@ Asset compilation and collection.
 """
 
 from __future__ import print_function
-
+from datetime import datetime
 import argparse
 import glob
 import traceback
@@ -18,17 +18,22 @@ from .utils.cmd import cmd, django_cmd
 
 # setup baseline paths
 
+ALL_SYSTEMS = ['lms', 'studio']
 COFFEE_DIRS = ['lms', 'cms', 'common']
 # A list of directories.  Each will be paired with a sibling /css directory.
-SASS_DIRS = [
+COMMON_SASS_DIRECTORIES = [
+    path("common/static/sass"),
+]
+LMS_SASS_DIRECTORIES = [
     path("lms/static/sass"),
     path("lms/static/themed_sass"),
-    path("cms/static/sass"),
-    path("common/static/sass"),
     path("lms/static/certificates/sass"),
 ]
+CMS_SASS_DIRECTORIES = [
+    path("cms/static/sass"),
+]
+THEME_SASS_DIRECTORIES = []
 SASS_LOAD_PATHS = ['common/static', 'common/static/sass']
-SASS_CACHE_PATH = '/tmp/sass-cache'
 
 
 def configure_paths():
@@ -43,7 +48,7 @@ def configure_paths():
         css_dir = theme_root / "static" / "css"
         if sass_dir.isdir():
             css_dir.mkdir_p()
-            SASS_DIRS.append(sass_dir)
+            THEME_SASS_DIRECTORIES.append(sass_dir)
 
     if edxapp_env.env_tokens.get("COMPREHENSIVE_THEME_DIR", ""):
         theme_dir = path(edxapp_env.env_tokens["COMPREHENSIVE_THEME_DIR"])
@@ -51,16 +56,39 @@ def configure_paths():
         lms_css = theme_dir / "lms" / "static" / "css"
         if lms_sass.isdir():
             lms_css.mkdir_p()
-            SASS_DIRS.append(lms_sass)
+            THEME_SASS_DIRECTORIES.append(lms_sass)
         cms_sass = theme_dir / "cms" / "static" / "sass"
         cms_css = theme_dir / "cms" / "static" / "css"
         if cms_sass.isdir():
             cms_css.mkdir_p()
-            SASS_DIRS.append(cms_sass)
+            THEME_SASS_DIRECTORIES.append(cms_sass)
 
 configure_paths()
 
 
+def applicable_sass_directories(systems=None):
+    """
+    Determine the applicable set of SASS directories to be
+    compiled for the specified list of systems.
+
+    Args:
+        systems: A list of systems (defaults to all)
+
+    Returns:
+        A list of SASS directories to be compiled.
+    """
+    if not systems:
+        systems = ALL_SYSTEMS
+    applicable_directories = []
+    applicable_directories.extend(COMMON_SASS_DIRECTORIES)
+    if "lms" in systems:
+        applicable_directories.extend(LMS_SASS_DIRECTORIES)
+    if "studio" in systems or "cms" in systems:
+        applicable_directories.extend(CMS_SASS_DIRECTORIES)
+    applicable_directories.extend(THEME_SASS_DIRECTORIES)
+    return applicable_directories
+
+
 class CoffeeScriptWatcher(PatternMatchingEventHandler):
     """
     Watches for coffeescript changes
@@ -98,7 +126,7 @@ class SassWatcher(PatternMatchingEventHandler):
         """
         register files with observer
         """
-        for dirname in SASS_LOAD_PATHS + SASS_DIRS:
+        for dirname in SASS_LOAD_PATHS + applicable_sass_directories():
             paths = []
             if '*' in dirname:
                 paths.extend(glob.glob(dirname))
@@ -184,6 +212,7 @@ def compile_coffeescript(*files):
 @task
 @no_help
 @cmdopts([
+    ('system=', 's', 'The system to compile sass for (defaults to all)'),
     ('debug', 'd', 'Debug mode'),
     ('force', '', 'Force full compilation'),
 ])
@@ -191,31 +220,59 @@ def compile_sass(options):
     """
     Compile Sass to CSS.
     """
+
+    # Note: import sass only when it is needed and not at the top of the file.
+    # This allows other paver commands to operate even without libsass being
+    # installed. In particular, this allows the install_prereqs command to be
+    # used to install the dependency.
+    import sass
+
     debug = options.get('debug')
-    parts = ["sass"]
-    parts.append("--update")
-    parts.append("--cache-location {cache}".format(cache=SASS_CACHE_PATH))
-    parts.append("--default-encoding utf-8")
+    force = options.get('force')
+    systems = getattr(options, 'system', ALL_SYSTEMS)
+    if isinstance(systems, basestring):
+        systems = systems.split(',')
     if debug:
-        parts.append("--sourcemap")
+        source_comments = True
+        output_style = 'nested'
     else:
-        parts.append("--style compressed --quiet")
-    if options.get('force'):
-        parts.append("--force")
-    parts.append("--load-path .")
-    for load_path in SASS_LOAD_PATHS + SASS_DIRS:
-        parts.append("--load-path {path}".format(path=load_path))
-
-    for sass_dir in SASS_DIRS:
+        source_comments = False
+        output_style = 'compressed'
+
+    timing_info = []
+    system_sass_directories = applicable_sass_directories(systems)
+    all_sass_directories = applicable_sass_directories()
+    dry_run = tasks.environment.dry_run
+    for sass_dir in system_sass_directories:
+        start = datetime.now()
         css_dir = sass_dir.parent / "css"
-        if css_dir:
-            parts.append("{sass}:{css}".format(sass=sass_dir, css=css_dir))
-        else:
-            parts.append(sass_dir)
 
-    sh(cmd(*parts))
+        if force:
+            if dry_run:
+                tasks.environment.info("rm -rf {css_dir}/*.css".format(
+                    css_dir=css_dir,
+                ))
+            else:
+                sh("rm -rf {css_dir}/*.css".format(css_dir=css_dir))
 
-    print("\t\tFinished compiling sass.")
+        if dry_run:
+            tasks.environment.info("libsass {sass_dir}".format(
+                sass_dir=sass_dir,
+            ))
+        else:
+            sass.compile(
+                dirname=(sass_dir, css_dir),
+                include_paths=SASS_LOAD_PATHS + all_sass_directories,
+                source_comments=source_comments,
+                output_style=output_style,
+            )
+            duration = datetime.now() - start
+            timing_info.append((sass_dir, css_dir, duration))
+
+    print("\t\tFinished compiling Sass:")
+    if not dry_run:
+        for sass_dir, css_dir, duration in timing_info:
+            print(">> {} -> {} in {}s".format(sass_dir, css_dir, duration))
 
 
 def compile_templated_sass(systems, settings):
@@ -224,15 +281,15 @@ def compile_templated_sass(systems, settings):
     `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
     `settings` is the Django settings module to use.
     """
-    for sys in systems:
-        if sys == "studio":
-            sys = "cms"
+    for system in systems:
+        if system == "studio":
+            system = "cms"
         sh(django_cmd(
-            sys, settings, 'preprocess_assets',
-            '{sys}/static/sass/*.scss'.format(sys=sys),
-            '{sys}/static/themed_sass'.format(sys=sys)
+            system, settings, 'preprocess_assets',
+            '{system}/static/sass/*.scss'.format(system=system),
+            '{system}/static/themed_sass'.format(system=system)
         ))
-        print("\t\tFinished preprocessing {} assets.".format(sys))
+        print("\t\tFinished preprocessing {} assets.".format(system))
 
 
 def process_xmodule_assets():
@@ -308,7 +365,7 @@ def update_assets(args):
     """
     parser = argparse.ArgumentParser(prog='paver update_assets')
     parser.add_argument(
-        'system', type=str, nargs='*', default=['lms', 'studio'],
+        'system', type=str, nargs='*', default=ALL_SYSTEMS,
         help="lms or studio",
     )
     parser.add_argument(
@@ -332,7 +389,7 @@ def update_assets(args):
     compile_templated_sass(args.system, args.settings)
     process_xmodule_assets()
     compile_coffeescript()
-    call_task('pavelib.assets.compile_sass', options={'debug': args.debug})
+    call_task('pavelib.assets.compile_sass', options={'system': args.system, 'debug': args.debug})
 
     if args.collect:
         collect_assets(args.system, args.settings)

pavelib/paver_tests/test_assets.py

+"""Unit tests for the Paver asset tasks."""
+
+import ddt
+from paver.easy import call_task
+
+from .utils import PaverTestCase
+
+
+@ddt.ddt
+class TestPaverAssetTasks(PaverTestCase):
+    """
+    Test the Paver asset tasks.
+    """
+    @ddt.data(
+        [""],
+        ["--force"],
+        ["--debug"],
+        ["--system=lms"],
+        ["--system=lms --force"],
+        ["--system=studio"],
+        ["--system=studio --force"],
+        ["--system=lms,studio"],
+        ["--system=lms,studio --force"],
+    )
+    @ddt.unpack
+    def test_compile_sass(self, options):
+        """
+        Test the "compile_sass" task.
+        """
+        parameters = options.split(" ")
+        system = []
+        if "--system=studio" not in parameters:
+            system += ["lms"]
+        if "--system=lms" not in parameters:
+            system += ["studio"]
+        debug = "--debug" in parameters
+        force = "--force" in parameters
+        self.reset_task_messages()
+        call_task('pavelib.assets.compile_sass', options={"system": system, "debug": debug, "force": force})
+        expected_messages = []
+        if force:
+            expected_messages.append("rm -rf common/static/css/*.css")
+        expected_messages.append("libsass common/static/sass")
+        if "lms" in system:
+            if force:
+                expected_messages.append("rm -rf lms/static/css/*.css")
+            expected_messages.append("libsass lms/static/sass")
+            if force:
+                expected_messages.append("rm -rf lms/static/css/*.css")
+            expected_messages.append("libsass lms/static/themed_sass")
+            if force:
+                expected_messages.append("rm -rf lms/static/certificates/css/*.css")
+            expected_messages.append("libsass lms/static/certificates/sass")
+        if "studio" in system:
+            if force:
+                expected_messages.append("rm -rf cms/static/css/*.css")
+            expected_messages.append("libsass cms/static/sass")
+        self.assertEquals(self.task_messages, expected_messages)

pavelib/paver_tests/test_servers.py

@@ -11,21 +11,19 @@ EXPECTED_COFFEE_COMMAND = (
     "{platform_root}/cms {platform_root}/common -type f -name \"*.coffee\"`"
 )
 EXPECTED_SASS_COMMAND = (
-    "sass --update --cache-location /tmp/sass-cache --default-encoding utf-8 --style compressed"
-    " --quiet"
-    " --load-path ."
-    " --load-path common/static"
-    " --load-path common/static/sass"
-    " --load-path lms/static/sass"
-    " --load-path lms/static/themed_sass"
-    " --load-path cms/static/sass --load-path common/static/sass"
-    " --load-path lms/static/certificates/sass"
-    " lms/static/sass:lms/static/css"
-    " lms/static/themed_sass:lms/static/css"
-    " cms/static/sass:cms/static/css"
-    " common/static/sass:common/static/css"
-    " lms/static/certificates/sass:lms/static/certificates/css"
+    "libsass {sass_directory}"
 )
+EXPECTED_COMMON_SASS_DIRECTORIES = [
+    "common/static/sass",
+]
+EXPECTED_LMS_SASS_DIRECTORIES = [
+    "lms/static/sass",
+    "lms/static/themed_sass",
+    "lms/static/certificates/sass",
+]
+EXPECTED_CMS_SASS_DIRECTORIES = [
+    "cms/static/sass",
+]
 EXPECTED_PREPROCESS_ASSETS_COMMAND = (
     "python manage.py {system} --settings={asset_settings} preprocess_assets"
     " {system}/static/sass/*.scss {system}/static/themed_sass"
@@ -236,7 +234,7 @@ class TestPaverServerTasks(PaverTestCase):
             ))
             expected_messages.append("xmodule_assets common/static/xmodule")
             expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
-            expected_messages.append(EXPECTED_SASS_COMMAND)
+            expected_messages.extend(self.expected_sass_commands(system=system))
         if expected_collect_static:
             expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
                 system=system, asset_settings=expected_asset_settings
@@ -278,7 +276,7 @@ class TestPaverServerTasks(PaverTestCase):
             ))
             expected_messages.append("xmodule_assets common/static/xmodule")
             expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
-            expected_messages.append(EXPECTED_SASS_COMMAND)
+            expected_messages.extend(self.expected_sass_commands())
         if expected_collect_static:
             expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
                 system="lms", asset_settings=expected_asset_settings
@@ -302,3 +300,15 @@ class TestPaverServerTasks(PaverTestCase):
         )
         expected_messages.append(EXPECTED_CELERY_COMMAND.format(settings="dev_with_worker"))
         self.assertEquals(self.task_messages, expected_messages)
+
+    def expected_sass_commands(self, system=None):
+        """
+        Returns the expected SASS commands for the specified system.
+        """
+        expected_sass_directories = []
+        expected_sass_directories.extend(EXPECTED_COMMON_SASS_DIRECTORIES)
+        if system != 'cms':
+            expected_sass_directories.extend(EXPECTED_LMS_SASS_DIRECTORIES)
+        if system != 'lms':
+            expected_sass_directories.extend(EXPECTED_CMS_SASS_DIRECTORIES)
+        return [EXPECTED_SASS_COMMAND.format(sass_directory=directory) for directory in expected_sass_directories]

pavelib/prereqs.py

@@ -24,6 +24,7 @@ PYTHON_REQ_FILES = [
     'requirements/edx/github.txt',
     'requirements/edx/local.txt',
     'requirements/edx/base.txt',
+    'requirements/edx/paver.txt',
     'requirements/edx/post.txt',
 ]
 

requirements/edx/paver.txt

@@ -5,3 +5,4 @@ lazy==1.1
 path.py==7.2
 watchdog==0.7.1
 python-memcached
+libsass==0.10.0