Preprocess assets as Django management command

Rather than directly invoke command-line Python (and Mako) from the
assets Rakefile, or call an external Python script, use a Django
management command to preprocess all asset template files.

An "asset template file" is defined as a static asset file with a
file extension indicating that it needs to be run through a template
engine prior to Sass/CoffeeScript compilation or packaging with other
assets. The preprocess_assets management command will look through
all of the files listed in the `STATICFILES_DIRS`, preprocessing each
as needed. Preprocessing strips off the special template file
extension, creating a new file in the process.

Currently, the only variable accessible in an asset template file is
the `THEME_NAME`, defined in the settings.

cms/envs/common.py

@@ -323,6 +323,7 @@ INSTALLED_APPS = (
     'track',
 
     # For asset pipelining
+    'mitxmako',
     'pipeline',
     'staticfiles',
     'static_replace',

common/djangoapps/mitxmako/management/commands/preprocess_assets.py

+"""
+Preprocess templatized asset files, enabling asset authors to use
+Python/Django inside of Sass and CoffeeScript. This preprocessing
+will happen before the invocation of the asset compiler (currently
+handled by the asset Rakefile).
+
+For this to work, assets need to be named with the appropriate
+template extension (e.g., .mako for Mako templates). Currently Mako
+is the only template engine supported.
+"""
+import os
+
+from django.core.management.base import NoArgsCommand
+from django.conf import settings
+
+from mako.template import Template
+
+class Command(NoArgsCommand):
+    """
+    Basic management command to preprocess asset template files.
+    """
+
+    help = "Preprocess asset template files to ready them for compilation."
+
+    def handle_noargs(self, **options):
+        """
+        Walk over all of the static files directories specified in the
+        settings file, looking for asset template files (indicated by
+        a file extension like .mako).
+        """
+        for staticfiles_dir in getattr(settings, "STATICFILES_DIRS", []):
+            # Cribbed from the django-staticfiles app at:
+            # https://github.com/jezdez/django-staticfiles/blob/develop/staticfiles/finders.py#L52
+            if isinstance(staticfiles_dir, (list, tuple)):
+                prefix, staticfiles_dir = staticfiles_dir
+
+            # Walk over the current static files directory tree,
+            # preprocessing files that have a template extension.
+            for root, dirs, files in os.walk(staticfiles_dir):
+                for filename in files:
+                    outfile, extension = os.path.splitext(filename)
+                    # We currently only handle Mako templates
+                    if extension == ".mako":
+                        self.__preprocess(os.path.join(root, filename),
+                                          os.path.join(root, outfile))
+
+
+    def __context(self):
+        """
+        Return a dict that contains all of the available context
+        variables to the asset template.
+        """
+        # TODO: do we need to include anything else?
+        # TODO: do this with the django-settings-context-processor
+        return { "THEME_NAME" : getattr(settings, "THEME_NAME", None) }
+
+
+    def __preprocess(self, infile, outfile):
+        """
+        Run `infile` through the Mako template engine, storing the
+        result in `outfile`.
+        """
+        with open(outfile, "w") as _outfile:
+            _outfile.write(Template(filename=str(infile)).render(env=self.__context()))
+

lms/envs/common.py

@@ -662,6 +662,7 @@ INSTALLED_APPS = (
     'service_status',
 
     # For asset pipelining
+    'mitxmako',
     'pipeline',
     'staticfiles',
     'static_replace',

rakefiles/assets.rake

@@ -6,30 +6,6 @@ if USE_CUSTOM_THEME
     THEME_SASS = File.join(THEME_ROOT, "static", "sass")
 end
 
-# Run the specified file through the Mako templating engine, providing
-# the ENV_TOKENS to the templating context.
-def preprocess_with_mako(filename)
-    # simple command-line invocation of Mako engine
-    # cdodge: the .gsub() are used to translate true->True and false->False to make the generated
-    # python actually valid python. This is just a short term hack to unblock the release train
-    # until a real fix can be made by people who know this better
-    mako = "from mako.template import Template;" +
-           "print Template(filename=\"#{filename}\")" +
-           # Total hack. It works because a Python dict literal has
-           # the same format as a JSON object.
-           ".render(env=#{ENV_TOKENS.to_json.gsub("true","True").gsub("false","False")});"
-
-    # strip off the .mako extension
-    output_filename = filename.chomp(File.extname(filename))
-
-    # just pipe from stdout into the new file, exiting on failure
-    File.open(output_filename, 'w') do |file|
-      file.write(`python -c '#{mako}'`)
-      exit_code = $?.to_i
-      abort "#{mako} failed with #{exit_code}" if exit_code.to_i != 0
-    end
-end
-
 def xmodule_cmd(watch=false, debug=false)
     xmodule_cmd = 'xmodule_assets common/static/xmodule'
     if watch
@@ -84,11 +60,12 @@ namespace :assets do
     desc "Compile all assets in debug mode"
     multitask :debug
 
-    desc "Preprocess all static assets that have the .mako extension"
-    task :preprocess do
-      # Run assets through the Mako templating engine. Right now we
-      # just hardcode the asset filenames.
-      preprocess_with_mako("lms/static/sass/application.scss.mako")
+    desc "Preprocess all templatized static asset files"
+    task :preprocess, [:system, :env] do |t, args|
+      args.with_defaults(:system => "lms", :env => "dev")
+      sh(django_admin(args.system, args.env, "preprocess_assets")) do |ok, status|
+        abort "asset preprocessing failed!" if !ok
+      end
     end
 
     desc "Watch all assets for changes and automatically recompile"
@@ -138,7 +115,6 @@ namespace :assets do
         end
     end
 
-
     multitask :sass => 'assets:xmodule'
     namespace :sass do
         # In watch mode, sass doesn't immediately compile out of date files,
@@ -153,16 +129,25 @@ namespace :assets do
     end
 end
 
+# This task does the real heavy lifting to gather all of the static
+# assets. We want people to call it via the wrapper below, so we
+# don't provide a description so that it won't show up in rake -T.
+task :gather_assets, [:system, :env] => :assets do |t, args|
+    sh("#{django_admin(args.system, args.env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
+        if !ok
+            abort "collectstatic failed!"
+        end
+    end
+end
+
 [:lms, :cms].each do |system|
     # Per environment tasks
     environments(system).each do |env|
+        # This task wraps the one above, since we need the system and
+        # env arguments to be passed to all dependent tasks.
         desc "Compile coffeescript and sass, and then run collectstatic in the specified environment"
-        task "#{system}:gather_assets:#{env}" => :assets do
-            sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
-                if !ok
-                    abort "collectstatic failed!"
-                end
-            end
+        task "#{system}:gather_assets:#{env}" do
+          task(:gather_assets).invoke(system, env)
         end
     end
 end