Add RequireJS and RequireJS Optimizer configuration to the LMS

Respond to review comments:

- Rename build and config files for consistency between lms/studio.
- Fix merge conflicts with lms require config.
- Devstack uses optimized pipeline to skip require JS optimizer step
- Add tests for render_require_js_path_overrides

cms/envs/common.py

@@ -375,7 +375,7 @@ MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 EMBARGO_SITE_REDIRECT_URL = None
 
 ############################### Pipeline #######################################
-STATICFILES_STORAGE = 'cms.lib.django_require.staticstorage.OptimizedCachedRequireJsStorage'
+STATICFILES_STORAGE = 'django_require.staticstorage.OptimizedCachedRequireJsStorage'
 
 from rooted_paths import rooted_glob
 
@@ -515,7 +515,7 @@ REQUIRE_BASE_URL = "./"
 # A sensible value would be 'app.build.js'. Leave blank to use the built-in default build profile.
 # Set to False to disable running the default profile (e.g. if only using it to build Standalone
 # Modules)
-REQUIRE_BUILD_PROFILE = "build.js"
+REQUIRE_BUILD_PROFILE = "build-studio.js"
 
 # The name of the require.js script used by your project, relative to REQUIRE_BASE_URL.
 REQUIRE_JS = "js/vendor/require.js"

cms/envs/devstack.py

@@ -30,6 +30,11 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 LMS_BASE = "localhost:8000"
 FEATURES['PREVIEW_LMS_BASE'] = "preview." + LMS_BASE
 
+########################### PIPELINE #################################
+
+# Skip RequireJS optimizer in development
+STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
+
 ############################# ADVANCED COMPONENTS #############################
 
 # Make it easier to test advanced components in local dev

cms/static/build.js → cms/static/build-studio.js

@@ -61,7 +61,7 @@
          * As of 2.1.10, mainConfigFile can be an array of values, with the last
          * value's config take precedence over previous values in the array.
          */
-        mainConfigFile: 'require-config.js',
+        mainConfigFile: 'require-config-studio.js',
         /**
          * Set paths for modules. If relative paths, set relative to baseUrl above.
          * If a special value of "empty:" is used for the path value, then that

cms/templates/base.html

@@ -56,7 +56,7 @@ import json
       var require = {baseUrl: window.baseUrl};
     </script>
     <script type="text/javascript" src="${static.url("js/vendor/require.js")}"></script>
-    <script type="text/javascript" src="${static.url("require-config.js")}"></script>
+    <script type="text/javascript" src="${static.url("require-config-studio.js")}"></script>
 
     ## js templates
     <script id="system-feedback-tpl" type="text/template">

common/djangoapps/pipeline_mako/__init__.py

 from edxmako.shortcuts import render_to_string
+from django.conf import settings as django_settings
+from staticfiles.storage import staticfiles_storage
 
 from pipeline.conf import settings
 from pipeline.packager import Packager
@@ -79,3 +81,78 @@ def render_individual_js(package, paths, templates=None):
     if templates:
         tags.append(render_inline_js(package, templates))
     return '\n'.join(tags)
+
+
+def render_require_js_path_overrides(path_overrides):  # pylint: disable=invalid-name
+    """Render JavaScript to override default RequireJS paths.
+
+    The Django pipeline appends a hash to JavaScript files,
+    so if the JS asset isn't included in the bundle for the page,
+    we need to tell RequireJS where to look.
+
+    For example:
+
+        "js/vendor/jquery.min.js" --> "js/vendor/jquery.min.abcd1234"
+
+    We would then add a line in a <script> tag:
+
+        require.paths['jquery'] = 'js/vendor/jquery.min.abcd1234'
+
+    so that any reference to 'jquery' in a JavaScript module
+    will cause RequireJS to load '/static/js/vendor/jquery.min.abcd1234.js'
+
+    If running in DEBUG mode (as in devstack), the resolved JavaScript URLs
+    won't contain hashes, so the new paths will match the original paths.
+
+    Arguments:
+        path_overrides (dict): Mapping of RequireJS module names to
+            filesystem paths.
+
+    Returns:
+        unicode: The HTML of the <script> tag with the path overrides.
+
+    """
+    # Render the <script> tag that overrides the paths defined in `require.paths`
+    # Note: We don't use a Mako template to render this because Mako apparently
+    # acquires a lock when loading templates, which can lead to a deadlock if
+    # this function is called from within another template.
+    html = ['<script type="text/javascript">']
+
+    # The rendered <script> tag with overrides should be included *after*
+    # the application's RequireJS config, which defines a `require` object.
+    # Just in case the `require` object hasn't been loaded, we create a default
+    # object.  This will avoid a JavaScript error that might cause the rest of the
+    # page to fail; however, it may mean that these overrides won't be available
+    # to RequireJS.
+    html.extend([
+        'var require = require || {};',
+        'require.paths = require.paths || [];'
+    ])
+
+    # Specify override the base URL to point to STATIC_URL
+    html.append(
+        "require.baseUrl = '{url}'".format(
+            url=django_settings.STATIC_URL
+        )
+    )
+
+    for module, url_path in path_overrides.iteritems():
+        # Calculate the full URL, including any hashes added to the filename by the pipeline.
+        # This will also include the base static URL (for example, "/static/") and the
+        # ".js" extension.
+        actual_url = staticfiles_storage.url(url_path)
+
+        # RequireJS assumes that every file it tries to load has a ".js" extension, so
+        # we need to remove ".js" from the module path.
+        # RequireJS also already has a base URL set to the base static URL, so we can remove that.
+        path = actual_url.replace('.js', '').replace(django_settings.STATIC_URL, '')
+
+        # Add the path override to the inline JavaScript.
+        html.append(
+            "require.paths['{module}'] = '{path}';".format(
+                module=module,
+                path=path
+            )
+        )
+    html.append('</script>')
+    return "\n".join(html)

common/djangoapps/pipeline_mako/templates/static_content.html

@@ -2,6 +2,7 @@
 from staticfiles.storage import staticfiles_storage
 from pipeline_mako import compressed_css, compressed_js
 from django.utils.translation import get_language_bidi
+from require.templatetags.require import require_module
 %>
 
 <%def name='url(file, raw=False)'><%
@@ -37,6 +38,10 @@ except:
   %endif
 </%def>
 
+<%def name='require_module(module)'>
+  ${require_module(module)}
+</%def>
+
 <%def name="include(path)"><%
 from django.template.loaders.filesystem import _loader
 source, template_path = _loader.load_template_source(path)

common/djangoapps/pipeline_mako/tests/test_render.py

+"""Tests for rendering functions in the mako pipeline. """
+
+from django.test import TestCase
+from pipeline_mako import render_require_js_path_overrides
+
+
+class RequireJSPathOverridesTest(TestCase):
+    """Test RequireJS path overrides. """
+
+    OVERRIDES = {
+        'jquery': 'js/vendor/jquery.min.js',
+        'backbone': 'js/vendor/backbone-min.js',
+        'text': 'js/vendor/text.js'
+    }
+
+    OVERRIDES_JS = (
+        "<script type=\"text/javascript\">\n"
+        "var require = require || {};\n"
+        "require.paths = require.paths || [];\n"
+        "require.baseUrl = '/static/'\n"
+        "require.paths['jquery'] = 'js/vendor/jquery.min';\n"
+        "require.paths['text'] = 'js/vendor/text';\n"
+        "require.paths['backbone'] = 'js/vendor/backbone-min';\n"
+        "</script>"
+    )
+
+    def test_requirejs_path_overrides(self):
+        result = render_require_js_path_overrides(self.OVERRIDES)
+        self.assertEqual(result, self.OVERRIDES_JS)

lms/envs/common.py

@@ -1024,7 +1024,7 @@ X_FRAME_OPTIONS = 'ALLOW'
 
 ############################### Pipeline #######################################
 
-STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
+STATICFILES_STORAGE = 'django_require.staticstorage.OptimizedCachedRequireJsStorage'
 
 from rooted_paths import rooted_glob
 
@@ -1360,6 +1360,56 @@ PIPELINE_UGLIFYJS_BINARY = 'node_modules/.bin/uglifyjs'
 # Setting that will only affect the edX version of django-pipeline until our changes are merged upstream
 PIPELINE_COMPILE_INPLACE = True
 
+
+################################# DJANGO-REQUIRE ###############################
+
+# The baseUrl to pass to the r.js optimizer, relative to STATIC_ROOT.
+REQUIRE_BASE_URL = "./"
+
+# The name of a build profile to use for your project, relative to REQUIRE_BASE_URL.
+# A sensible value would be 'app.build.js'. Leave blank to use the built-in default build profile.
+# Set to False to disable running the default profile (e.g. if only using it to build Standalone
+# Modules)
+REQUIRE_BUILD_PROFILE = "build-lms.js"
+
+# The name of the require.js script used by your project, relative to REQUIRE_BASE_URL.
+REQUIRE_JS = "js/vendor/require.js"
+
+# A dictionary of standalone modules to build with almond.js.
+REQUIRE_STANDALONE_MODULES = {}
+
+# Whether to run django-require in debug mode.
+REQUIRE_DEBUG = False
+
+# A tuple of files to exclude from the compilation result of r.js.
+REQUIRE_EXCLUDE = ("build.txt",)
+
+# The execution environment in which to run r.js: auto, node or rhino.
+# auto will autodetect the environment and make use of node if available and rhino if not.
+# It can also be a path to a custom class that subclasses require.environments.Environment
+# and defines some "args" function that returns a list with the command arguments to execute.
+REQUIRE_ENVIRONMENT = "node"
+
+# In production, the Django pipeline appends a file hash to JavaScript file names.
+# This makes it difficult for RequireJS to load its requirements, since module names
+# specified in JavaScript code do not include the hash.
+# For this reason, we calculate the actual path including the hash on the server
+# when rendering the page.  We then override the default paths provided to RequireJS
+# so it can resolve the module name to the correct URL.
+#
+# If you want to load JavaScript dependencies using RequireJS
+# but you don't want to include those dependencies in the JS bundle for the page,
+# then you need to add the module and URL path to this dictionary.
+REQUIRE_JS_PATH_OVERRIDES = {
+    'jquery': 'js/vendor/jquery.min.js',
+    'jquery.cookie': 'js/vendor/jquery.cookie.js',
+    'underscore': 'js/vendor/underscore-min.js',
+    'underscore.string': 'js/vendor/underscore.string.min.js',
+    'backbone': 'js/vendor/backbone-min.js',
+    'text': 'js/vendor/text.js'
+}
+
+
 ################################# CELERY ######################################
 
 # Message configuration

lms/envs/devstack.py

@@ -76,6 +76,9 @@ DEBUG_TOOLBAR_CONFIG = {
 
 PIPELINE_SASS_ARGUMENTS = '--debug-info --require {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)
 
+# Skip RequireJS optimizer in development
+STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
+
 ########################### VERIFIED CERTIFICATES #################################
 
 FEATURES['AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'] = True
@@ -112,6 +115,10 @@ FEATURES['MILESTONES_APP'] = True
 ########################### Entrance Exams #################################
 FEATURES['ENTRANCE_EXAMS'] = True
 
+################################# DJANGO-REQUIRE ###############################
+
+# Whether to run django-require in debug mode.
+REQUIRE_DEBUG = DEBUG
 
 #####################################################################
 # See if the developer has any local overrides.

lms/static/build-lms.js

+(function () {
+    'use strict';
+
+    return {
+        /**
+         * List the modules that will be optimized. All their immediate and deep
+         * dependencies will be included in the module's file when the build is
+         * done.
+         */
+        modules: [],
+
+        /**
+         * By default all the configuration for optimization happens from the command
+         * line or by properties in the config file, and configuration that was
+         * passed to requirejs as part of the app's runtime "main" JS file is *not*
+         * considered. However, if you prefer the "main" JS file configuration
+         * to be read for the build so that you do not have to duplicate the values
+         * in a separate configuration, set this property to the location of that
+         * main JS file. The first requirejs({}), require({}), requirejs.config({}),
+         * or require.config({}) call found in that file will be used.
+         * As of 2.1.10, mainConfigFile can be an array of values, with the last
+         * value's config take precedence over previous values in the array.
+         */
+        mainConfigFile: 'require-config-lms.js',
+
+        /**
+         * Set paths for modules. If relative paths, set relative to baseUrl above.
+         * If a special value of "empty:" is used for the path value, then that
+         * acts like mapping the path to an empty file. It allows the optimizer to
+         * resolve the dependency to path, but then does not include it in the output.
+         * Useful to map module names that are to resources on a CDN or other
+         * http: URL when running in the browser and during an optimization that
+         * file should be skipped because it has no dependencies.
+         */
+        paths: {
+            'gettext': 'empty:'
+        },
+
+        /**
+         * If shim config is used in the app during runtime, duplicate the config
+         * here. Necessary if shim config is used, so that the shim's dependencies
+         * are included in the build. Using "mainConfigFile" is a better way to
+         * pass this information though, so that it is only listed in one place.
+         * However, if mainConfigFile is not an option, the shim config can be
+         * inlined in the build config.
+         */
+        shim: {},
+
+        /**
+         * Introduced in 2.1.2: If using "dir" for an output directory, normally the
+         * optimize setting is used to optimize the build bundles (the "modules"
+         * section of the config) and any other JS file in the directory. However, if
+         * the non-build bundle JS files will not be loaded after a build, you can
+         * skip the optimization of those files, to speed up builds. Set this value
+         * to true if you want to skip optimizing those other non-build bundle JS
+         * files.
+         */
+        skipDirOptimize: true,
+
+        /**
+         * When the optimizer copies files from the source location to the
+         * destination directory, it will skip directories and files that start
+         * with a ".". If you want to copy .directories or certain .files, for
+         * instance if you keep some packages in a .packages directory, or copy
+         * over .htaccess files, you can set this to null. If you want to change
+         * the exclusion rules, change it to a different regexp. If the regexp
+         * matches, it means the directory will be excluded. This used to be
+         * called dirExclusionRegExp before the 1.0.2 release.
+         * As of 1.0.3, this value can also be a string that is converted to a
+         * RegExp via new RegExp().
+         */
+        fileExclusionRegExp: /^\.|spec/,
+
+        /**
+         * Allow CSS optimizations. Allowed values:
+         * - "standard": @import inlining and removal of comments, unnecessary
+         * whitespace and line returns.
+         * Removing line returns may have problems in IE, depending on the type
+         * of CSS.
+         * - "standard.keepLines": like "standard" but keeps line returns.
+         * - "none": skip CSS optimizations.
+         * - "standard.keepComments": keeps the file comments, but removes line
+         * returns.  (r.js 1.0.8+)
+         * - "standard.keepComments.keepLines": keeps the file comments and line
+         * returns. (r.js 1.0.8+)
+         * - "standard.keepWhitespace": like "standard" but keeps unnecessary whitespace.
+         */
+        optimizeCss: 'none',
+
+        /**
+         * How to optimize all the JS files in the build output directory.
+         * Right now only the following values are supported:
+         * - "uglify": Uses UglifyJS to minify the code.
+         * - "uglify2": Uses UglifyJS2.
+         * - "closure": Uses Google's Closure Compiler in simple optimization
+         * mode to minify the code. Only available if REQUIRE_ENVIRONMENT is "rhino" (the default).
+         * - "none": No minification will be done.
+         */
+        optimize: 'uglify2',
+
+        /**
+         * Sets the logging level. It is a number:
+         * TRACE: 0,
+         * INFO: 1,
+         * WARN: 2,
+         * ERROR: 3,
+         * SILENT: 4
+         * Default is 0.
+         */
+        logLevel: 0
+    };
+} ())

lms/static/js_test.yml

@@ -58,6 +58,7 @@ lib_paths:
     - xmodule_js/common_static/js/vendor/edxnotes/annotator-full.min.js
     - xmodule_js/common_static/js/test/i18n.js
     - xmodule_js/common_static/js/vendor/date.js
+    - xmodule_js/common_static/js/vendor/text.js
 
 # Paths to source JavaScript files
 src_paths:

lms/static/require-config-lms.js

@@ -39,7 +39,6 @@
     } else {
         paths.tinymce = "js/vendor/tinymce/js/tinymce/jquery.tinymce.min";
     }
-
     config = {
         // NOTE: baseUrl has been previously set in lms/static/templates/main.html
         waitSeconds: 60,
@@ -47,7 +46,18 @@
             "annotator_1.2.9": "js/vendor/edxnotes/annotator-full.min",
             "date": "js/vendor/date",
             "backbone": "js/vendor/backbone-min",
+            "gettext": "/i18n",
+            "jquery": "js/vendor/jquery.min",
+            "jquery.cookie": "js/vendor/jquery.cookie",
+            "jquery.url": "js/vendor/url.min",
+            "text": "js/vendor/text",
+            "underscore": "js/vendor/underscore-min",
             "underscore.string": "js/vendor/underscore.string.min",
+
+            // This module defines some global functions.
+            // TODO: replace these with RequireJS-compatible modules
+            "utility": "js/src/utility",
+
             // Files needed by OVA
             "annotator": "js/vendor/ova/annotator-full",
             "annotator-harvardx": "js/vendor/ova/annotator-full-firebase-auth",
@@ -80,6 +90,14 @@
             "jquery": {
                 exports: "$"
             },
+            "jquery.cookie": {
+                deps: ["jquery"],
+                exports: "jQuery.fn.cookie"
+            },
+            "jquery.url": {
+                deps: ["jquery"],
+                exports: "jQuery.url"
+            },
             "underscore": {
                 exports: "_"
             },
@@ -87,6 +105,9 @@
                 deps: ["underscore", "jquery"],
                 exports: "Backbone"
             },
+            "gettext": {
+                exports: "gettext"
+            },
             "logger": {
                 exports: "Logger"
             },

lms/templates/main.html

@@ -14,9 +14,12 @@
     % if responsive:
     <meta name="viewport" content="width=device-width, initial-scale=1">
     % endif
-<%! from django.utils.translation import ugettext as _ %>
-<%! from microsite_configuration import microsite %>
-<%! from microsite_configuration import page_title_breadcrumbs %>
+<%!
+from django.utils.translation import ugettext as _
+from pipeline_mako import render_require_js_path_overrides
+from microsite_configuration import microsite
+from microsite_configuration import page_title_breadcrumbs
+%>
 
 <%namespace name='static' file='static_content.html'/>
 <%! from django.utils.http import urlquote_plus %>
@@ -58,7 +61,13 @@
     })(this);
   </script>
 
-  <script type="text/javascript" src="/jsi18n/"></script>
+  ## RequireJS-enabled pages should include the "i18n" module as a requirement.
+  ## If RequireJS is not enabled, then we will need to load i18n explicitly
+  ## so that JavaScript on the page can use internationalization functions.
+  % if not enable_require_js:
+  <script type="text/javascript" src="/i18n.js"></script>
+  % endif
+
 
   <link rel="icon" type="image/x-icon" href="${static.url(microsite.get_value('favicon_path', settings.FAVICON_PATH))}" />
 
@@ -67,10 +76,15 @@
   <%static:css group='style-app-extend1'/>
   <%static:css group='style-app-extend2'/>
 
-  % if disable_courseware_js:
-    <%static:js group='base_vendor'/>
+  % if enable_require_js:
+    <script type="text/javascript" src="${static.url("require-config-lms.js")}"></script>
+    ${render_require_js_path_overrides(settings.REQUIRE_JS_PATH_OVERRIDES)}
   % else:
-    <%static:js group='main_vendor'/>
+    % if disable_courseware_js:
+      <%static:js group='base_vendor'/>
+    % else:
+      <%static:js group='main_vendor'/>
+    % endif
   % endif
 
   <script>
@@ -149,7 +163,7 @@
     </div>
 
     <%block name="footer">
-      ## Can be overridden by child templates wanting to hide the footer. 
+      ## Can be overridden by child templates wanting to hide the footer.
       <%
         if theme_enabled() and not is_microsite():
           footer_file = 'theme-footer.html'
@@ -163,7 +177,8 @@
 
   </div>
 
-  % if not disable_courseware_js:
+  <script>window.baseUrl = "${settings.STATIC_URL}";</script>
+  % if not enable_require_js and not disable_courseware_js:
     <%static:js group='application'/>
     <%static:js group='module-js'/>
   % endif

lms/templates/navigation-edx.html

@@ -142,7 +142,9 @@ site_status_msg = get_site_status_msg(course_id)
 <![endif]-->
 % endif
 
-%if not user.is_authenticated():
+## The forgot password modal JavaScript assumes that JQuery is loaded,
+## which is not necessarily the case when using RequireJS.
+%if not enable_require_js and not user.is_authenticated():
   <%include file="forgot_password_modal.html" />
 %endif
 

lms/templates/navigation.html

@@ -141,7 +141,9 @@ site_status_msg = get_site_status_msg(course_id)
 <![endif]-->
 % endif
 
-%if not user.is_authenticated():
+## The forgot password modal JavaScript assumes that JQuery is loaded,
+## which is not necessarily the case when using RequireJS.
+%if not enable_require_js and not user.is_authenticated():
   <%include file="forgot_password_modal.html" />
 %endif
 

lms/urls.py

@@ -104,7 +104,7 @@ js_info_dict = {
 
 urlpatterns += (
     # Serve catalog of localized strings to be rendered by Javascript
-    url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict),
+    url(r'^i18n.js$', 'django.views.i18n.javascript_catalog', js_info_dict),
 )
 
 # sysadmin dashboard, to see what courses are loaded, to delete & load courses

pavelib/assets.py

@@ -165,7 +165,7 @@ def collect_assets(systems, settings):
     `settings` is the Django settings module to use.
     """
     for sys in systems:
-        sh(django_cmd(sys, settings, "collectstatic --noinput > /dev/null"))
+        sh(django_cmd(sys, settings, "collectstatic --clear --noinput > /dev/null"))
 
 
 @task