Support running Studio with optimized assets

cms/envs/bok_choy.py

@@ -36,7 +36,7 @@ from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import
 INSTALLED_APPS += ('django_extensions',)
 
 # Redirect to the test_root folder within the repo
-TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=no-value-for-parameter
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
 GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
 LOG_DIR = (TEST_ROOT / "log").abspath()
 
@@ -64,7 +64,7 @@ STATICFILES_FINDERS = (
     'staticfiles.finders.FileSystemFinder',
 )
 STATICFILES_DIRS = (
-    (TEST_ROOT / "staticfiles").abspath(),
+    (TEST_ROOT / "staticfiles" / "cms").abspath(),
 )
 
 # Silence noisy logs

cms/envs/devstack_optimized.py

+"""
+Settings to run Studio in devstack using optimized static assets.
+
+This configuration changes Studio to use the optimized static assets generated for testing,
+rather than picking up the files directly from the source tree.
+
+The following Paver command can be used to run Studio in optimized mode:
+
+  paver devstack studio --optimized
+
+You can also generate the assets explicitly and then run Studio:
+
+  paver update_assets cms --settings=test_static_optimized
+  paver devstack studio --settings=devstack_optimized --fast
+
+Note that changes to JavaScript assets will not be picked up automatically
+as they are for non-optimized devstack. Instead, update_assets must be
+invoked each time that changes have been made.
+"""
+
+########################## Devstack settings ###################################
+
+from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
+
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
+
+############################ STATIC FILES #############################
+
+# Enable debug so that static assets are served by Django
+DEBUG = True
+
+#  Serve static files at /static directly from the staticfiles directory under test root.
+# Note: optimized files for testing are generated with settings from test_static_optimized
+STATIC_URL = "/static/"
+STATICFILES_FINDERS = (
+    'staticfiles.finders.FileSystemFinder',
+)
+STATICFILES_DIRS = (
+    (TEST_ROOT / "staticfiles" / "cms").abspath(),
+)

cms/envs/test_static_optimized.py

@@ -10,36 +10,26 @@ support both generating static assets to a directory and also serving static
 from the same directory.
 """
 
-import os
-from path import path  # pylint: disable=no-name-in-module
+# Start with the common settings
+from .common import *  # pylint: disable=wildcard-import, unused-wildcard-import
 
-# Pylint gets confused by path.py instances, which report themselves as class
-# objects. As a result, pylint applies the wrong regex in validating names,
-# and throws spurious errors. Therefore, we disable invalid-name checking.
-# pylint: disable=invalid-name
+# Use an in-memory database since this settings file is only used for updating assets
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+    },
 
+}
+######################### Static file overrides ####################################
 
-########################## Prod-like settings ###################################
-# These should be as close as possible to the settings we use in production.
-# As in prod, we read in environment and auth variables from JSON files.
-# Unlike in prod, we use the JSON files stored in this repo.
-# This is a convenience for ensuring (a) that we can consistently find the files
-# and (b) that the files are the same in Jenkins as in local dev.
-os.environ['SERVICE_VARIANT'] = 'bok_choy'
-os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname()  # pylint: disable=no-value-for-parameter
-
-from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import
-
-######################### Testing overrides ####################################
-
-# Redirects to the test_root folder within the repo
-TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=no-value-for-parameter
+# Redirect to the test_root folder within the repo
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
 LOG_DIR = (TEST_ROOT / "log").abspath()
 
-# Stores the static files under test root so that they don't overwrite existing static assets
-STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
+# Store the static files under test root so that they don't overwrite existing static assets
+STATIC_ROOT = (TEST_ROOT / "staticfiles" / "cms").abspath()
 
-# Disables uglify when tests are running (used by build.js).
+# Disable uglify when tests are running (used by build.js).
 # 1. Uglify is by far the slowest part of the build process
 # 2. Having full source code makes debugging tests easier for developers
 os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none'

lms/envs/bok_choy.py

@@ -75,7 +75,7 @@ EDXNOTES_PUBLIC_API = 'http://localhost:8042/api/v1'
 EDXNOTES_INTERNAL_API = 'http://localhost:8042/api/v1'
 
 # Enable django-pipeline and staticfiles
-STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
+STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath()
 
 # Silence noisy logs
 import logging

lms/envs/devstack_optimized.py

+"""
+Settings to run LMS in devstack using optimized static assets.
+
+This configuration changes LMS to use the optimized static assets generated for testing,
+rather than picking up the files directly from the source tree.
+
+The following Paver command can be used to run LMS in optimized mode:
+
+  paver devstack lms --optimized
+
+You can also generate the assets explicitly and then run Studio:
+
+  paver update_assets lms --settings=test_static_optimized
+  paver devstack lms --settings=devstack_optimized --fast
+
+Note that changes to JavaScript assets will not be picked up automatically
+as they are for non-optimized devstack. Instead, update_assets must be
+invoked each time that changes have been made.
+"""
+
+########################## Devstack settings ###################################
+
+from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
+
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
+
+############################ STATIC FILES #############################
+
+# Enable debug so that static assets are served by Django
+DEBUG = True
+
+#  Serve static files at /static directly from the staticfiles directory under test root.
+# Note: optimized files for testing are generated with settings from test_static_optimized
+STATIC_URL = "/static/"
+STATICFILES_FINDERS = (
+    'staticfiles.finders.FileSystemFinder',
+)
+STATICFILES_DIRS = (
+    (TEST_ROOT / "staticfiles" / "lms").abspath(),
+)

lms/envs/test_static_optimized.py

 """
 Settings used when generating static assets for use in tests.
 
-Bok Choy uses two different settings files:
+For example, Bok Choy uses two different settings files:
 1. test_static_optimized is used when invoking collectstatic
 2. bok_choy is used when running CMS and LMS
 
 Note: it isn't possible to have a single settings file, because Django doesn't
 support both generating static assets to a directory and also serving static
 from the same directory.
-
 """
 
-# TODO: update the Bok Choy tests to run with optimized static assets (as is done in Studio)
+# Start with the common settings
+from .common import *  # pylint: disable=wildcard-import, unused-wildcard-import
+
+# Use an in-memory database since this settings file is only used for updating assets
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+    },
+
+}
+
+# Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up
+XQUEUE_INTERFACE = {
+    "url": "https://sandbox-xqueue.edx.org",
+    "django_auth": {
+        "username": "lms",
+        "password": "***REMOVED***"
+    },
+    "basic_auth": ('anant', 'agarwal'),
+}
+
+
+######################### Static file overrides ####################################
+
+# Redirect to the test_root folder within the repo
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
+LOG_DIR = (TEST_ROOT / "log").abspath()
+
+# Store the static files under test root so that they don't overwrite existing static assets
+STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath()
 
-from .bok_choy import *  # pylint: disable=wildcard-import, unused-wildcard-import
+# Disable uglify when tests are running (used by build.js).
+# 1. Uglify is by far the slowest part of the build process
+# 2. Having full source code makes debugging tests easier for developers
+os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none'

pavelib/assets.py

@@ -3,12 +3,12 @@ Asset compilation and collection.
 """
 from __future__ import print_function
 import argparse
+from paver import tasks
 from paver.easy import sh, path, task, cmdopts, needs, consume_args, call_task, no_help
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 import glob
 import traceback
-import os
 from .utils.envs import Env
 from .utils.cmd import cmd, django_cmd
 
@@ -191,6 +191,10 @@ def watch_assets(options):
     """
     Watch for changes to asset files, and regenerate js/css
     """
+    # Don't watch assets when performing a dry run
+    if tasks.environment.dry_run:
+        return
+
     observer = Observer()
 
     CoffeeScriptWatcher().register(observer)
@@ -246,10 +250,10 @@ def update_assets(args):
     compile_templated_sass(args.system, args.settings)
     process_xmodule_assets()
     compile_coffeescript()
-    call_task('compile_sass', options={'debug': args.debug})
+    call_task('pavelib.assets.compile_sass', options={'debug': args.debug})
 
     if args.collect:
         collect_assets(args.system, args.settings)
 
     if args.watch:
-        call_task('watch_assets', options={'background': not args.debug})
+        call_task('pavelib.assets.watch_assets', options={'background': not args.debug})

pavelib/paver_tests/utils.py

+"""Unit tests for the Paver server tasks."""
+
+import os
+from paver import tasks
+from unittest import TestCase
+
+
+class PaverTestCase(TestCase):
+    """
+    Base class for Paver test cases.
+    """
+    def setUp(self):
+        super(PaverTestCase, self).setUp()
+
+        # Show full length diffs upon test failure
+        self.maxDiff = None  # pylint: disable=invalid-name
+
+        # Create a mock Paver environment
+        tasks.environment = MockEnvironment()
+
+        # Don't run pre-reqs
+        os.environ['NO_PREREQ_INSTALL'] = 'true'
+
+    def tearDown(self):
+        super(PaverTestCase, self).tearDown()
+        tasks.environment = tasks.Environment()
+        del os.environ['NO_PREREQ_INSTALL']
+
+    @property
+    def task_messages(self):
+        """Returns the messages output by the Paver task."""
+        return tasks.environment.messages
+
+    def reset_task_messages(self):
+        """Clear the recorded message"""
+        tasks.environment.messages = []
+
+
+class MockEnvironment(tasks.Environment):
+    """
+    Mock environment that collects information about Paver commands.
+    """
+    def __init__(self):
+        super(MockEnvironment, self).__init__()
+        self.dry_run = True
+        self.messages = []
+
+    def info(self, message, *args):
+        """Capture any messages that have been recorded"""
+        if args:
+            output = message % args
+        else:
+            output = message
+        if not output.startswith("--->"):
+            self.messages.append(output)

pavelib/servers.py

@@ -2,24 +2,36 @@
 Run and manage servers for local development.
 """
 from __future__ import print_function
-import sys
 import argparse
 from paver.easy import *
+
+from .assets import collect_assets
 from .utils.cmd import django_cmd
 from .utils.process import run_process, run_multi_processes
 
 
 DEFAULT_PORT = {"lms": 8000, "studio": 8001}
 DEFAULT_SETTINGS = 'devstack'
+OPTIMIZED_SETTINGS = "devstack_optimized"
+OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized"
 
+ASSET_SETTINGS_HELP = (
+    "Settings file used for updating assets. Defaults to the value of the settings variable if not provided."
+)
 
-def run_server(system, settings=None, port=None, skip_assets=False, contracts=False):
-    """
-    Start the server for the specified `system` (lms or studio).
-    `settings` is the Django settings module to use; if not provided, use the default.
-    `port` is the port to run the server on; if not provided, use the default port for the system.
 
-    If `skip_assets` is True, skip the asset compilation step.
+def run_server(
+        system, fast=False, settings=None, asset_settings=None, port=None, contracts=False
+):
+    """Start the server for LMS or Studio.
+
+    Args:
+        system (str): The system to be run (lms or studio).
+        fast (bool): If true, then start the server immediately without updating assets (defaults to False).
+        settings (str): The Django settings module to use; if not provided, use the default.
+        asset_settings (str) The settings to use when generating assets. If not provided, assets are not generated.
+        port (str): The port number to run the server on. If not provided, uses the default port for the system.
+        contracts (bool) If true then PyContracts is enabled (defaults to False).
     """
     if system not in ['lms', 'studio']:
         print("System must be either lms or studio", file=sys.stderr)
@@ -28,9 +40,13 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa
     if not settings:
         settings = DEFAULT_SETTINGS
 
-    if not skip_assets:
-        # Local dev settings use staticfiles to serve assets, so we can skip the collecstatic step
-        args = [system, '--settings={}'.format(settings), '--skip-collect', '--watch']
+    if not fast and asset_settings:
+        args = [system, '--settings={}'.format(asset_settings), '--watch']
+        # The default settings use DEBUG mode for running the server which means that
+        # the optimized assets are ignored, so we skip collectstatic in that case
+        # to save time.
+        if settings == DEFAULT_SETTINGS:
+            args.append('--skip-collect')
         call_task('pavelib.assets.update_assets', args=args)
 
     if port is None:
@@ -48,40 +64,55 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings"),
+    ("asset-settings=", "a", ASSET_SETTINGS_HELP),
     ("port=", "p", "Port"),
-    ("fast", "f", "Skip updating assets")
+    ("fast", "f", "Skip updating assets"),
 ])
 def lms(options):
     """
     Run the LMS server.
     """
-    settings = getattr(options, 'settings', None)
+    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
+    asset_settings = getattr(options, 'asset-settings', settings)
     port = getattr(options, 'port', None)
     fast = getattr(options, 'fast', False)
-    run_server('lms', settings=settings, port=port, skip_assets=fast)
+    run_server(
+        'lms',
+        fast=fast,
+        settings=settings,
+        asset_settings=asset_settings,
+        port=port,
+    )
 
 
 @task
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings"),
+    ("asset-settings=", "a", ASSET_SETTINGS_HELP),
     ("port=", "p", "Port"),
-    ("fast", "f", "Skip updating assets")
+    ("fast", "f", "Skip updating assets"),
 ])
 def studio(options):
     """
     Run the Studio server.
     """
-    settings = getattr(options, 'settings', None)
+    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
+    asset_settings = getattr(options, 'asset-settings', settings)
     port = getattr(options, 'port', None)
     fast = getattr(options, 'fast', False)
-    run_server('studio', settings=settings, port=port, skip_assets=fast)
+    run_server(
+        'studio',
+        fast=fast,
+        settings=settings,
+        asset_settings=asset_settings,
+        port=port,
+    )
 
 
 @task
 @needs('pavelib.prereqs.install_prereqs')
 @consume_args
-@no_help
 def devstack(args):
     """
     Start the devstack lms or studio server
@@ -89,6 +120,9 @@ def devstack(args):
     parser = argparse.ArgumentParser(prog='paver devstack')
     parser.add_argument('system', type=str, nargs=1, help="lms or studio")
     parser.add_argument('--fast', action='store_true', default=False, help="Skip updating assets")
+    parser.add_argument('--optimized', action='store_true', default=False, help="Run with optimized assets")
+    parser.add_argument('--settings', type=str, default=DEFAULT_SETTINGS, help="Settings file")
+    parser.add_argument('--asset-settings', type=str, default=None, help=ASSET_SETTINGS_HELP)
     parser.add_argument(
         '--no-contracts',
         action='store_true',
@@ -96,7 +130,18 @@ def devstack(args):
         help="Disable contracts. By default, they're enabled in devstack."
     )
     args = parser.parse_args(args)
-    run_server(args.system[0], settings='devstack', skip_assets=args.fast, contracts=(not args.no_contracts))
+    settings = args.settings
+    asset_settings = args.asset_settings if args.asset_settings else settings
+    if args.optimized:
+        settings = OPTIMIZED_SETTINGS
+        asset_settings = OPTIMIZED_ASSETS_SETTINGS
+    run_server(
+        args.system[0],
+        fast=args.fast,
+        settings=settings,
+        asset_settings=asset_settings,
+        contracts=not args.no_contracts,
+    )
 
 
 @task
@@ -116,33 +161,70 @@ def celery(options):
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings for both LMS and Studio"),
+    ("asset_settings=", "a", "Django settings for updating assets for both LMS and Studio (defaults to settings)"),
     ("worker_settings=", "w", "Celery worker Django settings"),
     ("fast", "f", "Skip updating assets"),
+    ("optimized", "o", "Run with optimized assets"),
     ("settings_lms=", "l", "Set LMS only, overriding the value from --settings (if provided)"),
+    ("asset_settings_lms=", "al", "Set LMS only, overriding the value from --asset_settings (if provided)"),
     ("settings_cms=", "c", "Set Studio only, overriding the value from --settings (if provided)"),
+    ("asset_settings_cms=", "ac", "Set Studio only, overriding the value from --asset_settings (if provided)"),
 ])
 def run_all_servers(options):
     """
     Runs Celery workers, Studio, and LMS.
     """
     settings = getattr(options, 'settings', DEFAULT_SETTINGS)
-    settings_lms = getattr(options, 'settings_lms', settings)
-    settings_cms = getattr(options, 'settings_cms', settings)
+    asset_settings = getattr(options, 'asset_settings', settings)
     worker_settings = getattr(options, 'worker_settings', 'dev_with_worker')
     fast = getattr(options, 'fast', False)
+    optimized = getattr(options, 'optimized', False)
+
+    if optimized:
+        settings = OPTIMIZED_SETTINGS
+        asset_settings = OPTIMIZED_ASSETS_SETTINGS
+
+    settings_lms = getattr(options, 'settings_lms', settings)
+    settings_cms = getattr(options, 'settings_cms', settings)
+    asset_settings_lms = getattr(options, 'asset_settings_lms', asset_settings)
+    asset_settings_cms = getattr(options, 'asset_settings_cms', asset_settings)
 
     if not fast:
-        args = ['lms', '--settings={}'.format(settings_lms), '--skip-collect']
+        # First update assets for both LMS and Studio but don't collect static yet
+        args = [
+            'lms', 'studio',
+            '--settings={}'.format(asset_settings),
+            '--skip-collect'
+        ]
         call_task('pavelib.assets.update_assets', args=args)
 
-        args = ['studio', '--settings={}'.format(settings_cms), '--skip-collect']
-        call_task('pavelib.assets.update_assets', args=args)
+        # Now collect static for each system separately with the appropriate settings.
+        # Note that the default settings use DEBUG mode for running the server which
+        # means that the optimized assets are ignored, so we skip collectstatic in that
+        # case to save time.
+        if settings != DEFAULT_SETTINGS:
+            collect_assets(['lms'], asset_settings_lms)
+            collect_assets(['studio'], asset_settings_cms)
 
+        # Install an asset watcher to regenerate files that change
         call_task('pavelib.assets.watch_assets', options={'background': True})
+
+    # Start up LMS, CMS and Celery
+    lms_port = DEFAULT_PORT['lms']
+    cms_port = DEFAULT_PORT['studio']
+    lms_runserver_args = ["0.0.0.0:{}".format(lms_port)]
+    cms_runserver_args = ["0.0.0.0:{}".format(cms_port)]
+
     run_multi_processes([
-        django_cmd('lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['lms'])),
-        django_cmd('studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['studio'])),
-        django_cmd('lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.')
+        django_cmd(
+            'lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', *lms_runserver_args
+        ),
+        django_cmd(
+            'studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', *cms_runserver_args
+        ),
+        django_cmd(
+            'lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.'
+        )
     ])
 
 

pavelib/utils/process.py

@@ -9,6 +9,8 @@ import signal
 import psutil
 import atexit
 
+from paver import tasks
+
 
 def kill_process(proc):
     """
@@ -41,6 +43,14 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None):
         err_log_file = open(err_log, 'w')
         kwargs['stderr'] = err_log_file
 
+    # If the user is performing a dry run of a task, then just log
+    # the command strings and return so that no destructive operations
+    # are performed.
+    if tasks.environment.dry_run:
+        for cmd in cmd_list:
+            tasks.environment.info(cmd)
+        return
+
     try:
         for cmd in cmd_list:
             pids.extend([subprocess.Popen(cmd, **kwargs)])