Support running Studio with optimized assets

cms/envs/bok_choy.py

@@ -36,7 +36,7 @@ from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import
 INSTALLED_APPS += ('django_extensions',)
 
 # Redirect to the test_root folder within the repo
-TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=no-value-for-parameter
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
 GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
 LOG_DIR = (TEST_ROOT / "log").abspath()
 
@@ -64,7 +64,7 @@ STATICFILES_FINDERS = (
     'staticfiles.finders.FileSystemFinder',
 )
 STATICFILES_DIRS = (
-    (TEST_ROOT / "staticfiles").abspath(),
+    (TEST_ROOT / "staticfiles" / "cms").abspath(),
 )
 
 # Silence noisy logs

cms/envs/devstack_optimized.py

+"""
+Settings to run Studio in devstack using optimized static assets.
+
+This configuration changes Studio to use the optimized static assets generated for testing,
+rather than picking up the files directly from the source tree.
+
+The following Paver command can be used to run Studio in optimized mode:
+
+  paver devstack studio --optimized
+
+You can also generate the assets explicitly and then run Studio:
+
+  paver update_assets cms --settings=test_static_optimized
+  paver devstack studio --settings=devstack_optimized --fast
+
+Note that changes to JavaScript assets will not be picked up automatically
+as they are for non-optimized devstack. Instead, update_assets must be
+invoked each time that changes have been made.
+"""
+
+########################## Devstack settings ###################################
+
+from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
+
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
+
+############################ STATIC FILES #############################
+
+# Enable debug so that static assets are served by Django
+DEBUG = True
+
+#  Serve static files at /static directly from the staticfiles directory under test root.
+# Note: optimized files for testing are generated with settings from test_static_optimized
+STATIC_URL = "/static/"
+STATICFILES_FINDERS = (
+    'staticfiles.finders.FileSystemFinder',
+)
+STATICFILES_DIRS = (
+    (TEST_ROOT / "staticfiles" / "cms").abspath(),
+)

cms/envs/test_static_optimized.py

@@ -10,36 +10,26 @@ support both generating static assets to a directory and also serving static
 from the same directory.
 """
 
-import os
-from path import path  # pylint: disable=no-name-in-module
+# Start with the common settings
+from .common import *  # pylint: disable=wildcard-import, unused-wildcard-import
 
-# Pylint gets confused by path.py instances, which report themselves as class
-# objects. As a result, pylint applies the wrong regex in validating names,
-# and throws spurious errors. Therefore, we disable invalid-name checking.
-# pylint: disable=invalid-name
+# Use an in-memory database since this settings file is only used for updating assets
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+    },
 
+}
+######################### Static file overrides ####################################
 
-########################## Prod-like settings ###################################
-# These should be as close as possible to the settings we use in production.
-# As in prod, we read in environment and auth variables from JSON files.
-# Unlike in prod, we use the JSON files stored in this repo.
-# This is a convenience for ensuring (a) that we can consistently find the files
-# and (b) that the files are the same in Jenkins as in local dev.
-os.environ['SERVICE_VARIANT'] = 'bok_choy'
-os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname()  # pylint: disable=no-value-for-parameter
-
-from .aws import *  # pylint: disable=wildcard-import, unused-wildcard-import
-
-######################### Testing overrides ####################################
-
-# Redirects to the test_root folder within the repo
-TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=no-value-for-parameter
+# Redirect to the test_root folder within the repo
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
 LOG_DIR = (TEST_ROOT / "log").abspath()
 
-# Stores the static files under test root so that they don't overwrite existing static assets
-STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
+# Store the static files under test root so that they don't overwrite existing static assets
+STATIC_ROOT = (TEST_ROOT / "staticfiles" / "cms").abspath()
 
-# Disables uglify when tests are running (used by build.js).
+# Disable uglify when tests are running (used by build.js).
 # 1. Uglify is by far the slowest part of the build process
 # 2. Having full source code makes debugging tests easier for developers
 os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none'

lms/envs/bok_choy.py

@@ -75,7 +75,7 @@ EDXNOTES_PUBLIC_API = 'http://localhost:8042/api/v1'
 EDXNOTES_INTERNAL_API = 'http://localhost:8042/api/v1'
 
 # Enable django-pipeline and staticfiles
-STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
+STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath()
 
 # Silence noisy logs
 import logging

lms/envs/devstack_optimized.py

+"""
+Settings to run LMS in devstack using optimized static assets.
+
+This configuration changes LMS to use the optimized static assets generated for testing,
+rather than picking up the files directly from the source tree.
+
+The following Paver command can be used to run LMS in optimized mode:
+
+  paver devstack lms --optimized
+
+You can also generate the assets explicitly and then run Studio:
+
+  paver update_assets lms --settings=test_static_optimized
+  paver devstack lms --settings=devstack_optimized --fast
+
+Note that changes to JavaScript assets will not be picked up automatically
+as they are for non-optimized devstack. Instead, update_assets must be
+invoked each time that changes have been made.
+"""
+
+########################## Devstack settings ###################################
+
+from .devstack import *  # pylint: disable=wildcard-import, unused-wildcard-import
+
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
+
+############################ STATIC FILES #############################
+
+# Enable debug so that static assets are served by Django
+DEBUG = True
+
+#  Serve static files at /static directly from the staticfiles directory under test root.
+# Note: optimized files for testing are generated with settings from test_static_optimized
+STATIC_URL = "/static/"
+STATICFILES_FINDERS = (
+    'staticfiles.finders.FileSystemFinder',
+)
+STATICFILES_DIRS = (
+    (TEST_ROOT / "staticfiles" / "lms").abspath(),
+)

lms/envs/test_static_optimized.py

 """
 Settings used when generating static assets for use in tests.
 
-Bok Choy uses two different settings files:
+For example, Bok Choy uses two different settings files:
 1. test_static_optimized is used when invoking collectstatic
 2. bok_choy is used when running CMS and LMS
 
 Note: it isn't possible to have a single settings file, because Django doesn't
 support both generating static assets to a directory and also serving static
 from the same directory.
-
 """
 
-# TODO: update the Bok Choy tests to run with optimized static assets (as is done in Studio)
+# Start with the common settings
+from .common import *  # pylint: disable=wildcard-import, unused-wildcard-import
+
+# Use an in-memory database since this settings file is only used for updating assets
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+    },
+
+}
+
+# Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up
+XQUEUE_INTERFACE = {
+    "url": "https://sandbox-xqueue.edx.org",
+    "django_auth": {
+        "username": "lms",
+        "password": "***REMOVED***"
+    },
+    "basic_auth": ('anant', 'agarwal'),
+}
+
+
+######################### Static file overrides ####################################
+
+# Redirect to the test_root folder within the repo
+TEST_ROOT = REPO_ROOT / "test_root"  # pylint: disable=no-value-for-parameter
+LOG_DIR = (TEST_ROOT / "log").abspath()
+
+# Store the static files under test root so that they don't overwrite existing static assets
+STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath()
 
-from .bok_choy import *  # pylint: disable=wildcard-import, unused-wildcard-import
+# Disable uglify when tests are running (used by build.js).
+# 1. Uglify is by far the slowest part of the build process
+# 2. Having full source code makes debugging tests easier for developers
+os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none'

pavelib/assets.py

@@ -3,12 +3,12 @@ Asset compilation and collection.
 """
 from __future__ import print_function
 import argparse
+from paver import tasks
 from paver.easy import sh, path, task, cmdopts, needs, consume_args, call_task, no_help
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 import glob
 import traceback
-import os
 from .utils.envs import Env
 from .utils.cmd import cmd, django_cmd
 
@@ -191,6 +191,10 @@ def watch_assets(options):
     """
     Watch for changes to asset files, and regenerate js/css
     """
+    # Don't watch assets when performing a dry run
+    if tasks.environment.dry_run:
+        return
+
     observer = Observer()
 
     CoffeeScriptWatcher().register(observer)
@@ -246,10 +250,10 @@ def update_assets(args):
     compile_templated_sass(args.system, args.settings)
     process_xmodule_assets()
     compile_coffeescript()
-    call_task('compile_sass', options={'debug': args.debug})
+    call_task('pavelib.assets.compile_sass', options={'debug': args.debug})
 
     if args.collect:
         collect_assets(args.system, args.settings)
 
     if args.watch:
-        call_task('watch_assets', options={'background': not args.debug})
+        call_task('pavelib.assets.watch_assets', options={'background': not args.debug})

pavelib/paver_tests/test_servers.py

+"""Unit tests for the Paver server tasks."""
+
+import ddt
+import os
+from paver.easy import call_task
+
+from .utils import PaverTestCase
+
+EXPECTED_COFFEE_COMMAND = (
+    "node_modules/.bin/coffee --compile `find {platform_root}/lms "
+    "{platform_root}/cms {platform_root}/common -type f -name \"*.coffee\"`"
+)
+EXPECTED_SASS_COMMAND = (
+    "sass --update --cache-location /tmp/sass-cache --default-encoding utf-8 --style compressed"
+    " --quiet --load-path common/static --load-path common/static/sass"
+    " --load-path lms/static/sass --load-path lms/static/certificates/sass"
+    " --load-path cms/static/sass --load-path common/static/sass"
+    " lms/static/sass:lms/static/css lms/static/certificates/sass:lms/static/certificates/css"
+    " cms/static/sass:cms/static/css common/static/sass:common/static/css"
+)
+EXPECTED_PREPROCESS_ASSETS_COMMAND = (
+    "python manage.py {system} --settings={asset_settings} preprocess_assets"
+)
+EXPECTED_COLLECT_STATIC_COMMAND = (
+    "python manage.py {system} --settings={asset_settings} collectstatic --noinput > /dev/null"
+)
+EXPECTED_CELERY_COMMAND = (
+    "python manage.py lms --settings={settings} celery worker --beat --loglevel=INFO --pythonpath=."
+)
+EXPECTED_RUN_SERVER_COMMAND = (
+    "python manage.py {system} --settings={settings} runserver --traceback --pythonpath=. 0.0.0.0:{port}"
+)
+
+
+@ddt.ddt
+class TestPaverServerTasks(PaverTestCase):
+    """
+    Test the Paver server tasks.
+    """
+    @ddt.data(
+        [{}],
+        [{"settings": "aws"}],
+        [{"asset-settings": "test_static_optimized"}],
+        [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
+        [{"fast": True}],
+        [{"port": 8030}],
+    )
+    @ddt.unpack
+    def test_lms(self, options):
+        """
+        Test the "devstack" task.
+        """
+        self.verify_server_task("lms", options)
+
+    @ddt.data(
+        [{}],
+        [{"settings": "aws"}],
+        [{"asset-settings": "test_static_optimized"}],
+        [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
+        [{"fast": True}],
+        [{"port": 8031}],
+    )
+    @ddt.unpack
+    def test_studio(self, options):
+        """
+        Test the "devstack" task.
+        """
+        self.verify_server_task("studio", options)
+
+    @ddt.data(
+        [{}],
+        [{"settings": "aws"}],
+        [{"asset-settings": "test_static_optimized"}],
+        [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
+        [{"fast": True}],
+        [{"optimized": True}],
+        [{"optimized": True, "fast": True}],
+        [{"no-contracts": True}],
+    )
+    @ddt.unpack
+    def test_devstack(self, server_options):
+        """
+        Test the "devstack" task.
+        """
+        options = server_options.copy()
+
+        # First test with LMS
+        options["system"] = "lms"
+        self.verify_server_task("devstack", options, contracts_default=True)
+
+        # Then test with Studio
+        options["system"] = "cms"
+        self.verify_server_task("devstack", options, contracts_default=True)
+
+    @ddt.data(
+        [{}],
+        [{"settings": "aws"}],
+        [{"asset_settings": "test_static_optimized"}],
+        [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
+        [{"fast": True}],
+        [{"optimized": True}],
+        [{"optimized": True, "fast": True}],
+    )
+    @ddt.unpack
+    def test_run_all_servers(self, options):
+        """
+        Test the "run_all_servers" task.
+        """
+        self.verify_run_all_servers_task(options)
+
+    @ddt.data(
+        [{}],
+        [{"settings": "aws"}],
+    )
+    @ddt.unpack
+    def test_celery(self, options):
+        """
+        Test the "celery" task.
+        """
+        settings = options.get("settings", "dev_with_worker")
+        call_task("pavelib.servers.celery", options=options)
+        self.assertEquals(self.task_messages, [EXPECTED_CELERY_COMMAND.format(settings=settings)])
+
+    @ddt.data(
+        [{}],
+        [{"settings": "aws"}],
+    )
+    @ddt.unpack
+    def test_update_db(self, options):
+        """
+        Test the "update_db" task.
+        """
+        settings = options.get("settings", "devstack")
+        call_task("pavelib.servers.update_db", options=options)
+        db_command = "python manage.py {server} --settings={settings} syncdb --migrate --traceback --pythonpath=."
+        self.assertEquals(
+            self.task_messages,
+            [
+                db_command.format(server="lms", settings=settings),
+                db_command.format(server="cms", settings=settings),
+            ]
+        )
+
+    @ddt.data(
+        ["lms", {}],
+        ["lms", {"settings": "aws"}],
+        ["cms", {}],
+        ["cms", {"settings": "aws"}],
+    )
+    @ddt.unpack
+    def test_check_settings(self, system, options):
+        """
+        Test the "check_settings" task.
+        """
+        settings = options.get("settings", "devstack")
+        call_task("pavelib.servers.check_settings", args=[system, settings])
+        self.assertEquals(
+            self.task_messages,
+            [
+                "echo 'import {system}.envs.{settings}' "
+                "| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format(
+                    system=system, settings=settings
+                ),
+            ]
+        )
+
+    def verify_server_task(self, task_name, options, contracts_default=False):
+        """
+        Verify the output of a server task.
+        """
+        settings = options.get("settings", None)
+        asset_settings = options.get("asset-settings", None)
+        is_optimized = options.get("optimized", False)
+        is_fast = options.get("fast", False)
+        no_contracts = options.get("no-contracts", not contracts_default)
+        if task_name == "devstack":
+            system = options.get("system")
+        elif task_name == "studio":
+            system = "cms"
+        else:
+            system = "lms"
+        port = options.get("port", "8000" if system == "lms" else "8001")
+        self.reset_task_messages()
+        if task_name == "devstack":
+            args = ["studio" if system == "cms" else system]
+            if settings:
+                args.append("--settings={settings}".format(settings=settings))
+            if asset_settings:
+                args.append("--asset-settings={asset_settings}".format(asset_settings=asset_settings))
+            if is_optimized:
+                args.append("--optimized")
+            if is_fast:
+                args.append("--fast")
+            if no_contracts:
+                args.append("--no-contracts")
+            call_task("pavelib.servers.devstack", args=args)
+        else:
+            call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options)
+        expected_messages = []
+        expected_settings = settings if settings else "devstack"
+        expected_asset_settings = asset_settings if asset_settings else expected_settings
+        if is_optimized:
+            expected_settings = "devstack_optimized"
+            expected_asset_settings = "test_static_optimized"
+        expected_collect_static = not is_fast and expected_settings != "devstack"
+        platform_root = os.getcwd()
+        if not is_fast:
+            expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
+                system=system, asset_settings=expected_asset_settings
+            ))
+            expected_messages.append("xmodule_assets common/static/xmodule")
+            expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
+            expected_messages.append(EXPECTED_SASS_COMMAND)
+        if expected_collect_static:
+            expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
+                system=system, asset_settings=expected_asset_settings
+            ))
+        expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format(
+            system=system,
+            settings=expected_settings,
+            port=port,
+        )
+        if not no_contracts:
+            expected_run_server_command += " --contracts"
+        expected_messages.append(expected_run_server_command)
+        self.assertEquals(self.task_messages, expected_messages)
+
+    def verify_run_all_servers_task(self, options):
+        """
+        Verify the output of a server task.
+        """
+        settings = options.get("settings", None)
+        asset_settings = options.get("asset_settings", None)
+        is_optimized = options.get("optimized", False)
+        is_fast = options.get("fast", False)
+        self.reset_task_messages()
+        call_task("pavelib.servers.run_all_servers", options=options)
+        expected_settings = settings if settings else "devstack"
+        expected_asset_settings = asset_settings if asset_settings else expected_settings
+        if is_optimized:
+            expected_settings = "devstack_optimized"
+            expected_asset_settings = "test_static_optimized"
+        expected_collect_static = not is_fast and expected_settings != "devstack"
+        platform_root = os.getcwd()
+        expected_messages = []
+        if not is_fast:
+            expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
+                system="lms", asset_settings=expected_asset_settings
+            ))
+            expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
+                system="cms", asset_settings=expected_asset_settings
+            ))
+            expected_messages.append("xmodule_assets common/static/xmodule")
+            expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
+            expected_messages.append(EXPECTED_SASS_COMMAND)
+        if expected_collect_static:
+            expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
+                system="lms", asset_settings=expected_asset_settings
+            ))
+            expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
+                system="cms", asset_settings=expected_asset_settings
+            ))
+        expected_messages.append(
+            EXPECTED_RUN_SERVER_COMMAND.format(
+                system="lms",
+                settings=expected_settings,
+                port=8000,
+            )
+        )
+        expected_messages.append(
+            EXPECTED_RUN_SERVER_COMMAND.format(
+                system="cms",
+                settings=expected_settings,
+                port=8001,
+            )
+        )
+        expected_messages.append(EXPECTED_CELERY_COMMAND.format(settings="dev_with_worker"))
+        self.assertEquals(self.task_messages, expected_messages)

pavelib/paver_tests/utils.py

+"""Unit tests for the Paver server tasks."""
+
+import os
+from paver import tasks
+from unittest import TestCase
+
+
+class PaverTestCase(TestCase):
+    """
+    Base class for Paver test cases.
+    """
+    def setUp(self):
+        super(PaverTestCase, self).setUp()
+
+        # Show full length diffs upon test failure
+        self.maxDiff = None  # pylint: disable=invalid-name
+
+        # Create a mock Paver environment
+        tasks.environment = MockEnvironment()
+
+        # Don't run pre-reqs
+        os.environ['NO_PREREQ_INSTALL'] = 'true'
+
+    def tearDown(self):
+        super(PaverTestCase, self).tearDown()
+        tasks.environment = tasks.Environment()
+        del os.environ['NO_PREREQ_INSTALL']
+
+    @property
+    def task_messages(self):
+        """Returns the messages output by the Paver task."""
+        return tasks.environment.messages
+
+    def reset_task_messages(self):
+        """Clear the recorded message"""
+        tasks.environment.messages = []
+
+
+class MockEnvironment(tasks.Environment):
+    """
+    Mock environment that collects information about Paver commands.
+    """
+    def __init__(self):
+        super(MockEnvironment, self).__init__()
+        self.dry_run = True
+        self.messages = []
+
+    def info(self, message, *args):
+        """Capture any messages that have been recorded"""
+        if args:
+            output = message % args
+        else:
+            output = message
+        if not output.startswith("--->"):
+            self.messages.append(output)

pavelib/servers.py

@@ -2,24 +2,36 @@
 Run and manage servers for local development.
 """
 from __future__ import print_function
-import sys
 import argparse
 from paver.easy import *
+
+from .assets import collect_assets
 from .utils.cmd import django_cmd
 from .utils.process import run_process, run_multi_processes
 
 
 DEFAULT_PORT = {"lms": 8000, "studio": 8001}
 DEFAULT_SETTINGS = 'devstack'
+OPTIMIZED_SETTINGS = "devstack_optimized"
+OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized"
 
+ASSET_SETTINGS_HELP = (
+    "Settings file used for updating assets. Defaults to the value of the settings variable if not provided."
+)
 
-def run_server(system, settings=None, port=None, skip_assets=False, contracts=False):
-    """
-    Start the server for the specified `system` (lms or studio).
-    `settings` is the Django settings module to use; if not provided, use the default.
-    `port` is the port to run the server on; if not provided, use the default port for the system.
 
-    If `skip_assets` is True, skip the asset compilation step.
+def run_server(
+        system, fast=False, settings=None, asset_settings=None, port=None, contracts=False
+):
+    """Start the server for LMS or Studio.
+
+    Args:
+        system (str): The system to be run (lms or studio).
+        fast (bool): If true, then start the server immediately without updating assets (defaults to False).
+        settings (str): The Django settings module to use; if not provided, use the default.
+        asset_settings (str) The settings to use when generating assets. If not provided, assets are not generated.
+        port (str): The port number to run the server on. If not provided, uses the default port for the system.
+        contracts (bool) If true then PyContracts is enabled (defaults to False).
     """
     if system not in ['lms', 'studio']:
         print("System must be either lms or studio", file=sys.stderr)
@@ -28,9 +40,13 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa
     if not settings:
         settings = DEFAULT_SETTINGS
 
-    if not skip_assets:
-        # Local dev settings use staticfiles to serve assets, so we can skip the collecstatic step
-        args = [system, '--settings={}'.format(settings), '--skip-collect', '--watch']
+    if not fast and asset_settings:
+        args = [system, '--settings={}'.format(asset_settings), '--watch']
+        # The default settings use DEBUG mode for running the server which means that
+        # the optimized assets are ignored, so we skip collectstatic in that case
+        # to save time.
+        if settings == DEFAULT_SETTINGS:
+            args.append('--skip-collect')
         call_task('pavelib.assets.update_assets', args=args)
 
     if port is None:
@@ -48,40 +64,55 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings"),
+    ("asset-settings=", "a", ASSET_SETTINGS_HELP),
     ("port=", "p", "Port"),
-    ("fast", "f", "Skip updating assets")
+    ("fast", "f", "Skip updating assets"),
 ])
 def lms(options):
     """
     Run the LMS server.
     """
-    settings = getattr(options, 'settings', None)
+    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
+    asset_settings = getattr(options, 'asset-settings', settings)
     port = getattr(options, 'port', None)
     fast = getattr(options, 'fast', False)
-    run_server('lms', settings=settings, port=port, skip_assets=fast)
+    run_server(
+        'lms',
+        fast=fast,
+        settings=settings,
+        asset_settings=asset_settings,
+        port=port,
+    )
 
 
 @task
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings"),
+    ("asset-settings=", "a", ASSET_SETTINGS_HELP),
     ("port=", "p", "Port"),
-    ("fast", "f", "Skip updating assets")
+    ("fast", "f", "Skip updating assets"),
 ])
 def studio(options):
     """
     Run the Studio server.
     """
-    settings = getattr(options, 'settings', None)
+    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
+    asset_settings = getattr(options, 'asset-settings', settings)
     port = getattr(options, 'port', None)
     fast = getattr(options, 'fast', False)
-    run_server('studio', settings=settings, port=port, skip_assets=fast)
+    run_server(
+        'studio',
+        fast=fast,
+        settings=settings,
+        asset_settings=asset_settings,
+        port=port,
+    )
 
 
 @task
 @needs('pavelib.prereqs.install_prereqs')
 @consume_args
-@no_help
 def devstack(args):
     """
     Start the devstack lms or studio server
@@ -89,6 +120,9 @@ def devstack(args):
     parser = argparse.ArgumentParser(prog='paver devstack')
     parser.add_argument('system', type=str, nargs=1, help="lms or studio")
     parser.add_argument('--fast', action='store_true', default=False, help="Skip updating assets")
+    parser.add_argument('--optimized', action='store_true', default=False, help="Run with optimized assets")
+    parser.add_argument('--settings', type=str, default=DEFAULT_SETTINGS, help="Settings file")
+    parser.add_argument('--asset-settings', type=str, default=None, help=ASSET_SETTINGS_HELP)
     parser.add_argument(
         '--no-contracts',
         action='store_true',
@@ -96,7 +130,18 @@ def devstack(args):
         help="Disable contracts. By default, they're enabled in devstack."
     )
     args = parser.parse_args(args)
-    run_server(args.system[0], settings='devstack', skip_assets=args.fast, contracts=(not args.no_contracts))
+    settings = args.settings
+    asset_settings = args.asset_settings if args.asset_settings else settings
+    if args.optimized:
+        settings = OPTIMIZED_SETTINGS
+        asset_settings = OPTIMIZED_ASSETS_SETTINGS
+    run_server(
+        args.system[0],
+        fast=args.fast,
+        settings=settings,
+        asset_settings=asset_settings,
+        contracts=not args.no_contracts,
+    )
 
 
 @task
@@ -116,33 +161,70 @@ def celery(options):
 @needs('pavelib.prereqs.install_prereqs')
 @cmdopts([
     ("settings=", "s", "Django settings for both LMS and Studio"),
+    ("asset_settings=", "a", "Django settings for updating assets for both LMS and Studio (defaults to settings)"),
     ("worker_settings=", "w", "Celery worker Django settings"),
     ("fast", "f", "Skip updating assets"),
+    ("optimized", "o", "Run with optimized assets"),
     ("settings_lms=", "l", "Set LMS only, overriding the value from --settings (if provided)"),
+    ("asset_settings_lms=", "al", "Set LMS only, overriding the value from --asset_settings (if provided)"),
     ("settings_cms=", "c", "Set Studio only, overriding the value from --settings (if provided)"),
+    ("asset_settings_cms=", "ac", "Set Studio only, overriding the value from --asset_settings (if provided)"),
 ])
 def run_all_servers(options):
     """
     Runs Celery workers, Studio, and LMS.
     """
     settings = getattr(options, 'settings', DEFAULT_SETTINGS)
-    settings_lms = getattr(options, 'settings_lms', settings)
-    settings_cms = getattr(options, 'settings_cms', settings)
+    asset_settings = getattr(options, 'asset_settings', settings)
     worker_settings = getattr(options, 'worker_settings', 'dev_with_worker')
     fast = getattr(options, 'fast', False)
+    optimized = getattr(options, 'optimized', False)
+
+    if optimized:
+        settings = OPTIMIZED_SETTINGS
+        asset_settings = OPTIMIZED_ASSETS_SETTINGS
+
+    settings_lms = getattr(options, 'settings_lms', settings)
+    settings_cms = getattr(options, 'settings_cms', settings)
+    asset_settings_lms = getattr(options, 'asset_settings_lms', asset_settings)
+    asset_settings_cms = getattr(options, 'asset_settings_cms', asset_settings)
 
     if not fast:
-        args = ['lms', '--settings={}'.format(settings_lms), '--skip-collect']
+        # First update assets for both LMS and Studio but don't collect static yet
+        args = [
+            'lms', 'studio',
+            '--settings={}'.format(asset_settings),
+            '--skip-collect'
+        ]
         call_task('pavelib.assets.update_assets', args=args)
 
-        args = ['studio', '--settings={}'.format(settings_cms), '--skip-collect']
-        call_task('pavelib.assets.update_assets', args=args)
+        # Now collect static for each system separately with the appropriate settings.
+        # Note that the default settings use DEBUG mode for running the server which
+        # means that the optimized assets are ignored, so we skip collectstatic in that
+        # case to save time.
+        if settings != DEFAULT_SETTINGS:
+            collect_assets(['lms'], asset_settings_lms)
+            collect_assets(['studio'], asset_settings_cms)
 
+        # Install an asset watcher to regenerate files that change
         call_task('pavelib.assets.watch_assets', options={'background': True})
+
+    # Start up LMS, CMS and Celery
+    lms_port = DEFAULT_PORT['lms']
+    cms_port = DEFAULT_PORT['studio']
+    lms_runserver_args = ["0.0.0.0:{}".format(lms_port)]
+    cms_runserver_args = ["0.0.0.0:{}".format(cms_port)]
+
     run_multi_processes([
-        django_cmd('lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['lms'])),
-        django_cmd('studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['studio'])),
-        django_cmd('lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.')
+        django_cmd(
+            'lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', *lms_runserver_args
+        ),
+        django_cmd(
+            'studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', *cms_runserver_args
+        ),
+        django_cmd(
+            'lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.'
+        )
     ])
 
 

pavelib/utils/process.py

@@ -9,6 +9,8 @@ import signal
 import psutil
 import atexit
 
+from paver import tasks
+
 
 def kill_process(proc):
     """
@@ -41,6 +43,14 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None):
         err_log_file = open(err_log, 'w')
         kwargs['stderr'] = err_log_file
 
+    # If the user is performing a dry run of a task, then just log
+    # the command strings and return so that no destructive operations
+    # are performed.
+    if tasks.environment.dry_run:
+        for cmd in cmd_list:
+            tasks.environment.info(cmd)
+        return
+
     try:
         for cmd in cmd_list:
             pids.extend([subprocess.Popen(cmd, **kwargs)])