Fix sys.path manipulation in pavement.py, with detailed comment

Restore reverted changes from paver part 1

AUTHORS

@@ -131,3 +131,4 @@ Toddi Norum <toddi@edx.org>
 Xavier Antoviaque <xavier@antoviaque.org>
 Ali Reza Sharafat <ali.sharafat@gmail.com>
 Avinash Sajjanshetty <avinashsajjan@gmail.com>
+David Glance <david.glance@gmail.com>

cms/djangoapps/contentstore/management/commands/delete_course.py

@@ -6,9 +6,6 @@ from .prompt import query_yes_no
 from contentstore.utils import delete_course_and_groups
 
 
-#
-# To run from command line: rake cms:delete_course LOC=edX/111/Foo1
-#
 class Command(BaseCommand):
     help = '''Delete a MongoDB backed course'''
 

cms/djangoapps/contentstore/management/commands/edit_course_tabs.py

@@ -5,8 +5,6 @@
 #
 # Run it this way:
 #   ./manage.py cms --settings dev edit_course_tabs --course Stanford/CS99/2013_spring
-# Or via rake:
-#   rake django-admin[edit_course_tabs,cms,dev,"--course Stanford/CS99/2013_spring --delete 4"]
 #
 from optparse import make_option
 from django.core.management.base import BaseCommand, CommandError

cms/djangoapps/contentstore/management/commands/xlint.py

 """
 Verify the structure of courseware as to it's suitability for import
-To run test: rake cms:xlint DATA_DIR=../data [COURSE_DIR=content-edx-101 (optional parameter)]
 """
 from django.core.management.base import BaseCommand, CommandError
 from xmodule.modulestore.xml_importer import perform_xlint

cms/envs/bok_choy.py

@@ -21,7 +21,7 @@ from xmodule.x_module import prefer_xmodules
 
 ######################### Testing overrides ####################################
 
-# Needed for the `reset_db` management command
+# Needed for the reset database management command
 INSTALLED_APPS += ('django_extensions',)
 
 # Redirect to the test_root folder within the repo

common/djangoapps/edxmako/management/commands/preprocess_assets.py

@@ -2,7 +2,7 @@
 Preprocess templatized asset files, enabling asset authors to use
 Python/Django inside of Sass and CoffeeScript. This preprocessing
 will happen before the invocation of the asset compiler (currently
-handled by the asset Rakefile).
+handled by the assets paver file).
 
 For this to work, assets need to be named with the appropriate
 template extension (e.g., .mako for Mako templates). Currently Mako

common/djangoapps/student/management/commands/anonymized_id_mapping.py

@@ -19,6 +19,7 @@ from student.models import anonymous_id_for_user
 class Command(BaseCommand):
     """Add our handler to the space where django-admin looks up commands."""
 
+    # TODO: revisit now that rake has been deprecated
     # It appears that with the way Rake invokes these commands, we can't
     # have more than one arg passed through...annoying.
     args = ("course_id", )

docs/en_us/developers/source/index.rst

@@ -33,6 +33,7 @@ Internationalization
 
     i18n.rst
     i18n_translators_guide.rst
+    pavelib.rst
 
 Indices and tables
 ==================

docs/en_us/developers/source/pavelib.rst

+*******************************************
+Paver
+*******************************************
+
+
+Paver provides a standardised way of managing development and operational tasks in edX.
+
+To run individual commands, use the following syntax:
+
+paver <command_name> --option=<option value>
+
+
+Paver Commands
+*******************************************
+
+Paver commands are grouped as follows:
+
+- Prereqs_ Install all of the prerequisite environments for Python, Node and Ruby
+- Docs_ Docs is used to build and then optionally display the EdX docs relating to development, authoring and data management
+- Assets_ Assets will compile Sass (CSS), Coffeescript (Javascript) and XModule assets. Optionally it can call Django’s collectstatic method
+- `Run Servers`_ Run servers
+
+
+.. _Prereqs:
+
+Prereqs
+=============
+
+Install all of the prerequisite for Python, Node and Ruby
+
+   **install_prereqs** : installs Ruby, Node and Python requirements
+
+::
+
+   paver install_prereqs
+
+..
+
+
+.. _Docs:
+
+Docs
+=============
+
+Docs is used to build and then optionally display the EdX docs relating to development, authoring and data management
+
+   **build_docs**:  Invoke sphinx 'make build' to generate docs.
+
+    *--type=* <dev, author, data> Type of docs to compile
+
+    *--verbose* Display verbose output
+
+::
+
+   paver build_docs --type=dev --verbose
+
+..
+
+
+.. _Assets:
+
+Assets
+=============
+
+Assets will compile Sass (CSS), CoffeeScript (Javascript) and XModule assets. Optionally it can call Django's collectstatic command.
+
+
+   **update_assets**: Compiles Coffeescript, Sass, Xmodule and runs collectstatic
+
+    *system* lms or studio
+
+    *--settings=* Django settings e.g. aws, dev
+
+    *--debug* Disable Sass compression
+
+    *--skip-collect* Skip collection of static assets
+
+::
+
+   paver update_assets lms
+
+..
+
+.. _Run Servers:
+
+Run Servers
+=============
+
+    **lms**: runs LMS server
+
+     *--settings=* Django settings e.g. aws, dev
+
+     *--fast*   Skip updating assets
+
+::
+
+   paver lms --settings=dev
+
+..
+
+
+    **studio**: runs Studio
+
+     *--settings=* Django settings e.g. aws, dev
+
+     *--fast*   Skip updating assets
+
+::
+
+   paver studio --settings=dev
+
+..
+
+    **devstack**: runs LMS or Studio (for use within a Vagrant devstack VM)
+
+     *system*   LMS or Studio
+
+     *--fast*   Skip updating assets
+
+::
+
+   paver devstack lms
+
+..
+
+
+    **run_all_servers**: runs lms, cms and celery workers
+
+     *--settings=* Django settings e.g. aws, dev
+
+     *--worker_settings=* Django settings for celery workers
+
+
+::
+
+   paver run_all_servers --settings=dev --worker_settings=celery
+
+..
+
+
+    **run_celery**: runs celery for specified system
+
+     *--settings=* Environment settings e.g. aws, dev
+
+::
+
+   paver celery --settings=dev
+
+..
+
+    **update_db**: runs syncdb and then migrate
+
+     *--settings=* Django settings e.g. aws, dev
+
+::
+
+   paver update_db --settings=dev
+
+..
+
+
+    **check_settings**: checks settings files
+
+     *system*: System to check (lms or studio)
+     *settings*: Django settings to check.
+
+::
+
+   paver check_settings lms aws
+
+..
+

docs/en_us/internal/development.md

@@ -6,6 +6,7 @@
 
 To install all of the libraries needed for our rake commands, run `bundle install`.
 This will read the `Gemfile` and install all of the gems specified there.
+Note: Rake has been deprecated to the Python Paver. This is only needed until rake is fully deprecated
 
 ### Python
 
@@ -46,7 +47,7 @@ After MongoDB daemon is successfully running, check out the course data
 directories that you want to work with into the `GITHUB_REPO_ROOT` (by default,
 `../data`). Then run the following command:
 
-    rake resetdb
+    paver update_db
 
 ## Installing
 
@@ -60,10 +61,10 @@ the repo:
 
 Both the LMS and Studio can be started using the following shortcut tasks
 
-    rake lms  # Start the LMS
-    rake cms  # Start studio
-    rake lms[cms.dev]  # Start LMS to run alongside Studio
-    rake lms[cms.dev_preview]  # Start LMS to run alongside Studio in preview mode
+    paver lms  # Start the LMS
+    paver studio  # Start studio
+    paver lms --settings=cms.dev  # Start LMS to run alongside Studio
+    paver lms --settings=cms.dev_preview  # Start LMS to run alongside Studio in preview mode
 
 Under the hood, this executes `./manage.py {lms|cms} --settings $ENV runserver`,
 which starts a local development server.
@@ -72,13 +73,12 @@ Both of these commands take arguments to start the servers in different environm
 or with additional options:
 
     # Start the LMS using the test configuration, on port 5000
-    rake lms[test,5000]  # Executes ./manage.py lms --settings test runserver 5000
+    paver lms --settings=test --port=5000  # Executes ./manage.py lms --settings test runserver 5000
 
-*N.B.* You may have to escape the `[` characters, depending on your shell: `rake "lms[test,5000]"`
+To get a full list of available paver tasks, run:
 
-To get a full list of available rake tasks, use:
+     paver --help
 
-    rake -T
 
 ### Troubleshooting
 

docs/en_us/internal/discussion.md

@@ -74,7 +74,7 @@ To fully test the discussion forum, you might want to act as a moderator or an a
 
 First make sure that the database is up-to-date:
 
-    rake resetdb
+    paver update_db
 
 If you have created users in the edx-platform django apps when the comment service was not running, you will need to one-way sync the users into the comment service back end database:
 

lms/envs/bok_choy.py

@@ -24,7 +24,7 @@ from .aws import *  # pylint: disable=W0401, W0614
 
 ######################### Testing overrides ####################################
 
-# Needed for the `reset_db` management command
+# Needed for the reset database management command
 INSTALLED_APPS += ('django_extensions',)
 
 # Redirect to the test_root folder within the repo

lms/static/coffee/README.md

@@ -34,10 +34,9 @@ Conveniently, you can install Node via `apt-get`, then use npm:
 Compiling
 ---------
 
-The dev server will automatically compile coffeescript files that have changed.
-Simply start the server using:
-    
-    $ rake runserver
+CoffeeScript is compiled when you update assets using the command:
+
+    $ paver update_assets
 
 Testing
 -------

pavelib/__init__.py

+__all__ = ["assets", "servers", "docs", "prereqs"]

pavelib/assets.py

+"""
+Asset compilation and collection.
+"""
+import argparse
+from paver.easy import *
+from .utils.envs import Env
+from .utils.cmd import cmd, django_cmd
+
+
+COFFEE_DIRS = ['lms', 'cms', 'common']
+SASS_LOAD_PATHS = ['./common/static/sass']
+SASS_UPDATE_DIRS = ['*/static']
+SASS_CACHE_PATH = '/tmp/sass-cache'
+
+
+def theme_sass_paths():
+    """
+    Return the a list of paths to the theme's sass assets,
+    or an empty list if no theme is configured.
+    """
+    edxapp_env = Env()
+
+    if edxapp_env.feature_flags.get('USE_CUSTOM_THEME', False):
+        theme_name = edxapp_env.env_tokens.get('THEME_NAME', '')
+        theme_root = path(edxapp_env.REPO_ROOT).dirname() / "themes" / theme_name
+        return [theme_root / "static" / "sass"]
+
+    else:
+        return []
+
+
+def compile_coffeescript():
+    """
+    Compile CoffeeScript to JavaScript.
+    """
+    dirs = " ".join([Env.REPO_ROOT / coffee_dir for coffee_dir in COFFEE_DIRS])
+    sh(cmd(
+        "node_modules/.bin/coffee", "--compile",
+        " `find {dirs} -type f -name \"*.coffee\"`".format(dirs=dirs)
+    ))
+
+
+def compile_sass(debug):
+    """
+    Compile Sass to CSS.
+    """
+    theme_paths = theme_sass_paths()
+    sh(cmd(
+        'sass', '' if debug else '--style compressed',
+        "--cache-location {cache}".format(cache=SASS_CACHE_PATH),
+        "--load-path", " ".join(SASS_LOAD_PATHS + theme_paths),
+        "--update", "-E", "utf-8", " ".join(SASS_UPDATE_DIRS + theme_paths)
+    ))
+
+
+def compile_templated_sass(systems, settings):
+    """
+    Render Mako templates for Sass files.
+    `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
+    `settings` is the Django settings module to use.
+    """
+    for sys in systems:
+        sh(django_cmd(sys, settings, 'preprocess_assets'))
+
+
+def process_xmodule_assets():
+    """
+    Process XModule static assets.
+    """
+    sh('xmodule_assets common/static/xmodule')
+
+
+def collect_assets(systems, settings):
+    """
+    Collect static assets, including Django pipeline processing.
+    `systems` is a list of systems (e.g. 'lms' or 'studio' or both)
+    `settings` is the Django settings module to use.
+    """
+    for sys in systems:
+        sh(django_cmd(sys, settings, "collectstatic --noinput > /dev/null"))
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@consume_args
+def update_assets(args):
+    """
+    Compile CoffeeScript and Sass, then collect static assets.
+    """
+    parser = argparse.ArgumentParser(prog='paver update_assets')
+    parser.add_argument('system', type=str, nargs='*', default=['lms', 'studio'], help="lms or studio")
+    parser.add_argument('--settings', type=str, default="dev", help="Django settings module")
+    parser.add_argument('--debug', action='store_true', default=False, help="Disable Sass compression")
+    parser.add_argument('--skip-collect', action='store_true', default=False, help="Skip collection of static assets")
+    args = parser.parse_args(args)
+
+    compile_templated_sass(args.system, args.settings)
+    process_xmodule_assets()
+    compile_coffeescript()
+    compile_sass(args.debug)
+
+    if not args.skip_collect:
+        collect_assets(args.system, args.settings)

pavelib/docs.py

+import sys
+from paver.easy import *
+
+
+DOC_PATHS = {
+    "dev": "docs/en_us/developers",
+    "author": "docs/en_us/course_authors",
+    "data": "docs/en_us/data",
+    "default": "docs/en_us"
+}
+
+
+def valid_doc_types():
+    """
+    Return a comma-separated string of valid doc types.
+    """
+    return ", ".join(DOC_PATHS.keys())
+
+
+def doc_path(options, allow_default=True):
+    """
+    Parse `options` (from the Paver task args) to determine the path
+    to the documentation directory.
+    If the specified path is not one of the valid options, print an error
+    message and exit.
+
+    If `allow_default` is False, then require that a type is specified,
+    and exit with an error message if it isn't.
+    """
+    doc_type = getattr(options, 'type', 'default')
+    path = DOC_PATHS.get(doc_type)
+
+    if doc_type == 'default' and not allow_default:
+        print "You must specify a documentation type using '--type'.  Valid options are: {options}".format(
+            options=valid_doc_types())
+        sys.exit(1)
+
+    if path is None:
+        print "Invalid documentation type '{doc_type}'.  Valid options are: {options}".format(
+            doc_type=doc_type, options=valid_doc_types())
+        sys.exit(1)
+
+    else:
+        return path
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("type=", "t", "Type of docs to compile"),
+    ("verbose", "v", "Display verbose output"),
+])
+def build_docs(options):
+    """
+    Invoke sphinx 'make build' to generate docs.
+    """
+    verbose = getattr(options, 'verbose', False)
+
+    cmd = "cd {dir}; make html quiet={quiet}".format(
+        dir=doc_path(options),
+        quiet="false" if verbose else "true"
+    )
+
+    sh(cmd)

pavelib/prereqs.py

+"""
+Install Python, Ruby, and Node prerequisites.
+"""
+
+import os
+import hashlib
+from distutils import sysconfig
+from paver.easy import *
+from .utils.envs import Env
+
+
+PREREQS_MD5_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
+NPM_REGISTRY = "http://registry.npmjs.org/"
+PYTHON_REQ_FILES = [
+    'requirements/edx/pre.txt',
+    'requirements/edx/base.txt',
+    'requirements/edx/post.txt'
+]
+
+
+def read_in_chunks(infile, chunk_size=1024 * 64):
+    """
+    Yield a chunk of size `chunksize` from `infile` (a file handle).
+    """
+    chunk = infile.read(chunk_size)
+    while chunk:
+        yield chunk
+        chunk = infile.read(chunk_size)
+
+
+def compute_fingerprint(path_list):
+    """
+    Hash the contents of all the files and directories in `path_list`.
+    Returns the hex digest.
+    """
+
+    hasher = hashlib.sha1()
+
+    for path in path_list:
+
+        # For directories, create a hash based on the filenames in the directory
+        if os.path.isdir(path):
+            for _, _, filenames in os.walk(path):
+                for name in filenames:
+                    hasher.update(name)
+
+        # For files, hash the contents of the file
+        if os.path.isfile(path):
+            with open(path, "rb") as file_handle:
+                for chunk in read_in_chunks(file_handle):
+                    hasher.update(chunk)
+
+    return hasher.hexdigest()
+
+
+def prereq_cache(cache_name, paths, install_func):
+    """
+    Conditionally execute `install_func()` only if the files/directories
+    specified by `paths` have changed.
+
+    If the code executes successfully (no exceptions are thrown), the cache
+    is updated with the new hash.
+    """
+    # Retrieve the old hash
+    cache_filename = cache_name.replace(" ", "_")
+    cache_file_path = os.path.join(PREREQS_MD5_DIR, "{}.sha1".format(cache_filename))
+    old_hash = None
+    if os.path.isfile(cache_file_path):
+        with open(cache_file_path) as cache_file:
+            old_hash = cache_file.read()
+
+    # Compare the old hash to the new hash
+    # If they do not match (either the cache hasn't been created, or the files have changed),
+    # then execute the code within the block.
+    new_hash = compute_fingerprint(paths)
+    if new_hash != old_hash:
+        install_func()
+
+        # Update the cache with the new hash
+        # If the code executed within the context fails (throws an exception),
+        # then this step won't get executed.
+        try:
+            os.makedirs(PREREQS_MD5_DIR)
+        except OSError:
+            if not os.path.isdir(PREREQS_MD5_DIR):
+                raise
+
+        with open(cache_file_path, "w") as cache_file:
+            cache_file.write(new_hash)
+
+    else:
+        print '{cache} unchanged, skipping...'.format(cache=cache_name)
+
+
+def install_ruby_prereqs():
+    """
+    Installs Ruby prereqs
+    """
+    sh('bundle install --quiet')
+
+
+def install_node_prereqs():
+    """
+    Installs Node prerequisites
+    """
+    sh("npm config set registry {}".format(NPM_REGISTRY))
+    sh('npm install')
+
+
+def install_python_prereqs():
+    """
+    Installs Python prerequisites
+    """
+    for req_file in PYTHON_REQ_FILES:
+        sh("pip install -q --exists-action w -r {req_file}".format(req_file=req_file))
+
+
+@task
+def install_prereqs():
+    """
+    Installs Ruby, Node and Python prerequisites
+    """
+    prereq_cache("Ruby prereqs", ["Gemfile"], install_ruby_prereqs)
+    prereq_cache("Node prereqs", ["package.json"], install_node_prereqs)
+    prereq_cache("Python prereqs", PYTHON_REQ_FILES + [sysconfig.get_python_lib()], install_python_prereqs)

pavelib/servers.py

+"""
+Run and manage servers for local development.
+"""
+
+import argparse
+from paver.easy import *
+from .utils.cmd import django_cmd
+from .utils.process import write_stderr, run_process, run_multi_processes
+
+
+DEFAULT_PORT = {"lms": 8000, "studio": 8001}
+DEFAULT_SETTINGS = 'dev'
+
+
+def run_server(system, settings=None, port=None, skip_assets=False):
+    """
+    Start the server for the specified `system` (lms or studio).
+    `settings` is the Django settings module to use; if not provided, use the default.
+    `port` is the port to run the server on; if not provided, use the default port for the system.
+
+    If `skip_assets` is True, skip the asset compilation step.
+    """
+    if system not in ['lms', 'studio']:
+        print "System must be either lms or studio"
+        exit(1)
+
+    if not skip_assets:
+        # Local dev settings use staticfiles to serve assets, so we can skip the collecstatic step
+        args = [system, '--settings={}'.format(settings), '--skip-collect']
+        call_task('pavelib.assets.update_assets', args=args)
+
+    if port is None:
+        port = DEFAULT_PORT[system]
+
+    if settings is None:
+        settings = DEFAULT_SETTINGS
+
+    run_process(django_cmd(
+        system, settings, 'runserver', '--traceback',
+        '--pythonpath=.', '0.0.0.0:{}'.format(port)))
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("settings=", "s", "Django settings"),
+    ("port=", "p", "Port"),
+    ("fast", "f", "Skip updating assets")
+])
+def lms(options):
+    """
+    Run the LMS server.
+    """
+    settings = getattr(options, 'settings', None)
+    port = getattr(options, 'port', None)
+    fast = getattr(options, 'fast', False)
+    run_server('lms', settings=settings, port=port, skip_assets=fast)
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("settings=", "s", "Django settings"),
+    ("port=", "p", "Port"),
+    ("fast", "f", "Skip updating assets")
+])
+def studio(options):
+    """
+    Run the Studio server.
+    """
+    settings = getattr(options, 'settings', None)
+    port = getattr(options, 'port', None)
+    fast = getattr(options, 'fast', False)
+    run_server('studio', settings=settings, port=port, skip_assets=fast)
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@consume_args
+def devstack(args):
+    """
+    Start the devstack lms or studio server
+    """
+    parser = argparse.ArgumentParser(prog='paver devstack')
+    parser.add_argument('system', type=str, nargs=1, help="lms or studio")
+    parser.add_argument('--fast', action='store_true', default=False, help="Skip updating assets")
+    args = parser.parse_args(args)
+    run_server(args.system[0], settings='devstack', skip_assets=args.fast)
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("settings=", "s", "Django settings"),
+])
+def celery(options):
+    """
+    Runs Celery workers.
+    """
+    settings = getattr(options, 'settings', 'dev_with_worker')
+    run_process(django_cmd('lms', settings, 'celery', 'worker', '--loglevel=INFO', '--pythonpath=.'))
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("settings=", "s", "Django settings"),
+    ("worker_settings=", "w", "Celery worker Django settings"),
+    ("fast", "f", "Skip updating assets")
+])
+def run_all_servers(options):
+    """
+    Runs Celery workers, Studio, and LMS.
+    """
+    settings = getattr(options, 'settings', 'dev')
+    worker_settings = getattr(options, 'worker_settings', 'dev_with_worker')
+    fast = getattr(options, 'fast', False)
+
+    if not fast:
+        for system in ['lms', 'studio']:
+            args = [system, '--settings={}'.format(settings), '--skip-collect']
+            call_task('pavelib.assets.update_assets', args=args)
+
+    run_multi_processes([
+        django_cmd('lms', settings, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['lms'])),
+        django_cmd('studio', settings, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['studio'])),
+        django_cmd('lms', worker_settings, 'celery', 'worker', '--loglevel=INFO', '--pythonpath=.')
+    ])
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("settings=", "s", "Django settings"),
+])
+def update_db():
+    """
+    Runs syncdb and then migrate.
+    """
+    settings = getattr(options, 'settings', 'dev')
+    sh(django_cmd('lms', settings, 'syncdb', '--traceback', '--pythonpath=.'))
+    sh(django_cmd('lms', settings, 'migrate', '--traceback', '--pythonpath=.'))
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@consume_args
+def check_settings(args):
+    """
+    Checks settings files.
+    """
+    parser = argparse.ArgumentParser(prog='paver check_settings')
+    parser.add_argument('system', type=str, nargs=1, help="lms or studio")
+    parser.add_argument('settings', type=str, nargs=1, help='Django settings')
+    args = parser.parse_args(args)
+
+    system = args.system[0]
+    settings = args.settings[0]
+
+    try:
+        import_cmd = "echo 'import {system}.envs.{settings}'".format(system=system, settings=settings)
+        django_shell_cmd = django_cmd(system, settings, 'shell', '--plain', '--pythonpath=.')
+        sh("{import_cmd} | {shell_cmd}".format(import_cmd=import_cmd, shell_cmd=django_shell_cmd))
+
+    except:
+        write_stderr("Failed to import settings\n")

pavelib/utils/cmd.py

+"""
+Helper functions for constructing shell commands.
+"""
+
+def cmd(*args):
+    """
+    Concatenate the arguments into a space-separated shell command.
+    """
+    return " ".join([str(arg) for arg in args])
+
+
+def django_cmd(sys, settings, *args):
+    """
+    Construct a Django management command.
+
+    `sys` is either 'lms' or 'studio'.
+    `settings` is the Django settings module (such as "dev" or "test")
+    `args` are concatenated to form the rest of the command.
+    """
+    # Maintain backwards compatibility with manage.py,
+    # which calls "studio" "cms"
+    sys = 'cms' if sys == 'studio' else sys
+    return cmd("python manage.py", sys, "--settings={}".format(settings), *args)

pavelib/utils/envs.py

+"""
+Helper functions for loading environment settings.
+"""
+import os
+import sys
+import json
+from lazy import lazy
+from path import path
+
+
+class Env(object):
+    """
+    Load information about the execution environment.
+    """
+
+    # Root of the git repository (edx-platform)
+    REPO_ROOT = path(__file__).dirname().dirname().dirname()
+
+    # Service variant (lms, cms, etc.) configured with an environment variable
+    # We use this to determine which envs.json file to load.
+    SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
+
+    @lazy
+    def env_tokens(self):
+        """
+        Return a dict of environment settings.
+        If we couldn't find the JSON file, issue a warning and return an empty dict.
+        """
+
+        # Find the env JSON file
+        env_path = "env.json"
+        if self.SERVICE_VARIANT is not None:
+            env_path = self.REPO_ROOT.dirname() / "{service}.env.json".format(service=self.SERVICE_VARIANT)
+
+        # If the file does not exist, issue a warning and return an empty dict
+        if not os.path.isfile(env_path):
+            print "Warning: could not find environment JSON file at '{path}'".format(path=env_path)
+            return dict()
+
+        # Otherwise, load the file as JSON and return the resulting dict
+        try:
+            with open(env_path) as env_file:
+                return json.load(env_file)
+
+        except ValueError:
+            print "Error: Could not parse JSON in {path}".format(path=env_path)
+            sys.exit(1)
+
+    @lazy
+    def feature_flags(self):
+        """
+        Return a dictionary of feature flags configured by the environment.
+        """
+        return self.env_tokens.get('FEATURES', dict())

pavelib/utils/process.py

+"""
+Helper functions for managing processes.
+"""
+
+import sys
+import os
+import subprocess
+import signal
+import psutil
+
+
+def write_stderr(message):
+    """
+    Print a `message` str to stderr.
+    """
+    sys.stderr.write(message)
+    sys.stderr.flush()
+
+
+def kill_process(proc):
+    """
+    Kill the process `proc` created with `subprocess`.
+    """
+    p1_group = psutil.Process(proc.pid)
+
+    child_pids = p1_group.get_children(recursive=True)
+
+    for child_pid in child_pids:
+        os.kill(child_pid.pid, signal.SIGKILL)
+
+
+def run_multi_processes(cmd_list, out_log=None, err_log=None):
+    """
+    Run each shell command in `cmd_list` in a separate process,
+    piping stdout to `out_log` (a path) and stderr to `err_log` (also a path).
+
+    Terminates the processes on CTRL-C and ensures the processes are killed
+    if an error occurs.
+    """
+    kwargs = {'shell': True, 'cwd': None}
+    pids = []
+
+    if out_log:
+        out_log_file = open(out_log, 'w')
+        kwargs['stdout'] = out_log_file
+
+    if err_log:
+        err_log_file = open(err_log, 'w')
+        kwargs['stderr'] = err_log_file
+
+    try:
+        for cmd in cmd_list:
+            pids.extend([subprocess.Popen(cmd, **kwargs)])
+
+        def _signal_handler(*args):
+            print("\nEnding...")
+
+        signal.signal(signal.SIGINT, _signal_handler)
+        print("Enter CTL-C to end")
+        signal.pause()
+        print("Processes ending")
+
+    except Exception as err:
+        write_stderr("Error running process {}\n".format(err))
+
+    finally:
+        for pid in pids:
+            kill_process(pid)
+
+
+def run_process(cmd, out_log=None, err_log=None):
+    """
+    Run the shell command `cmd` in a separate process,
+    piping stdout to `out_log` (a path) and stderr to `err_log` (also a path).
+
+    Terminates the process on CTRL-C or if an error occurs.
+    """
+    return run_multi_processes([cmd], out_log=out_log, err_log=err_log)

pavement.py

+import sys
+import os
+
+# Ensure that we can import pavelib, and that our copy of pavelib
+# takes precedence over anything else installed in the virtualenv.
+# In local dev, we usually don't need to do this, because Python
+# automatically puts the current working directory on the system path.
+# In Jenkins, however, we have multiple copies of the edx-platform repo,
+# each of which run "pip install -e ." (as part of requirements/edx/local.txt)
+# Until we re-run pip install, the other copies of edx-platform could
+# take precedence, leading to some very strange results.
+sys.path.insert(1, os.path.dirname(__file__))
+
+from pavelib import *

rakelib/assets.rake

-# Theming constants
-USE_CUSTOM_THEME = ENV_TOKENS.has_key?('FEATURES') && ENV_TOKENS['FEATURES']['USE_CUSTOM_THEME']
-if USE_CUSTOM_THEME
-    THEME_NAME = ENV_TOKENS['THEME_NAME']
-    THEME_ROOT = File.join(ENV_ROOT, "themes", THEME_NAME)
-    THEME_SASS = File.join(THEME_ROOT, "static", "sass")
-end
-
-MINIMAL_DARWIN_NOFILE_LIMIT = 8000
-
-def xmodule_cmd(watch=false, debug=false)
-    xmodule_cmd = 'xmodule_assets common/static/xmodule'
-    if watch
-        "watchmedo shell-command " +
-                  "--patterns='*.js;*.coffee;*.sass;*.scss;*.css' " +
-                  "--recursive " +
-                  "--command='#{xmodule_cmd}' " +
-                  "--wait " +
-                  "common/lib/xmodule"
-    else
-        xmodule_cmd
-    end
-end
-
-def coffee_cmd(watch=false, debug=false)
-    if watch && Launchy::Application.new.host_os_family.darwin?
-        available_files = Process::getrlimit(:NOFILE)[0]
-        if available_files < MINIMAL_DARWIN_NOFILE_LIMIT
-            Process.setrlimit(:NOFILE, MINIMAL_DARWIN_NOFILE_LIMIT)
-
-        end
-    end
-    if watch
-        "node_modules/.bin/coffee --compile --watch lms/ cms/ common/"
-    else
-        "node_modules/.bin/coffee --compile `find lms/ cms/ common/ -type f -name *.coffee` "
-    end
-end
-
-def sass_cmd(watch=false, debug=false)
-    sass_load_paths = ["./common/static/sass"]
-    sass_watch_paths = ["*/static"]
-    if USE_CUSTOM_THEME
-      sass_load_paths << THEME_SASS
-      sass_watch_paths << THEME_SASS
-    end
-
-    "sass #{debug ? '' : '--style compressed'} " +
-          "--cache-location /tmp/sass-cache " +
-          "--load-path #{sass_load_paths.join(' ')} " +
-          "#{watch ? '--watch' : '--update'} -E utf-8 #{sass_watch_paths.join(' ')}"
-end
-
-# This task takes arguments purely to pass them via dependencies to the preprocess task
-desc "Compile all assets"
-task :assets, [:system, :env] => 'assets:all'
-
-namespace :assets do
-
-    desc "Compile all assets in debug mode"
-    multitask :debug
-
-    desc "Preprocess all templatized static asset files"
-    task :preprocess, [:system, :env] do |t, args|
-      args.with_defaults(:system => "lms", :env => "dev")
-      sh(django_admin(args.system, args.env, "preprocess_assets")) do |ok, status|
-        abort "asset preprocessing failed!" if !ok
-      end
-    end
-
-    desc "Watch all assets for changes and automatically recompile"
-    task :watch => 'assets:_watch' do
-        puts "Press ENTER to terminate".red
-        $stdin.gets
-    end
-
-    {:xmodule => [:install_python_prereqs],
-     :coffee => [:install_node_prereqs, :'assets:coffee:clobber'],
-     :sass => [:install_ruby_prereqs, :preprocess]}.each_pair do |asset_type, prereq_tasks|
-        # This task takes arguments purely to pass them via dependencies to the preprocess task
-        desc "Compile all #{asset_type} assets"
-        task asset_type, [:system, :env] => prereq_tasks do |t, args|
-            cmd = send(asset_type.to_s + "_cmd", watch=false, debug=false)
-            if cmd.kind_of?(Array)
-                cmd.each {|c| sh(c)}
-            else
-                sh(cmd)
-            end
-        end
-
-        # This task takes arguments purely to pass them via dependencies to the preprocess task
-        multitask :all, [:system, :env] => asset_type
-        multitask :debug => "assets:#{asset_type}:debug"
-        multitask :_watch => "assets:#{asset_type}:_watch"
-
-        namespace asset_type do
-            desc "Compile all #{asset_type} assets in debug mode"
-            task :debug => prereq_tasks do
-                cmd = send(asset_type.to_s + "_cmd", watch=false, debug=true)
-                sh(cmd)
-            end
-
-            desc "Watch all #{asset_type} assets and compile on change"
-            task :watch => "assets:#{asset_type}:_watch" do
-                puts "Press ENTER to terminate".red
-                $stdin.gets
-            end
-
-            # Fully compile before watching for changes
-            task :_watch => (prereq_tasks + ["assets:#{asset_type}:debug"]) do
-                cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true)
-                if cmd.kind_of?(Array)
-                    cmd.each {|c| singleton_process(c)}
-                else
-                    singleton_process(cmd)
-                end
-            end
-        end
-    end
-
-    multitask :sass => 'assets:xmodule'
-    namespace :sass do
-        multitask :debug => 'assets:xmodule:debug'
-    end
-
-    multitask :coffee => 'assets:xmodule'
-    namespace :coffee do
-        multitask :debug => 'assets:xmodule:debug'
-
-        desc "Remove compiled coffeescript files"
-        task :clobber do
-            FileList['*/static/coffee/**/*.js'].each {|f| File.delete(f)}
-        end
-    end
-
-    namespace :xmodule do
-        # Only start the xmodule watcher after the coffee and sass watchers have already started
-        task :_watch => ['assets:coffee:_watch', 'assets:sass:_watch']
-    end
-end
-
-# This task does the real heavy lifting to gather all of the static
-# assets. We want people to call it via the wrapper below, so we
-# don't provide a description so that it won't show up in rake -T.
-task :gather_assets, [:system, :env] => :assets do |t, args|
-    sh("#{django_admin(args.system, args.env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
-        if !ok
-            abort "collectstatic failed!"
-        end
-    end
-end
-
-[:lms, :cms].each do |system|
-    # Per environment tasks
-    environments(system).each do |env|
-        # This task wraps the one above, since we need the system and
-        # env arguments to be passed to all dependent tasks.
-        desc "Compile coffeescript and sass, and then run collectstatic in the specified environment"
-        task "#{system}:gather_assets:#{env}" do
-          Rake::Task[:gather_assets].invoke(system, env)
-        end
-    end
-end

rakelib/assets_deprecated.rake

+# assets tasks deprecated to paver
+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by,  *args)
+
+    task deprecated, [:system, :env] do |t,args|
+
+        # Need to install paver dependencies for the commands to work!
+        sh("pip install Paver==1.2.1 psutil==1.2.1 lazy==1.1 path.py==3.0.1")
+
+        args.with_defaults(:system => "lms", :env => "dev")
+
+        if deprecated_by.nil?
+            puts("Task #{deprecated} has been deprecated.".red)
+
+        else
+            if deprecated.include? "gather_assets"
+                new_cmd = deprecated_by
+            else
+                new_cmd = deprecated_by + " #{args.system} --settings=#{args.env}"
+            end
+
+            puts("Task #{deprecated} has been deprecated. Use #{new_cmd} instead.".red)
+            sh(new_cmd)
+        end
+    end
+end
+
+deprecated("assets:coffee", "paver update_assets")
+deprecated("assets:coffee:clobber", nil)
+deprecated("assets:coffee:debug", "paver update_assets --debug")
+deprecated("assets:coffee:watch", "paver update_assets")
+
+deprecated("assets:sass", "paver update_assets")
+deprecated("assets:sass:debug", "paver update_assets --debug")
+deprecated("assets:sass:watch", "paver update_assets")
+
+deprecated("assets:xmodule", "paver update_assets")
+deprecated("assets:xmodule:debug", "paver update_assets --debug")
+deprecated("assets:xmodule:watch", "paver update_assets")
+
+deprecated("assets:debug", "paver update_assets --debug")
+deprecated("assets:watch", "paver update_assets")
+
+deprecated("assets", "paver update_assets")
+
+[:lms, :cms].each do |system|
+
+    deprecated("#{system}:gather_assets", "paver update_assets #{system}")
+    environments(system).each do |env|
+      deprecated("#{system}:gather_assets:#{env}", "paver update_assets #{system} --settings=#{env}")
+    end
+end
+

rakelib/bok_choy.rake

@@ -182,9 +182,7 @@ namespace :'test:bok_choy' do
         sh("#{REPO_ROOT}/scripts/reset-test-db.sh")
 
         # Collect static assets
-        Rake::Task["gather_assets"].invoke('lms', 'bok_choy')
-        Rake::Task["gather_assets"].reenable
-        Rake::Task["gather_assets"].invoke('cms', 'bok_choy')
+        sh("paver update_assets --settings=bok_choy")
     end
 
     desc "Run acceptance tests that use the bok-choy framework but skip setup"

rakelib/devstack.rake

-DEVSTACK_PORTS = {
-    "lms" => '8000',
-    "studio" => '8001'
-}
-
-# Abort if system is not one we recognize
-def check_devstack_sys(sys_name)
-    if not DEVSTACK_PORTS.has_key?(sys_name)
-        puts "Devstack system must be either 'lms' or 'studio'"
-        exit 1
-    end
-end
-
-# Convert "studio" to "cms"
-def old_system(sys_name)
-    if sys_name == "studio"
-        return "cms"
-    else
-        return sys_name
-    end
-end
-
-namespace :devstack do
-
-    desc "Start the server"
-    task :start, [:system] do |t, args|
-        check_devstack_sys(args.system)
-        port = DEVSTACK_PORTS[args.system]
-        sys = old_system(args.system)
-        sh("./manage.py #{sys} runserver --settings=devstack 0.0.0.0:#{port}")
-    end
-
-    desc "Update static assets"
-    task :assets, [:system] do |t, args|
-        check_devstack_sys(args.system)
-        Rake::Task["assets"].invoke(old_system(args.system), 'devstack')
-    end
-
-    desc "Update Python, Ruby, and Node requirements"
-    task :install => [:install_prereqs]
-end
-
-
-desc "Start the devstack lms or studio server"
-task :devstack, [:system] => ['devstack:install', 'devstack:assets'] do |t, args|
-    Rake::Task['devstack:start'].invoke(args.system)
-end

rakelib/devstack_deprecated.rake

+# devstack tasks deprecated to paver
+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by,  *args)
+
+    task deprecated, [:system] do |t,args|
+
+        # Need to install paver dependencies for the commands to work!
+        sh("pip install Paver==1.2.1 psutil==1.2.1 lazy==1.1 path.py==3.0.1")
+
+        args.with_defaults(:system => 'lms')
+        deprecated_by = "#{deprecated_by} #{args.system}"
+
+        puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead. Waiting 5 seconds...".red)
+        sleep(5)
+        sh(deprecated_by)
+        exit
+    end
+end
+
+deprecated("devstack:start", "paver devstack --fast")
+deprecated("devstack:assets", "paver update_assets --settings=devstack")
+deprecated("devstack:install", "paver install_prereqs")
+deprecated("devstack", "paver devstack")

rakelib/django.rake

-default_options = {
-    :lms => '8000',
-    :cms => '8001',
-}
-
-task :predjango => :install_python_prereqs do
-    sh("find . -type f -name *.pyc -delete")
-    sh('pip install -q --no-index -r requirements/edx/local.txt')
-end
-
-
-task :fastlms do
-    # this is >2 times faster that rake [lms], and does not need web, good for local dev
-    sh("./manage.py lms runserver --traceback")
-end
-
-# Start :system locally with the specified :env and :options.
-#
-# This task should be invoked via the wrapper below, so we don't
-# include a description to keep it from showing up in rake -T.
-task :runserver, [:system, :env, :options] => [:install_prereqs, 'assets:_watch', :predjango] do |t, args|
-    sh(django_admin(args.system, args.env, 'runserver', args.options))
-end
-
-[:lms, :cms].each do |system|
-    desc <<-desc
-        Start the #{system} locally with the specified environment (defaults to dev).
-        Other useful environments are devplus (for dev testing with a real local database)
-        desc
-    task system, [:env, :options] do |t, args|
-        args.with_defaults(:env => 'dev', :options => default_options[system])
-        Rake::Task[:runserver].invoke(system, args.env, args.options)
-    end
-
-    desc "Start #{system} Celery worker"
-    task "#{system}_worker", [:options] => [:predjango] do |t, args|
-      args.with_defaults(:options => default_options[system])
-      command = 'celery worker'
-      sh("./manage.py #{system} --settings dev_with_worker #{command} --loglevel=INFO #{args.join(' ')}")
-    end
-
-    # Per environment tasks
-    environments(system).each do |env|
-        desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
-        task "#{system}:check_settings:#{env}" => :predjango do
-            sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}")
-        end
-    end
-end
-
-desc "Reset the relational database used by django. WARNING: this will delete all of your existing users"
-task :resetdb, [:env] do |t, args|
-    args.with_defaults(:env => 'dev')
-    sh(django_admin(:lms, args.env, 'syncdb'))
-    sh(django_admin(:lms, args.env, 'migrate'))
-end
-
-task :runserver => :lms
-
-desc "Run django-admin <action> against the specified system and environment"
-task "django-admin", [:action, :system, :env, :options] do |t, args|
-    # If no system was explicitly set, we want to run both CMS and LMS for migrate and syncdb.
-    no_system_set = !args.system
-    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
-    sh(django_admin(args.system, args.env, args.action, args.options))
-    if no_system_set and (args.action == 'migrate' or args.action == 'syncdb')
-      sh(django_admin('cms', args.env, args.action, args.options))
-    end
-end
-
-desc "Set the staff bit for a user"
-task :set_staff, [:user, :system, :env] do |t, args|
-    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
-    sh(django_admin(args.system, args.env, 'set_staff', args.user))
-end
-
-namespace :cms do
-  desc "Clone existing MongoDB based course"
-  task :clone do
-
-    if ENV['SOURCE_LOC'] and ENV['DEST_LOC']
-      sh(django_admin(:cms, :dev, :clone, ENV['SOURCE_LOC'], ENV['DEST_LOC']))
-    else
-      raise "You must pass in a SOURCE_LOC and DEST_LOC parameters"
-    end
-  end
-
-  desc "Delete existing MongoDB based course"
-  task :delete_course do
-
-    if ENV['LOC'] and ENV['COMMIT']
-        sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'], ENV['COMMIT']))
-    elsif ENV['LOC']
-      sh(django_admin(:cms, :dev, :delete_course, ENV['LOC']))
-    else
-      raise "You must pass in a LOC parameter"
-    end
-  end
-
-  desc "Import course data within the given DATA_DIR variable"
-  task :import do
-    if ENV['DATA_DIR'] and ENV['COURSE_DIR']
-      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'], ENV['COURSE_DIR']))
-    elsif ENV['DATA_DIR']
-      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR']))
-    else
-      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
-        "Example: \`rake cms:import DATA_DIR=../data\`"
-    end
-  end
-
-  desc "Import course data within the given DATA_DIR variable"
-  task :xlint do
-    if ENV['DATA_DIR'] and ENV['COURSE_DIR']
-      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR']))
-    elsif ENV['DATA_DIR']
-      sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR']))
-    else
-      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
-        "Example: \`rake cms:import DATA_DIR=../data\`"
-    end
-  end
-
-  desc "Export course data to a tar.gz file"
-  task :export do
-    if ENV['COURSE_ID'] and ENV['OUTPUT_PATH']
-      sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH']))
-    else
-      raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" +
-        "Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`"
-    end
-  end
-end

rakelib/django_deprecated.rake

+# django assets tasks deprecated to paver
+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by)
+
+    task deprecated, [:arg1, :arg2, :arg3, :arg4] do |t,args|
+
+        # Need to install paver dependencies for the commands to work!
+        sh("pip install Paver==1.2.1 psutil==1.2.1 lazy==1.1 path.py==3.0.1")
+
+        if deprecated == "cms" or deprecated == "lms"
+            args.with_defaults(:arg1 => "dev", :arg2 => "")
+            port = args.arg2 == "" ? "" : "--port=#{args.arg2}"
+            new_cmd = deprecated_by + " --settings=#{args.arg1} #{port}"
+        else
+            new_cmd = deprecated_by
+        end
+
+        puts("Task #{deprecated} has been deprecated. Use #{new_cmd} instead. Waiting 5 seconds...".red)
+        sleep(5)
+        sh(new_cmd)
+        exit
+    end
+end
+
+deprecated('lms','paver lms')
+deprecated('fastlms', 'paver lms --fast')
+
+deprecated('cms','paver studio')
+deprecated('fastcms', 'paver studio --fast')
+
+deprecated('cms:clone', 'python manage.py cms -h')
+deprecated('cms:delete_course', 'python manage.py cms -h')
+deprecated('cms:export', 'python manage.py cms -h')
+deprecated('cms:import', 'python manage.py cms -h')
+deprecated('cms:xlint', 'python manage.py cms -h')
+deprecated('set_staff', 'python manage.py cms -h')
+
+deprecated("django-admin", "python manage.py -h")
+deprecated("resetdb", "paver update_db")
+
+
+[:lms, :cms].each do |system|
+
+    deprecated("#{system}:resetdb", "paver update_db")
+    deprecated("#{system}_worker", "paver celery")
+
+    environments(system).each do |env|
+      deprecated("#{system}:resetdb:#{env}", "paver update_db")
+      deprecated("#{system}:#{env}", "paver #{system} --settings=#{env}")
+      deprecated("#{system}:check_settings:#{env}", "paver check_settings #{system} #{env}")
+    end
+
+end

rakelib/docs.rake

-require 'launchy'
-
-# --- Develop and public documentation ---
-desc "Invoke sphinx 'make build' to generate docs."
-task :builddocs, [:type, :quiet] do |t, args|
-    args.with_defaults(:quiet => "quiet")
-    if args.type == 'dev'
-        path = "docs/en_us/developers"
-    elsif args.type == 'author'
-        path = "docs/en_us/course_authors"
-    elsif args.type == 'data'
-        path = "docs/en_us/data"
-    else
-        path = "docs/en_us"
-    end
-
-    Dir.chdir(path) do
-        if args.quiet == 'verbose'
-            sh('make html quiet=false')
-        else
-            sh('make html quiet=true')
-        end
-    end
-end
-
-desc "Show docs in browser: dev, author, data."
-task :showdocs, [:options] do |t, args|
-    if args.options == 'dev'
-        path = "docs/en_us/developers"
-    elsif args.options == 'author'
-        path = "docs/en_us/course_authors"
-    elsif args.options == 'data'
-        path = "docs/en_us/data"
-    else
-        path = "docs/en_us/developers"
-    end
-
-    Launchy.open("#{path}/build/html/index.html")
-end
-
-desc "Build docs and show them in browser"
-task :doc, [:type, :quiet] =>  :builddocs do |t, args|
-    Rake::Task["showdocs"].invoke(args.type, args.quiet)
-end
-
-
-# Run documentation tests
-desc "Run documentation tests"
-task :test_docs => :install_python_prereqs do
-    # Be sure that sphinx can build docs w/o exceptions.
-    test_message = "If a docs test fails, you should run '%s' and look at whole output and fix exceptions.
-(You shouldn't fix rst warnings and errors for this to pass, just get rid of exceptions.)"
-    puts (test_message  % ["rake doc[docs,verbose]"]).colorize( :light_green )
-    test_sh('docs', 'rake builddocs')
-end
-
-
-# Add documentation tests to the main test command
-task :test => :'test_docs'

rakelib/docs_deprecated.rake

+# doc tasks deprecated to paver
+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by)
+
+     task deprecated, [:type, :quiet] do |t,args|
+
+        # Need to install paver dependencies for the commands to work!
+        sh("pip install Paver==1.2.1 psutil==1.2.1 lazy==1.1 path.py==3.0.1")
+
+        args.with_defaults(:quiet => "quiet")
+        new_cmd = [deprecated_by]
+
+        if args.quiet == 'verbose' and deprecated == 'builddocs'
+            new_cmd << '--verbose'
+        end
+
+        if not args.type.nil?
+            new_cmd << "--type=#{args.type}"
+        end
+
+        new_cmd = new_cmd.join(" ")
+
+        puts("Task #{deprecated} has been deprecated. Use \"#{new_cmd}\" instead. Waiting 5 seconds...".red)
+        sleep(5)
+        sh(new_cmd)
+    end
+
+end
+
+
+deprecated('builddocs','paver build_docs')
+deprecated('showdocs','paver build_docs')
+deprecated('doc','paver build_docs')

rakelib/js_test.rake

@@ -51,11 +51,18 @@ def print_js_test_cmds(mode)
     end
 end
 
+# Paver migration hack: because the CoffeeScript-specific asset command has been deprecated,
+# we compile CoffeeScript ourselves
+def compile_coffeescript()
+    sh("node_modules/.bin/coffee --compile `find lms cms common -type f -name \"*.coffee\"`")
+end
 
 namespace :'test:js' do
 
     desc "Run the JavaScript tests and print results to the console"
-    task :run, [:env] => [:clean_test_files, :'assets:coffee', JS_REPORT_DIR] do |t, args|
+    task :run, [:env] => [:clean_test_files, JS_REPORT_DIR] do |t, args|
+        compile_coffeescript()
+
         if args[:env].nil?
             puts "Running all test suites.  To run a specific test suite, try:"
             print_js_test_cmds('run')
@@ -64,7 +71,9 @@ namespace :'test:js' do
     end
 
     desc "Run the JavaScript tests in your default browser"
-    task :dev, [:env] => [:clean_test_files, :'assets:coffee:_watch'] do |t, args|
+    task :dev, [:env] => [:clean_test_files] do |t, args|
+        compile_coffeescript()
+
         if args[:env].nil?
             puts "Error: No test suite specified.  Try one of these instead:"
             print_js_test_cmds('dev')
@@ -74,7 +83,8 @@ namespace :'test:js' do
     end
 
     desc "Run all JavaScript tests and collect coverage information"
-    task :coverage => [:clean_reports_dir, :clean_test_files, :'assets:coffee', JS_REPORT_DIR] do
+    task :coverage => [:clean_reports_dir, :clean_test_files, JS_REPORT_DIR] do
+        compile_coffeescript()
         js_test_tool(nil, 'run', true)
     end
 end

rakelib/prereqs.rake

-PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache')
-NPM_REGISTRY = "http://registry.npmjs.org/"
-
-CLOBBER.include(PREREQS_MD5_DIR)
-
-directory PREREQS_MD5_DIR
-
-desc "Install all prerequisites needed for the lms and cms"
-task :install_prereqs => [:install_node_prereqs, :install_ruby_prereqs, :install_python_prereqs]
-
-desc "Install all node prerequisites for the lms and cms"
-task :install_node_prereqs => "ws:migrate" do
-    unchanged = 'Node requirements unchanged, nothing to install'
-    when_changed(unchanged, ['package.json']) do
-        sh("npm config set registry '#{NPM_REGISTRY}'")
-        sh('npm install')
-    end unless ENV['NO_PREREQ_INSTALL']
-end
-
-desc "Install all ruby prerequisites for the lms and cms"
-task :install_ruby_prereqs => "ws:migrate" do
-    unchanged = 'Ruby requirements unchanged, nothing to install'
-    when_changed(unchanged, ['Gemfile']) do
-        sh('bundle install')
-    end unless ENV['NO_PREREQ_INSTALL']
-end
-
-desc "Install all python prerequisites for the lms and cms"
-task :install_python_prereqs => "ws:migrate" do
-    site_packages_dir = `python -c 'import os; import distutils.sysconfig as dusc; print dusc.get_python_lib()'`.chomp
-    unchanged = 'Python requirements unchanged, nothing to install'
-    when_changed(unchanged, ['requirements/**/*'], [site_packages_dir]) do
-        ENV['PIP_DOWNLOAD_CACHE'] ||= '.pip_download_cache'
-        sh('pip install -q --exists-action w -r requirements/edx/pre.txt')
-        sh('pip install -q --exists-action w -r requirements/edx/base.txt')
-        sh('pip install -q --exists-action w -r requirements/edx/post.txt')
-        # requirements/private.txt is used to install our libs as
-        # working dirs, or for personal-use tools.
-        if File.file?("requirements/private.txt")
-            sh('pip install -r requirements/private.txt')
-        end
-    end unless ENV['NO_PREREQ_INSTALL']
-end

rakelib/prereqs_deprecated.rake

+# prereqs tasks deprecated to paver
+
+require 'colorize'
+
+def deprecated(deprecated, deprecated_by)
+
+    task deprecated do
+
+        # Need to install paver dependencies for the commands to work!
+        sh("pip install Paver==1.2.1 psutil==1.2.1 lazy==1.1 path.py==3.0.1")
+
+        puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead.".red)
+        sh(deprecated_by)
+    end
+end
+
+deprecated('install_prereqs','paver install_prereqs')
+deprecated('install_node_prereqs','paver install_prereqs')
+deprecated('install_ruby_prereqs','paver install_prereqs')
+deprecated('install_python_prereqs','paver install_prereqs')
+

rakelib/workspace.rake

-MIGRATION_MARKER_DIR = File.join(REPO_ROOT, '.ws_migrations_complete')
-SKIP_MIGRATIONS = ENV['SKIP_WS_MIGRATIONS'] || false
-
-directory MIGRATION_MARKER_DIR
-
-namespace :ws do
-    task :migrate => MIGRATION_MARKER_DIR do
-        Dir['ws_migrations/*'].select{|m| File.executable?(m)}.each do |migration|
-            completion_file = File.join(MIGRATION_MARKER_DIR, File.basename(migration))
-            is_excluded = File.basename(migration).start_with?("README")
-            if ! File.exist?(completion_file) && ! is_excluded
-                sh(migration)
-                File.write(completion_file, "")
-            end
-        end unless SKIP_MIGRATIONS
-    end
-end

rakelib/workspace_deprecated.rake

+# acceptance tests deprecated to paver
+
+require 'colorize'
+
+task :'ws:migrate' do
+    puts "Task ws:migrate has been deprecated".red
+end

requirements/edx/base.txt

@@ -49,9 +49,11 @@ nltk==2.0.4
 oauthlib==0.5.1
 paramiko==1.9.0
 path.py==3.0.1
+Paver==1.2.1
 Pillow==1.7.8
 pip>=1.4
 polib==1.0.3
+psutil==1.2.1
 pycrypto>=2.6
 pygments==1.6
 pygraphviz==1.1

scripts/create-dev-env.sh

@@ -482,7 +482,7 @@ pip install -r $BASE/edx-platform/requirements/edx/pre.txt
 output "Installing edX requirements"
 # Install prereqs
 cd $BASE/edx-platform
-rake install_prereqs
+paver install_prereqs
 
 # Final dependecy
 output "Finishing Touches"
@@ -490,7 +490,7 @@ cd $BASE
 pip install argcomplete
 cd $BASE/edx-platform
 bundle install
-rake install_prereqs
+paver install_prereqs
 
 mkdir -p "$BASE/log"
 mkdir -p "$BASE/db"
@@ -523,7 +523,7 @@ if [[ ! $quiet ]]; then
 
    To start the Django on port 8000
 
-        $ rake lms
+        $ paver lms
 
    Or to start Django on a different <port#>
 

scripts/run.sh

-rake lms
+paver lms

scripts/vagrant-provisioning.sh

@@ -132,11 +132,11 @@ Connect to your virtual machine with "vagrant ssh".
 Some examples you can use from your virtual machine:
 
 - Start Learning management system (LMS):
-    $ rake lms[cms.dev,0.0.0.0:8000]
+    $ paver lms --settings=cms.dev
     =>  http://${MY_IP}:8000/
 
 - Start Studio:
-    $ rake cms[dev,0.0.0.0:8001]
+    $ paver studio --settings=dev
     => http://${MY_IP}:8001/
 
 See the README for more.

ws_migrations/README.rst

-Developer Workspace Migrations
-==============================
-
-This directory contains executable files which run once prior to
-installation of pre-requisites to bring a developers workspace
-into line.
-
-Specifications
---------------
-
-Each file in this directory should meet the following criteria
-
-* Executable (`chmod +x ws_migrations/foo.sh`)
-* Idempotent (ideally, each script is run only once, but no
-  guarantees are made by the caller, so the script must do
-  the right thing)
-* Either fast or verbose (if the script is going to take
-  a long time, it should notify the user of that)
-* A comment at the top of the file explaining the migration
-
-Execution
----------
-
-The scripts are run by the rake task `ws:migrate`. That task
-only runs a given script if a corresponding marker file
-in .completed-ws-migrations doesn't already exist.
-
-If the SKIP_WS_MIGRATIONS environment variable is set, then
-no workspace migrations will be run.
\ No newline at end of file

ws_migrations/clean_xmodule_assets.sh

-#! /bin/sh
-
-# Remove all of the old xmodule coffee and sass directories
-# in preparation to switching to use the xmodule_assets script
-
-rm -rf cms/static/coffee/descriptor
-rm -rf cms/static/coffee/module
-rm -rf cms/static/sass/descriptor
-rm -rf cms/static/sass/module
-rm -rf lms/static/coffee/module
-rm -rf lms/static/sass/module