Ability to run bok-choy in multiprocess mode.

This includes:
* Ability to specify number of processes to run bok-choy tests in
* A forked nose commit to get the multiprocess plugin's logging to work
* A different plugin (xunitmp) must be used for pulling together xunit results

This works by:
* Starting the various servers that are needed for the acceptance test environment
* Running the tests themselves in multiprocess mode

pavelib/bok_choy.py

@@ -27,6 +27,7 @@ __test__ = False  # do not collect
     ('extra_args=', 'e', 'adds as extra args to the test command'),
     ('default_store=', 's', 'Default modulestore'),
     ('test_dir=', 'd', 'Directory for finding tests (relative to common/test/acceptance)'),
+    ('num_processes=', 'n', 'Number of test threads (for multiprocessing)'),
     make_option("--verbose", action="store_const", const=2, dest="verbosity"),
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity"),
@@ -58,6 +59,7 @@ def test_bokchoy(options):
 
     opts = {
         'test_spec': getattr(options, 'test_spec', None),
+        'num_processes': int(getattr(options, 'num_processes', 1)),
         'fasttest': getattr(options, 'fasttest', False),
         'serversonly': getattr(options, 'serversonly', False),
         'testsonly': getattr(options, 'testsonly', False),

pavelib/paver_tests/test_paver_bok_choy_cmds.py

@@ -4,6 +4,7 @@ Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_bok_ch
 """
 import os
 import unittest
+from paver.easy import BuildFailure
 from pavelib.utils.test.suites import BokChoyTestSuite
 
 REPO_DIR = os.getcwd()
@@ -19,7 +20,7 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         Returns the command that is expected to be run for the given test spec
         and store.
         """
-        shard = os.environ.get('SHARD')
+
         expected_statement = (
             "DEFAULT_STORE={default_store} "
             "SCREENSHOT_DIR='{repo_dir}/test_root/log{shard_str}' "
@@ -32,11 +33,15 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         ).format(
             default_store=store,
             repo_dir=REPO_DIR,
-            shard_str='/shard_' + shard if shard else '',
+            shard_str='/shard_' + self.shard if self.shard else '',
             exp_text=name,
         )
         return expected_statement
 
+    def setUp(self):
+        super(TestPaverBokChoyCmd, self).setUp()
+        self.shard = os.environ.get('SHARD')
+
     def test_default(self):
         suite = BokChoyTestSuite('')
         name = 'tests'
@@ -89,3 +94,58 @@ class TestPaverBokChoyCmd(unittest.TestCase):
             suite.cmd,
             self._expected_command(name=test_dir)
         )
+
+    def test_verbosity_settings_1_process(self):
+        """
+        Using 1 process means paver should ask for the traditional xunit plugin for plugin results
+        """
+        expected_verbosity_string = (
+            "--with-xunit --xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml --verbosity=2".format(
+                repo_dir=REPO_DIR,
+                shard_str='/shard_' + self.shard if self.shard else ''
+            )
+        )
+        suite = BokChoyTestSuite('', num_processes=1)
+        self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
+
+    def test_verbosity_settings_2_processes(self):
+        """
+        Using multiple processes means specific xunit, coloring, and process-related settings should
+        be used.
+        """
+        process_count = 2
+        expected_verbosity_string = (
+            "--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml"
+            " --processes={procs} --no-color --process-timeout=1200".format(
+                repo_dir=REPO_DIR,
+                shard_str='/shard_' + self.shard if self.shard else '',
+                procs=process_count
+            )
+        )
+        suite = BokChoyTestSuite('', num_processes=process_count)
+        self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
+
+    def test_verbosity_settings_3_processes(self):
+        """
+        With the above test, validate that num_processes can be set to various values
+        """
+        process_count = 3
+        expected_verbosity_string = (
+            "--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml"
+            " --processes={procs} --no-color --process-timeout=1200".format(
+                repo_dir=REPO_DIR,
+                shard_str='/shard_' + self.shard if self.shard else '',
+                procs=process_count
+            )
+        )
+        suite = BokChoyTestSuite('', num_processes=process_count)
+        self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
+
+    def test_invalid_verbosity_and_processes(self):
+        """
+        If an invalid combination of verbosity and number of processors is passed in, a
+        BuildFailure should be raised
+        """
+        suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
+        with self.assertRaises(BuildFailure):
+            BokChoyTestSuite.verbosity_processes_string(suite)

pavelib/utils/test/suites/bokchoy_suite.py

@@ -3,7 +3,9 @@ Class used for defining and running Bok Choy acceptance test suite
 """
 from time import sleep
 
-from paver.easy import sh
+from common.test.acceptance.fixtures.course import CourseFixture, FixtureError
+
+from paver.easy import sh, BuildFailure
 from pavelib.utils.test.suites.suite import TestSuite
 from pavelib.utils.envs import Env
 from pavelib.utils.test import bokchoy_utils
@@ -16,6 +18,9 @@ except ImportError:
 
 __test__ = False  # do not collect
 
+DEFAULT_NUM_PROCESSES = 1
+DEFAULT_VERBOSITY = 2
+
 
 class BokChoyTestSuite(TestSuite):
     """
@@ -30,6 +35,9 @@ class BokChoyTestSuite(TestSuite):
       testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
       test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
       default_store - modulestore to use when running tests (split or draft)
+      num_processes - number of processes or threads to use in tests. Recommendation is that this
+      is less than or equal to the number of available processors.
+      See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html
     """
     def __init__(self, *args, **kwargs):
         super(BokChoyTestSuite, self).__init__(*args, **kwargs)
@@ -43,7 +51,8 @@ class BokChoyTestSuite(TestSuite):
         self.testsonly = kwargs.get('testsonly', False)
         self.test_spec = kwargs.get('test_spec', None)
         self.default_store = kwargs.get('default_store', None)
-        self.verbosity = kwargs.get('verbosity', 2)
+        self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY)
+        self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES)
         self.extra_args = kwargs.get('extra_args', '')
         self.har_dir = self.log_dir / 'hars'
         self.imports_dir = kwargs.get('imports_dir', None)
@@ -70,6 +79,16 @@ class BokChoyTestSuite(TestSuite):
         msg = colorize('green', "Confirming servers have started...")
         print msg
         bokchoy_utils.wait_for_test_servers()
+        try:
+            # Create course in order to seed forum data underneath. This is
+            # a workaround for a race condition. The first time a course is created;
+            # role permissions are set up for forums.
+            CourseFixture('foobar_org', '1117', 'seed_forum', 'seed_foo').install()
+            print 'Forums permissions/roles data has been seeded'
+        except FixtureError:
+            # this means it's already been done
+            pass
+
         if self.serversonly:
             self.run_servers_continuously()
 
@@ -83,6 +102,34 @@ class BokChoyTestSuite(TestSuite):
         sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
         bokchoy_utils.clear_mongo()
 
+    def verbosity_processes_string(self):
+        """
+        Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct
+        the proper combination for use with nosetests.
+        """
+        substring = []
+
+        if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES:
+            msg = 'Cannot pass in both num_processors and verbosity. Quitting'
+            raise BuildFailure(msg)
+
+        if self.num_processes != 1:
+            # Construct "multiprocess" nosetest substring
+            substring = [
+                "--with-xunitmp --xunitmp-file={}".format(self.xunit_report),
+                "--processes={}".format(self.num_processes),
+                "--no-color --process-timeout=1200"
+            ]
+
+        else:
+            substring = [
+                "--with-xunit",
+                "--xunit-file={}".format(self.xunit_report),
+                "--verbosity={}".format(self.verbosity),
+            ]
+
+        return " ".join(substring)
+
     def prepare_bokchoy_run(self):
         """
         Sets up and starts servers for a Bok Choy run. If --fasttest is not
@@ -160,9 +207,7 @@ class BokChoyTestSuite(TestSuite):
             "SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
             "nosetests",
             test_spec,
-            "--with-xunit",
-            "--xunit-file={}".format(self.xunit_report),
-            "--verbosity={}".format(self.verbosity),
+            "{}".format(self.verbosity_processes_string())
         ]
         if self.pdb:
             cmd.append("--pdb")

requirements/edx/base.txt

@@ -48,7 +48,7 @@ meliae==0.4.0
 mongoengine==0.10.0
 MySQL-python==1.2.5
 networkx==1.7
-nose==1.3.7
+nose-xunitmp==0.3.2
 oauthlib==0.7.2
 paramiko==1.9.0
 path.py==7.2

requirements/edx/github.txt

@@ -33,6 +33,9 @@ git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx
 # Used for testing
 -e git+https://github.com/gabrielfalcao/lettuce.git@b18b8fb711eb7a178c58574716032ad8de525912#egg=lettuce=1.8-support
 
+# nose fork needed for multiprocess support
+git+https://github.com/edx/nose.git@99c2aff0ff51bf228bfa5482e97e612c97a23245#egg=nose==1.3.7.1
+
 # Our libraries:
 -e git+https://github.com/edx/XBlock.git@a20c70f2e3df1cb716b9c7a25fecf57020543b7f#egg=XBlock
 -e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail

scripts/generic-ci-tests.sh

@@ -54,6 +54,7 @@ set -e
 # Note that you will still need to pass a value for 'TEST_SUITE'
 # or else no tests will be executed.
 SHARD=${SHARD:="all"}
+NUMBER_OF_BOKCHOY_THREADS=${NUMBER_OF_BOKCHOY_THREADS:=1}
 
 # Clean up previous builds
 git clean -qxfd
@@ -148,31 +149,31 @@ END
                 ;;
 
             "1")
-                paver test_bokchoy --extra_args="-a shard_1 --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1 --with-flaky"
                 ;;
 
             "2")
-                paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_2' --with-flaky"
                 ;;
 
             "3")
-                paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_3' --with-flaky"
                 ;;
 
             "4")
-                paver test_bokchoy --extra_args="-a 'shard_4' --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_4' --with-flaky"
                 ;;
 
             "5")
-                paver test_bokchoy --extra_args="-a 'shard_5' --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_5' --with-flaky"
                 ;;
 
             "6")
-                paver test_bokchoy --extra_args="-a 'shard_6' --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_6' --with-flaky"
                 ;;
 
             "7")
-                paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky"
+                paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky"
                 ;;
 
             # Default case because if we later define another bok-choy shard on Jenkins