Skip to content

E
edx-platform
Commit 6e80cedc by Ben Patterson
Options

Merge pull request #10138 from edx/benp/2-multiprocess-bokchoy-oct15 

Multi-process bok-choy tests
parents acd4c566 7eb9cecb
Showing with 126 additions and 15 deletions
pavelib/bok_choy.py
......@@ -27,6 +27,7 @@ __test__ = False # do not collect
('extra_args=', 'e', 'adds as extra args to the test command'),
('default_store=', 's', 'Default modulestore'),
('test_dir=', 'd', 'Directory for finding tests (relative to common/test/acceptance)'),
('num_processes=', 'n', 'Number of test threads (for multiprocessing)'),
make_option("--verbose", action="store_const", const=2, dest="verbosity"),
make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
make_option("-v", "--verbosity", action="count", dest="verbosity"),
......@@ -58,6 +59,7 @@ def test_bokchoy(options):
opts = {
'test_spec': getattr(options, 'test_spec', None),
'num_processes': int(getattr(options, 'num_processes', 1)),
'fasttest': getattr(options, 'fasttest', False),
'serversonly': getattr(options, 'serversonly', False),
'testsonly': getattr(options, 'testsonly', False),
......
pavelib/paver_tests/test_paver_bok_choy_cmds.py
......@@ -4,6 +4,7 @@ Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_bok_ch
"""
import os
import unittest
from paver.easy import BuildFailure
from pavelib.utils.test.suites import BokChoyTestSuite
REPO_DIR = os.getcwd()
......@@ -19,7 +20,7 @@ class TestPaverBokChoyCmd(unittest.TestCase):
Returns the command that is expected to be run for the given test spec
and store.
"""
shard = os.environ.get('SHARD')
expected_statement = (
"DEFAULT_STORE={default_store} "
"SCREENSHOT_DIR='{repo_dir}/test_root/log{shard_str}' "
......@@ -32,11 +33,15 @@ class TestPaverBokChoyCmd(unittest.TestCase):
).format(
default_store=store,
repo_dir=REPO_DIR,
shard_str='/shard_' + shard if shard else '',
shard_str='/shard_' + self.shard if self.shard else '',
exp_text=name,
)
return expected_statement
def setUp(self):
super(TestPaverBokChoyCmd, self).setUp()
self.shard = os.environ.get('SHARD')
def test_default(self):
suite = BokChoyTestSuite('')
name = 'tests'
......@@ -89,3 +94,58 @@ class TestPaverBokChoyCmd(unittest.TestCase):
suite.cmd,
self._expected_command(name=test_dir)
)
def test_verbosity_settings_1_process(self):
"""
Using 1 process means paver should ask for the traditional xunit plugin for plugin results
"""
expected_verbosity_string = (
"--with-xunit --xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml --verbosity=2".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else ''
)
)
suite = BokChoyTestSuite('', num_processes=1)
self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
def test_verbosity_settings_2_processes(self):
"""
Using multiple processes means specific xunit, coloring, and process-related settings should
be used.
"""
process_count = 2
expected_verbosity_string = (
"--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml"
" --processes={procs} --no-color --process-timeout=1200".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
procs=process_count
)
)
suite = BokChoyTestSuite('', num_processes=process_count)
self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
def test_verbosity_settings_3_processes(self):
"""
With the above test, validate that num_processes can be set to various values
"""
process_count = 3
expected_verbosity_string = (
"--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml"
" --processes={procs} --no-color --process-timeout=1200".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
procs=process_count
)
)
suite = BokChoyTestSuite('', num_processes=process_count)
self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
def test_invalid_verbosity_and_processes(self):
"""
If an invalid combination of verbosity and number of processors is passed in, a
BuildFailure should be raised
"""
suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
with self.assertRaises(BuildFailure):
BokChoyTestSuite.verbosity_processes_string(suite)
pavelib/utils/test/suites/bokchoy_suite.py
......@@ -3,7 +3,9 @@ Class used for defining and running Bok Choy acceptance test suite
"""
from time import sleep
from paver.easy import sh
from common.test.acceptance.fixtures.course import CourseFixture, FixtureError
from paver.easy import sh, BuildFailure
from pavelib.utils.test.suites.suite import TestSuite
from pavelib.utils.envs import Env
from pavelib.utils.test import bokchoy_utils
......@@ -16,6 +18,9 @@ except ImportError:
__test__ = False # do not collect
DEFAULT_NUM_PROCESSES = 1
DEFAULT_VERBOSITY = 2
class BokChoyTestSuite(TestSuite):
"""
......@@ -30,6 +35,9 @@ class BokChoyTestSuite(TestSuite):
testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
default_store - modulestore to use when running tests (split or draft)
num_processes - number of processes or threads to use in tests. Recommendation is that this
is less than or equal to the number of available processors.
See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html
"""
def __init__(self, *args, **kwargs):
super(BokChoyTestSuite, self).__init__(*args, **kwargs)
......@@ -43,7 +51,8 @@ class BokChoyTestSuite(TestSuite):
self.testsonly = kwargs.get('testsonly', False)
self.test_spec = kwargs.get('test_spec', None)
self.default_store = kwargs.get('default_store', None)
self.verbosity = kwargs.get('verbosity', 2)
self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY)
self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES)
self.extra_args = kwargs.get('extra_args', '')
self.har_dir = self.log_dir / 'hars'
self.imports_dir = kwargs.get('imports_dir', None)
......@@ -70,6 +79,16 @@ class BokChoyTestSuite(TestSuite):
msg = colorize('green', "Confirming servers have started...")
print msg
bokchoy_utils.wait_for_test_servers()
try:
# Create course in order to seed forum data underneath. This is
# a workaround for a race condition. The first time a course is created;
# role permissions are set up for forums.
CourseFixture('foobar_org', '1117', 'seed_forum', 'seed_foo').install()
print 'Forums permissions/roles data has been seeded'
except FixtureError:
# this means it's already been done
pass
if self.serversonly:
self.run_servers_continuously()
......@@ -83,6 +102,34 @@ class BokChoyTestSuite(TestSuite):
sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
bokchoy_utils.clear_mongo()
def verbosity_processes_string(self):
"""
Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct
the proper combination for use with nosetests.
"""
substring = []
if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES:
msg = 'Cannot pass in both num_processors and verbosity. Quitting'
raise BuildFailure(msg)
if self.num_processes != 1:
# Construct "multiprocess" nosetest substring
substring = [
"--with-xunitmp --xunitmp-file={}".format(self.xunit_report),
"--processes={}".format(self.num_processes),
"--no-color --process-timeout=1200"
]
else:
substring = [
"--with-xunit",
"--xunit-file={}".format(self.xunit_report),
"--verbosity={}".format(self.verbosity),
]
return " ".join(substring)
def prepare_bokchoy_run(self):
"""
Sets up and starts servers for a Bok Choy run. If --fasttest is not
......@@ -160,9 +207,7 @@ class BokChoyTestSuite(TestSuite):
"SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
"nosetests",
test_spec,
"--with-xunit",
"--xunit-file={}".format(self.xunit_report),
"--verbosity={}".format(self.verbosity),
"{}".format(self.verbosity_processes_string())
]
if self.pdb:
cmd.append("--pdb")
......
requirements/edx/base.txt
......@@ -48,7 +48,7 @@ meliae==0.4.0
mongoengine==0.10.0
MySQL-python==1.2.5
networkx==1.7
nose==1.3.7
nose-xunitmp==0.3.2
oauthlib==0.7.2
paramiko==1.9.0
path.py==7.2
......
requirements/edx/github.txt
......@@ -33,6 +33,9 @@ git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx
# Used for testing
-e git+https://github.com/gabrielfalcao/lettuce.git@b18b8fb711eb7a178c58574716032ad8de525912#egg=lettuce=1.8-support
# nose fork needed for multiprocess support
git+https://github.com/edx/nose.git@99c2aff0ff51bf228bfa5482e97e612c97a23245#egg=nose==1.3.7.1
# Our libraries:
-e git+https://github.com/edx/XBlock.git@a20c70f2e3df1cb716b9c7a25fecf57020543b7f#egg=XBlock
-e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail
......
scripts/generic-ci-tests.sh
......@@ -54,6 +54,7 @@ set -e
# Note that you will still need to pass a value for 'TEST_SUITE'
# or else no tests will be executed.
SHARD=${SHARD:="all"}
NUMBER_OF_BOKCHOY_THREADS=${NUMBER_OF_BOKCHOY_THREADS:=1}
# Clean up previous builds
git clean -qxfd
......@@ -148,31 +149,31 @@ END
;;
"1")
paver test_bokchoy --extra_args="-a shard_1 --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1 --with-flaky"
;;
"2")
paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_2' --with-flaky"
;;
"3")
paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_3' --with-flaky"
;;
"4")
paver test_bokchoy --extra_args="-a 'shard_4' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_4' --with-flaky"
;;
"5")
paver test_bokchoy --extra_args="-a 'shard_5' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_5' --with-flaky"
;;
"6")
paver test_bokchoy --extra_args="-a 'shard_6' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_6' --with-flaky"
;;
"7")
paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky"
;;
# Default case because if we later define another bok-choy shard on Jenkins
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment