Bug 1405141 - Add support for rerunning web-platform-tests without restarting, r=gbrown
This adds a --rerun=N flag which runs each selected test N times in a row.
MozReview-Commit-ID: 4dGuPDkjWeZ
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
@@ -243,17 +243,17 @@ class _RunnerManagerState(object):
stop = namedtuple("stop", [])
RunnerManagerState = _RunnerManagerState()
class TestRunnerManager(threading.Thread):
def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
- executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
+ executor_cls, executor_kwargs, stop_flag, rerun=1, pause_after_test=False,
pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None):
"""Thread that owns a single TestRunner process and any processes required
by the TestRunner (e.g. the Firefox binary).
TestRunnerManagers are responsible for launching the browser process and the
runner process, and for logging the test progress. The actual test running
is done by the TestRunner. In particular they:
@@ -274,16 +274,18 @@ class TestRunnerManager(threading.Thread
self.executor_cls = executor_cls
self.executor_kwargs = executor_kwargs
# Flags used to shut down this thread if we get a sigint
self.parent_stop_flag = stop_flag
self.child_stop_flag = multiprocessing.Event()
+ self.rerun = rerun
+ self.run_count = 0
self.pause_after_test = pause_after_test
self.pause_on_unexpected = pause_on_unexpected
self.restart_on_unexpected = restart_on_unexpected
self.debug_info = debug_info
self.manager_number = next_manager_number()
self.command_queue = Queue()
@@ -497,20 +499,24 @@ class TestRunnerManager(threading.Thread
test = None
while test is None:
while test_group is None or len(test_group) == 0:
test_group, group_metadata = self.test_source.group()
if test_group is None:
self.logger.info("No more tests")
return None, None, None
test = test_group.popleft()
+ self.run_count = 0
return test, test_group, group_metadata
def run_test(self):
+ self.run_count += 1
+ if self.rerun > 1:
+ self.logger.info("Run %d/%d" % (self.run_count, self.rerun))
assert isinstance(self.state, RunnerManagerState.running)
assert self.state.test is not None
if self.browser.update_settings(self.state.test):
self.logger.info("Restarting browser for new test environment")
return RunnerManagerState.restarting(self.state.test,
self.state.test_group,
self.state.group_metadata)
@@ -574,34 +580,39 @@ class TestRunnerManager(threading.Thread
((subtest_unexpected or is_unexpected)
and self.restart_on_unexpected))
if (self.pause_after_test or
(self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
self.logger.info("Pausing until the browser exits")
self.send_message("wait")
else:
- return self.after_test_end(restart_before_next)
+ return self.after_test_end(test, restart_before_next)
def wait_finished(self):
assert isinstance(self.state, RunnerManagerState.running)
# The browser should be stopped already, but this ensures we do any post-stop
# processing
self.logger.debug("Wait finished")
return self.after_test_end(True)
- def after_test_end(self, restart):
+ def after_test_end(self, test, restart):
assert isinstance(self.state, RunnerManagerState.running)
- test, test_group, group_metadata = self.get_next_test()
- if test is None:
- return RunnerManagerState.stop()
- if test_group != self.state.test_group:
- # We are starting a new group of tests, so force a restart
- restart = True
+ if self.run_count == self.rerun:
+ test, test_group, group_metadata = self.get_next_test()
+ if test is None:
+ return RunnerManagerState.stop()
+ if test_group != self.state.test_group:
+ # We are starting a new group of tests, so force a restart
+ restart = True
+ else:
+ test = test
+ test_group = self.state.test_group
+ group_metadata = self.state.group_metadata
if restart:
return RunnerManagerState.restarting(test, test_group, group_metadata)
else:
return RunnerManagerState.running(test, test_group, group_metadata)
def restart_runner(self):
"""Stop and restart the TestRunner"""
assert isinstance(self.state, RunnerManagerState.restarting)
@@ -680,16 +691,17 @@ def make_test_queue(tests, test_source_c
assert not queue.empty()
return queue
class ManagerGroup(object):
def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
browser_cls, browser_kwargs,
executor_cls, executor_kwargs,
+ rerun=1,
pause_after_test=False,
pause_on_unexpected=False,
restart_on_unexpected=True,
debug_info=None):
"""Main thread object that owns all the TestManager threads."""
self.suite_name = suite_name
self.size = size
self.test_source_cls = test_source_cls
@@ -697,16 +709,17 @@ class ManagerGroup(object):
self.browser_cls = browser_cls
self.browser_kwargs = browser_kwargs
self.executor_cls = executor_cls
self.executor_kwargs = executor_kwargs
self.pause_after_test = pause_after_test
self.pause_on_unexpected = pause_on_unexpected
self.restart_on_unexpected = restart_on_unexpected
self.debug_info = debug_info
+ self.rerun = rerun
self.pool = set()
# Event that is polled by threads so that they can gracefully exit in the face
# of sigint
self.stop_flag = threading.Event()
self.logger = structuredlog.StructuredLogger(suite_name)
def __enter__(self):
@@ -729,16 +742,17 @@ class ManagerGroup(object):
manager = TestRunnerManager(self.suite_name,
test_queue,
self.test_source_cls,
self.browser_cls,
self.browser_kwargs,
self.executor_cls,
self.executor_kwargs,
self.stop_flag,
+ self.rerun,
self.pause_after_test,
self.pause_on_unexpected,
self.restart_on_unexpected,
self.debug_info)
manager.start()
self.pool.add(manager)
self.wait()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -88,18 +88,20 @@ scheme host and port.""")
help="Path to manifest listing tests to include")
test_selection_group.add_argument("--tag", action="append", dest="tags",
help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
+ debugging_group.add_argument("--rerun", action="store", type=int, default=1,
+ help="Number of times to re run each test without restarts")
debugging_group.add_argument("--repeat", action="store", type=int, default=1,
- help="Number of times to run the tests")
+ help="Number of times to run the tests, restarting between each run")
debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
help="Run tests in a loop until one returns an unexpected result")
debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
help="Halt the test runner after each test (this happens by default if only a single test is run)")
debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
help="Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument('--pause-on-unexpected', action="store_true",
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -236,16 +236,17 @@ def run_tests(config, test_paths, produc
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
+ kwargs["rerun"],
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["restart_on_unexpected"],
kwargs["debug_info"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")