Bug 1405141 - Add support for rerunning web-platform-tests without restarting, r=gbrown
authorJames Graham <james@hoppipolla.co.uk>
Mon, 16 Oct 2017 14:32:27 +0100
changeset 388066 0faad69400fac3a8cb9340e820380a5fca11d57f
parent 388065 40e33c4b41fd68d572725d03705433a81a6b65ce
child 388067 00bec72d17c4591f10e7dc7930637a2dded95975
push id32740
push useracraciun@mozilla.com
push dateWed, 25 Oct 2017 09:30:59 +0000
treeherdermozilla-central@e56ae7213756 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1405141
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1405141 - Add support for rerunning web-platform-tests without restarting, r=gbrown This adds a --rerun=N flag which runs each selected test N times in a row. MozReview-Commit-ID: 4dGuPDkjWeZ
testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
@@ -243,17 +243,17 @@ class _RunnerManagerState(object):
     stop = namedtuple("stop", [])
 
 
 RunnerManagerState = _RunnerManagerState()
 
 
 class TestRunnerManager(threading.Thread):
     def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
-                 executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
+                 executor_cls, executor_kwargs, stop_flag, rerun=1, pause_after_test=False,
                  pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None):
         """Thread that owns a single TestRunner process and any processes required
         by the TestRunner (e.g. the Firefox binary).
 
         TestRunnerManagers are responsible for launching the browser process and the
         runner process, and for logging the test progress. The actual test running
         is done by the TestRunner. In particular they:
 
@@ -274,16 +274,18 @@ class TestRunnerManager(threading.Thread
 
         self.executor_cls = executor_cls
         self.executor_kwargs = executor_kwargs
 
         # Flags used to shut down this thread if we get a sigint
         self.parent_stop_flag = stop_flag
         self.child_stop_flag = multiprocessing.Event()
 
+        self.rerun = rerun
+        self.run_count = 0
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
 
         self.manager_number = next_manager_number()
 
         self.command_queue = Queue()
@@ -497,30 +499,33 @@ class TestRunnerManager(threading.Thread
         test = None
         while test is None:
             while test_group is None or len(test_group) == 0:
                 test_group, group_metadata = self.test_source.group()
                 if test_group is None:
                     self.logger.info("No more tests")
                     return None, None, None
             test = test_group.popleft()
+        self.run_count = 0
         return test, test_group, group_metadata
 
-
     def run_test(self):
         assert isinstance(self.state, RunnerManagerState.running)
         assert self.state.test is not None
 
         if self.browser.update_settings(self.state.test):
             self.logger.info("Restarting browser for new test environment")
             return RunnerManagerState.restarting(self.state.test,
                                                  self.state.test_group,
                                                  self.state.group_metadata)
 
         self.logger.test_start(self.state.test.id)
+        if self.rerun > 1:
+            self.logger.info("Run %d/%d" % (self.run_count, self.rerun))
+        self.run_count += 1
         self.send_message("run_test", self.state.test)
 
     def test_ended(self, test, results):
         """Handle the end of a test.
 
         Output the result of each subtest, and the result of the overall
         harness to the logs.
         """
@@ -574,34 +579,39 @@ class TestRunnerManager(threading.Thread
                                ((subtest_unexpected or is_unexpected)
                                 and self.restart_on_unexpected))
 
         if (self.pause_after_test or
             (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
             self.logger.info("Pausing until the browser exits")
             self.send_message("wait")
         else:
-            return self.after_test_end(restart_before_next)
+            return self.after_test_end(test, restart_before_next)
 
     def wait_finished(self):
         assert isinstance(self.state, RunnerManagerState.running)
         # The browser should be stopped already, but this ensures we do any post-stop
         # processing
         self.logger.debug("Wait finished")
 
         return self.after_test_end(True)
 
-    def after_test_end(self, restart):
+    def after_test_end(self, test, restart):
         assert isinstance(self.state, RunnerManagerState.running)
-        test, test_group, group_metadata = self.get_next_test()
-        if test is None:
-            return RunnerManagerState.stop()
-        if test_group != self.state.test_group:
-            # We are starting a new group of tests, so force a restart
-            restart = True
+        if self.run_count == self.rerun:
+            test, test_group, group_metadata = self.get_next_test()
+            if test is None:
+                return RunnerManagerState.stop()
+            if test_group != self.state.test_group:
+                # We are starting a new group of tests, so force a restart
+                restart = True
+        else:
+            test = test
+            test_group = self.state.test_group
+            group_metadata = self.state.group_metadata
         if restart:
             return RunnerManagerState.restarting(test, test_group, group_metadata)
         else:
             return RunnerManagerState.running(test, test_group, group_metadata)
 
     def restart_runner(self):
         """Stop and restart the TestRunner"""
         assert isinstance(self.state, RunnerManagerState.restarting)
@@ -680,16 +690,17 @@ def make_test_queue(tests, test_source_c
     assert not queue.empty()
     return queue
 
 
 class ManagerGroup(object):
     def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
                  browser_cls, browser_kwargs,
                  executor_cls, executor_kwargs,
+                 rerun=1,
                  pause_after_test=False,
                  pause_on_unexpected=False,
                  restart_on_unexpected=True,
                  debug_info=None):
         """Main thread object that owns all the TestManager threads."""
         self.suite_name = suite_name
         self.size = size
         self.test_source_cls = test_source_cls
@@ -697,16 +708,17 @@ class ManagerGroup(object):
         self.browser_cls = browser_cls
         self.browser_kwargs = browser_kwargs
         self.executor_cls = executor_cls
         self.executor_kwargs = executor_kwargs
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
+        self.rerun = rerun
 
         self.pool = set()
         # Event that is polled by threads so that they can gracefully exit in the face
         # of sigint
         self.stop_flag = threading.Event()
         self.logger = structuredlog.StructuredLogger(suite_name)
 
     def __enter__(self):
@@ -729,16 +741,17 @@ class ManagerGroup(object):
             manager = TestRunnerManager(self.suite_name,
                                         test_queue,
                                         self.test_source_cls,
                                         self.browser_cls,
                                         self.browser_kwargs,
                                         self.executor_cls,
                                         self.executor_kwargs,
                                         self.stop_flag,
+                                        self.rerun,
                                         self.pause_after_test,
                                         self.pause_on_unexpected,
                                         self.restart_on_unexpected,
                                         self.debug_info)
             manager.start()
             self.pool.add(manager)
         self.wait()
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -88,18 +88,20 @@ scheme host and port.""")
                                       help="Path to manifest listing tests to include")
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
 
     debugging_group = parser.add_argument_group("Debugging")
     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
                                  help="run under a debugger, e.g. gdb or valgrind")
     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
+    debugging_group.add_argument("--rerun", action="store", type=int, default=1,
+                                 help="Number of times to re run each test without restarts")
     debugging_group.add_argument("--repeat", action="store", type=int, default=1,
-                                 help="Number of times to run the tests")
+                                 help="Number of times to run the tests, restarting between each run")
     debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
                                  help="Run tests in a loop until one returns an unexpected result")
     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
                                  help="Halt the test runner after each test (this happens by default if only a single test is run)")
     debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
                                  help="Don't halt the test runner irrespective of the number of tests run")
 
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -117,17 +117,17 @@ def list_tests(test_paths, product, **kw
         print test
 
 
 def get_pause_after_test(test_loader, **kwargs):
     total_tests = sum(len(item) for item in test_loader.tests.itervalues())
     if kwargs["pause_after_test"] is None:
         if kwargs["repeat_until_unexpected"]:
             return False
-        if kwargs["repeat"] == 1 and total_tests == 1:
+        if kwargs["repeat"] == 1 and kwargs["rerun"] == 1 and total_tests == 1:
             return True
         return False
     return kwargs["pause_after_test"]
 
 
 def run_tests(config, test_paths, product, **kwargs):
     with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
         env.do_delayed_imports(logger, test_paths)
@@ -236,16 +236,17 @@ def run_tests(config, test_paths, produc
                     with ManagerGroup("web-platform-tests",
                                       kwargs["processes"],
                                       test_source_cls,
                                       test_source_kwargs,
                                       browser_cls,
                                       browser_kwargs,
                                       executor_cls,
                                       executor_kwargs,
+                                      kwargs["rerun"],
                                       kwargs["pause_after_test"],
                                       kwargs["pause_on_unexpected"],
                                       kwargs["restart_on_unexpected"],
                                       kwargs["debug_info"]) as manager_group:
                         try:
                             manager_group.run(test_type, test_loader.tests)
                         except KeyboardInterrupt:
                             logger.critical("Main thread got signal")