Bug 1405141 - Add support for rerunning web-platform-tests without restarting, r=gbrown
☠☠ backed out by e4a7c6646eb4 ☠ ☠
authorJames Graham <james@hoppipolla.co.uk>
Mon, 16 Oct 2017 14:32:27 +0100
changeset 685712 3b0ece162b1953995eb90ec80cdb333222ddd21e
parent 685711 238d4670044e159bb9bd6950e5a937030774c845
child 685713 97a964bf7f990416e039fc9747e19e46f7a6dcea
push id86016
push userkgupta@mozilla.com
push dateWed, 25 Oct 2017 01:53:44 +0000
reviewersgbrown
bugs1405141
milestone58.0a1
Bug 1405141 - Add support for rerunning web-platform-tests without restarting, r=gbrown This adds a --rerun=N flag which runs each selected test N times in a row. MozReview-Commit-ID: 4dGuPDkjWeZ
testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py
@@ -243,17 +243,17 @@ class _RunnerManagerState(object):
     stop = namedtuple("stop", [])
 
 
 RunnerManagerState = _RunnerManagerState()
 
 
 class TestRunnerManager(threading.Thread):
     def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
-                 executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
+                 executor_cls, executor_kwargs, stop_flag, rerun=1, pause_after_test=False,
                  pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None):
         """Thread that owns a single TestRunner process and any processes required
         by the TestRunner (e.g. the Firefox binary).
 
         TestRunnerManagers are responsible for launching the browser process and the
         runner process, and for logging the test progress. The actual test running
         is done by the TestRunner. In particular they:
 
@@ -274,16 +274,18 @@ class TestRunnerManager(threading.Thread
 
         self.executor_cls = executor_cls
         self.executor_kwargs = executor_kwargs
 
         # Flags used to shut down this thread if we get a sigint
         self.parent_stop_flag = stop_flag
         self.child_stop_flag = multiprocessing.Event()
 
+        self.rerun = rerun
+        self.run_count = 0
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
 
         self.manager_number = next_manager_number()
 
         self.command_queue = Queue()
@@ -497,20 +499,24 @@ class TestRunnerManager(threading.Thread
         test = None
         while test is None:
             while test_group is None or len(test_group) == 0:
                 test_group, group_metadata = self.test_source.group()
                 if test_group is None:
                     self.logger.info("No more tests")
                     return None, None, None
             test = test_group.popleft()
+        self.run_count = 0
         return test, test_group, group_metadata
 
 
     def run_test(self):
+        self.run_count += 1
+        if self.rerun > 1:
+            self.logger.info("Run %d/%d" % (self.run_count, self.rerun))
         assert isinstance(self.state, RunnerManagerState.running)
         assert self.state.test is not None
 
         if self.browser.update_settings(self.state.test):
             self.logger.info("Restarting browser for new test environment")
             return RunnerManagerState.restarting(self.state.test,
                                                  self.state.test_group,
                                                  self.state.group_metadata)
@@ -574,34 +580,39 @@ class TestRunnerManager(threading.Thread
                                ((subtest_unexpected or is_unexpected)
                                 and self.restart_on_unexpected))
 
         if (self.pause_after_test or
             (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
             self.logger.info("Pausing until the browser exits")
             self.send_message("wait")
         else:
-            return self.after_test_end(restart_before_next)
+            return self.after_test_end(test, restart_before_next)
 
     def wait_finished(self):
         assert isinstance(self.state, RunnerManagerState.running)
         # The browser should be stopped already, but this ensures we do any post-stop
         # processing
         self.logger.debug("Wait finished")
 
         return self.after_test_end(True)
 
-    def after_test_end(self, restart):
+    def after_test_end(self, test, restart):
         assert isinstance(self.state, RunnerManagerState.running)
-        test, test_group, group_metadata = self.get_next_test()
-        if test is None:
-            return RunnerManagerState.stop()
-        if test_group != self.state.test_group:
-            # We are starting a new group of tests, so force a restart
-            restart = True
+        if self.run_count == self.rerun:
+            test, test_group, group_metadata = self.get_next_test()
+            if test is None:
+                return RunnerManagerState.stop()
+            if test_group != self.state.test_group:
+                # We are starting a new group of tests, so force a restart
+                restart = True
+        else:
+            test = test
+            test_group = self.state.test_group
+            group_metadata = self.state.group_metadata
         if restart:
             return RunnerManagerState.restarting(test, test_group, group_metadata)
         else:
             return RunnerManagerState.running(test, test_group, group_metadata)
 
     def restart_runner(self):
         """Stop and restart the TestRunner"""
         assert isinstance(self.state, RunnerManagerState.restarting)
@@ -680,16 +691,17 @@ def make_test_queue(tests, test_source_c
     assert not queue.empty()
     return queue
 
 
 class ManagerGroup(object):
     def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
                  browser_cls, browser_kwargs,
                  executor_cls, executor_kwargs,
+                 rerun=1,
                  pause_after_test=False,
                  pause_on_unexpected=False,
                  restart_on_unexpected=True,
                  debug_info=None):
         """Main thread object that owns all the TestManager threads."""
         self.suite_name = suite_name
         self.size = size
         self.test_source_cls = test_source_cls
@@ -697,16 +709,17 @@ class ManagerGroup(object):
         self.browser_cls = browser_cls
         self.browser_kwargs = browser_kwargs
         self.executor_cls = executor_cls
         self.executor_kwargs = executor_kwargs
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
+        self.rerun = rerun
 
         self.pool = set()
         # Event that is polled by threads so that they can gracefully exit in the face
         # of sigint
         self.stop_flag = threading.Event()
         self.logger = structuredlog.StructuredLogger(suite_name)
 
     def __enter__(self):
@@ -729,16 +742,17 @@ class ManagerGroup(object):
             manager = TestRunnerManager(self.suite_name,
                                         test_queue,
                                         self.test_source_cls,
                                         self.browser_cls,
                                         self.browser_kwargs,
                                         self.executor_cls,
                                         self.executor_kwargs,
                                         self.stop_flag,
+                                        self.rerun,
                                         self.pause_after_test,
                                         self.pause_on_unexpected,
                                         self.restart_on_unexpected,
                                         self.debug_info)
             manager.start()
             self.pool.add(manager)
         self.wait()
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -88,18 +88,20 @@ scheme host and port.""")
                                       help="Path to manifest listing tests to include")
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
 
     debugging_group = parser.add_argument_group("Debugging")
     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
                                  help="run under a debugger, e.g. gdb or valgrind")
     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
+    debugging_group.add_argument("--rerun", action="store", type=int, default=1,
+                                 help="Number of times to re run each test without restarts")
     debugging_group.add_argument("--repeat", action="store", type=int, default=1,
-                                 help="Number of times to run the tests")
+                                 help="Number of times to run the tests, restarting between each run")
     debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
                                  help="Run tests in a loop until one returns an unexpected result")
     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
                                  help="Halt the test runner after each test (this happens by default if only a single test is run)")
     debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
                                  help="Don't halt the test runner irrespective of the number of tests run")
 
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -236,16 +236,17 @@ def run_tests(config, test_paths, produc
                     with ManagerGroup("web-platform-tests",
                                       kwargs["processes"],
                                       test_source_cls,
                                       test_source_kwargs,
                                       browser_cls,
                                       browser_kwargs,
                                       executor_cls,
                                       executor_kwargs,
+                                      kwargs["rerun"],
                                       kwargs["pause_after_test"],
                                       kwargs["pause_on_unexpected"],
                                       kwargs["restart_on_unexpected"],
                                       kwargs["debug_info"]) as manager_group:
                         try:
                             manager_group.run(test_type, test_loader.tests)
                         except KeyboardInterrupt:
                             logger.critical("Main thread got signal")