Bug 1405141 - Add stability checking to wptrunner, r=gbrown
authorJames Graham <james@hoppipolla.co.uk>
Mon, 16 Oct 2017 14:33:41 +0100
changeset 388067 00bec72d17c4591f10e7dc7930637a2dded95975
parent 388066 0faad69400fac3a8cb9340e820380a5fca11d57f
child 388068 4dfe93e7c0811d6c130e4e56c967909fec494b03
push id32740
push useracraciun@mozilla.com
push dateWed, 25 Oct 2017 09:30:59 +0000
treeherdermozilla-central@e56ae7213756 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1405141
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1405141 - Add stability checking to wptrunner, r=gbrown This adds a --verify flag that is compatible with other Mozilla test harnesses i.e. it runs each test 10 times without restarting and then runs it 5 times with restarts, and then repeats with chaos mode enabled. This uses the code from, and can replace, the |wpt run --stability| flag from upstream although that has different default behaviour (running 10 times with restarts). More work is needed to avoid duplicating all the code, however. MozReview-Commit-ID: 7oUEwJk7uhZ
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -69,16 +69,19 @@ scheme host and port.""")
                             default=False,
                             help="List the top level directories containing tests that will run.")
     mode_group.add_argument("--list-disabled", action="store_true",
                             default=False,
                             help="List the tests that are disabled on the current platform")
     mode_group.add_argument("--list-tests", action="store_true",
                             default=False,
                             help="List all tests that will run")
+    mode_group.add_argument("--verify", action="store_true",
+                            default=False,
+                            help="Run a stability check on the selected tests")
 
     test_selection_group = parser.add_argument_group("Test Selection")
     test_selection_group.add_argument("--test-types", action="store",
                                       nargs="*", default=wpttest.enabled_tests,
                                       choices=wpttest.enabled_tests,
                                       help="Test types to run")
     test_selection_group.add_argument("--include", action="append",
                                       help="URL prefix to include")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -256,26 +256,35 @@ def run_tests(config, test_paths, produc
 
                 unexpected_total += unexpected_count
                 logger.info("Got %i unexpected results" % unexpected_count)
                 if repeat_until_unexpected and unexpected_total > 0:
                     break
                 logger.suite_end()
     return unexpected_total == 0
 
+
+def check_stability(**kwargs):
+    import stability
+    return stability.check_stability(logger, **kwargs)
+
+
 def start(**kwargs):
     if kwargs["list_test_groups"]:
         list_test_groups(**kwargs)
     elif kwargs["list_disabled"]:
         list_disabled(**kwargs)
     elif kwargs["list_tests"]:
         list_tests(**kwargs)
+    elif kwargs["verify"]:
+        check_stability(**kwargs)
     else:
         return not run_tests(**kwargs)
 
+
 def main():
     """Main entry point when calling from the command line"""
     kwargs = wptcommandline.parse_args()
 
     try:
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.abspath(os.path.join(here, "prefs"))