Bug 1516135 [wpt PR 14649] - Produce a valid report when no tests are affected, a=testonly
authorRobert Ma <robertma@chromium.org>
Thu, 31 Jan 2019 15:44:59 +0000
changeset 457882 7e0785c0e88e4daaa94576f7c81add6ae85e4491
parent 457881 4f2fb429edd84e4a4a4b573ad5415afd89a66596
child 457883 02828ac2fb875fcba52d60fec94b59e2bd52fe8d
push id35518
push useropoprus@mozilla.com
push dateFri, 08 Feb 2019 09:55:14 +0000
treeherdermozilla-central@3a3e393396f4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1516135, 14649, 14471
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1516135 [wpt PR 14649] - Produce a valid report when no tests are affected, a=testonly Automatic update from web-platform-tests Produce a valid report when no tests are affected This is done in a roundabout way: 1. Instead of early exiting, `wpt run` now continues to call wptrunner when no tests are affected to make sure we have suite_start and suite_end lifecycle events. 2. However, wptrunner would then by default run all tests, so a new flag (--default-exclude) is added to wptrunner to only run the given list of tests and do not fail when no tests run. Fixes #14471 . -- wpt-commits: 79289cbd86b5dae60db7caec8d03f1512a1f9222 wpt-pr: 14649
testing/web-platform/tests/tools/wpt/run.py
testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
--- a/testing/web-platform/tests/tools/wpt/run.py
+++ b/testing/web-platform/tests/tools/wpt/run.py
@@ -505,21 +505,19 @@ def setup_wptrunner(venv, prompt=True, i
             include_uncommitted=True, include_new=True)
         # TODO: Perhaps use wptrunner.testloader.ManifestLoader here
         # and remove the manifest-related code from testfiles.
         # https://github.com/web-platform-tests/wpt/issues/14421
         tests_changed, tests_affected = testfiles.affected_testfiles(
             files_changed, manifest_path=kwargs.get("manifest_path"), manifest_update=kwargs["manifest_update"])
         test_list = tests_changed | tests_affected
         logger.info("Identified %s affected tests" % len(test_list))
-        if not test_list and not kwargs["test_list"]:
-            logger.info("Quitting because no tests were affected.")
-            exit()
         test_list = [os.path.relpath(item, wpt_root) for item in test_list]
         kwargs["test_list"] += test_list
+        kwargs["default_exclude"] = True
 
     if install_browser and not kwargs["channel"]:
         logger.info("--install-browser is given but --channel is not set, default to nightly channel")
         kwargs["channel"] = "nightly"
 
     if kwargs["channel"]:
         channel = install.get_channel(kwargs["product"], kwargs["channel"])
         if channel is not None:
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py
@@ -333,25 +333,27 @@ class EqualTimeChunker(TestChunker):
     def __call__(self, manifest_iter):
         manifest = list(manifest_iter)
         tests = self._get_chunk(manifest)
         for item in tests:
             yield item
 
 
 class TestFilter(object):
-    def __init__(self, test_manifests, include=None, exclude=None, manifest_path=None):
-        if manifest_path is not None and include is None:
-            self.manifest = manifestinclude.get_manifest(manifest_path)
-        else:
+    def __init__(self, test_manifests, include=None, exclude=None, manifest_path=None, explicit=False):
+        if manifest_path is None or include or explicit:
             self.manifest = manifestinclude.IncludeManifest.create()
             self.manifest.set_defaults()
+        else:
+            self.manifest = manifestinclude.get_manifest(manifest_path)
+
+        if include or explicit:
+            self.manifest.set("skip", "true")
 
         if include:
-            self.manifest.set("skip", "true")
             for item in include:
                 self.manifest.add_include(test_manifests, item)
 
         if exclude:
             for item in exclude:
                 self.manifest.add_exclude(test_manifests, item)
 
     def __call__(self, manifest_iter):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -132,16 +132,21 @@ scheme host and port.""")
                                       help="URL prefix to exclude")
     test_selection_group.add_argument("--include-manifest", type=abs_path,
                                       help="Path to manifest listing tests to include")
     test_selection_group.add_argument("--skip-timeout", action="store_true",
                                       help="Skip tests that are expected to time out")
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. "
                                            "Labels starting dir: are equivalent to top-level directories.")
+    test_selection_group.add_argument("--default-exclude", action="store_true",
+                                      default=False,
+                                      help="Only run the tests explicitly given in arguments. "
+                                           "No tests will run if the list is empty, and the "
+                                           "program will exit with status code 0.")
 
     debugging_group = parser.add_argument_group("Debugging")
     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
                                  help="run under a debugger, e.g. gdb or valgrind")
     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
     debugging_group.add_argument("--rerun", action="store", type=int, default=1,
                                  help="Number of times to re run each test without restarts")
     debugging_group.add_argument("--repeat", action="store", type=int, default=1,
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py
@@ -54,21 +54,22 @@ def get_loader(test_paths, product, debu
                                     extras=run_info_extras)
 
     test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"],
                                                manifest_download=kwargs["manifest_download"]).load()
 
     manifest_filters = []
     meta_filters = []
 
-    if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
+    if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"] or kwargs["default_exclude"]:
         manifest_filters.append(testloader.TestFilter(include=kwargs["include"],
                                                       exclude=kwargs["exclude"],
                                                       manifest_path=kwargs["include_manifest"],
-                                                      test_manifests=test_manifests))
+                                                      test_manifests=test_manifests,
+                                                      explicit=kwargs["default_exclude"]))
     if kwargs["tags"]:
         meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))
 
     ssl_enabled = sslutils.get_cls(kwargs["ssl_type"]).ssl_enabled
     test_loader = testloader.TestLoader(test_manifests,
                                         kwargs["test_types"],
                                         run_info,
                                         manifest_filters=manifest_filters,
@@ -164,17 +165,17 @@ def run_tests(config, test_paths, produc
             test_source_kwargs["depth"] = kwargs["run_by_dir"]
 
         logger.info("Using %i client processes" % kwargs["processes"])
 
         skipped_tests = 0
         test_total = 0
         unexpected_total = 0
 
-        if len(test_loader.test_ids) == 0:
+        if len(test_loader.test_ids) == 0 and kwargs["test_list"]:
             logger.error("Unable to find any tests at the path(s):")
             for path in kwargs["test_list"]:
                 logger.error("  %s" % path)
             logger.error("Please check spelling and make sure there are tests in the specified path(s).")
             return False
         kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs)
 
         ssl_config = {"type": kwargs["ssl_type"],
@@ -294,18 +295,22 @@ def run_tests(config, test_paths, produc
                     break
                 if len(test_loader.test_ids) == skipped_tests:
                     break
 
     if test_total == 0:
         if skipped_tests > 0:
             logger.warning("All requested tests were skipped")
         else:
-            logger.error("No tests ran")
-            return False
+            if kwargs["default_exclude"]:
+                logger.info("No tests ran")
+                return True
+            else:
+                logger.error("No tests ran")
+                return False
 
     if unexpected_total and not kwargs["fail_on_unexpected"]:
         logger.info("Tolerating %s unexpected results" % unexpected_total)
         return True
 
     return unexpected_total == 0