Bug 1447066 [wpt PR 10098] - Exit check_stability with an error when no tests run, a=testonly
authorRobert Ma <robertma@chromium.org>
Mon, 09 Apr 2018 17:27:11 +0000
changeset 413598 9cfa39c68846284152e6b171f1f5b68a6929b59d
parent 413597 b742444ad98c9465351172e653bf38499ec0a43a
child 413599 340f28fac78a794b23dade490620ce4abaceae8c
push id33850
push userapavel@mozilla.com
push dateMon, 16 Apr 2018 09:53:48 +0000
treeherdermozilla-central@6276ec7ebbf3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1447066, 10098, 10030
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1447066 [wpt PR 10098] - Exit check_stability with an error when no tests run, a=testonly Automatic update from web-platform-testsExit check_stability with an error when no tests run Similar to #10030 (which was for `wpt run`), this commit makes `wpt check-stability` return a non-zero exit code when no tests run. Also fix some code smells: * `run` was redefined by `do_delayed_imports` to a different function. Rename the other `run` method in this module to `setup_and_run`. * `retcode` was unnecessarily defined in the top scope. Remove it to avoid redefinition. * `global logger` and the call to `get_parser` were extraneous in `run` (now `setup_and_run`) and hence are removed. -- Address comments * Renaming `check_stability.run` breaks commands.json and naming conventions, so rename the other `run` in that module instead. * Only return errors for empty results when we know for sure tests are affected. wpt-commits: 7e214e852492e619254c14c351a85ac97661d847, 3a8328470d53c4501e31cec2775c4d33821c2275 wpt-pr: 10098 wpt-commits: 7e214e852492e619254c14c351a85ac97661d847, 3a8328470d53c4501e31cec2775c4d33821c2275 wpt-pr: 10098
testing/web-platform/tests/tools/ci/check_stability.py
--- a/testing/web-platform/tests/tools/ci/check_stability.py
+++ b/testing/web-platform/tests/tools/ci/check_stability.py
@@ -17,33 +17,34 @@ from tools.wpt import testfiles
 from tools.wpt.testfiles import get_git_cmd
 from tools.wpt.virtualenv import Virtualenv
 from tools.wpt.utils import Kwargs
 from tools.wpt.run import create_parser, setup_wptrunner
 from tools.wpt import markdown
 from tools import localpaths
 
 logger = None
-run, write_inconsistent, write_results = None, None, None
+stability_run, write_inconsistent, write_results = None, None, None
 wptrunner = None
 
 def setup_logging():
     """Set up basic debug logger."""
     global logger
     logger = logging.getLogger(here)
     handler = logging.StreamHandler(sys.stdout)
     formatter = logging.Formatter(logging.BASIC_FORMAT, None)
     handler.setFormatter(formatter)
     logger.addHandler(handler)
     logger.setLevel(logging.DEBUG)
 
 
 def do_delayed_imports():
-    global run, write_inconsistent, write_results, wptrunner
-    from tools.wpt.stability import run, write_inconsistent, write_results
+    global stability_run, write_inconsistent, write_results, wptrunner
+    from tools.wpt.stability import run as stability_run
+    from tools.wpt.stability import write_inconsistent, write_results
     from wptrunner import wptrunner
 
 
 class TravisFold(object):
     """Context for TravisCI folding mechanism. Subclasses object.
 
     See: https://blog.travis-ci.com/2013-05-22-improving-build-visibility-log-folds/
     """
@@ -249,22 +250,19 @@ def main():
     venv.install_requirements(os.path.join(wpt_root, "tools", "wptrunner", "requirements.txt"))
     venv.install("requests")
 
     args, wpt_args = get_parser().parse_known_args()
     return run(venv, wpt_args, **vars(args))
 
 
 def run(venv, wpt_args, **kwargs):
-    global logger
-
     do_delayed_imports()
 
     retcode = 0
-    parser = get_parser()
 
     wpt_args = create_parser().parse_args(wpt_args)
 
     with open(kwargs["config_file"], 'r') as config_fp:
         config = SafeConfigParser()
         config.readfp(config_fp)
         skip_tests = config.get("file detection", "skip_tests").split()
         ignore_changes = set(config.get("file detection", "ignore_changes").split())
@@ -330,17 +328,17 @@ def run(venv, wpt_args, **kwargs):
         logger.info("Using binary %s" % wpt_kwargs["binary"])
 
 
     with TravisFold("running_tests"):
         logger.info("Starting tests")
 
 
         wpt_logger = wptrunner.logger
-        iterations, results, inconsistent = run(venv, wpt_logger, **wpt_kwargs)
+        iterations, results, inconsistent = stability_run(venv, wpt_logger, **wpt_kwargs)
 
     if results:
         if inconsistent:
             write_inconsistent(logger.error, inconsistent, iterations)
             retcode = 2
         else:
             logger.info("All results were stable\n")
         with TravisFold("full_results"):
@@ -348,21 +346,22 @@ def run(venv, wpt_args, **kwargs):
                           pr_number=pr_number,
                           use_details=True)
             if pr_number:
                 post_results(results, iterations=iterations, url=results_url,
                              product=wpt_args.product, pr_number=pr_number,
                              status="failed" if inconsistent else "passed")
     else:
         logger.info("No tests run.")
+        # Be conservative and only return errors when we know for sure tests are changed.
+        if tests_changed:
+            retcode = 3
 
     return retcode
 
 
 if __name__ == "__main__":
     try:
-        retcode = main()
+        sys.exit(main())
     except Exception:
         import traceback
         traceback.print_exc()
         sys.exit(1)
-    else:
-        sys.exit(retcode)