Bug 1493384 [wpt PR 13158] - Also ignore ERROR in slow test check, a=testonly
authorRobert Ma <robertma@chromium.org>
Wed, 26 Sep 2018 10:47:17 +0000
changeset 486815 92cd7b62a53b6717d7fbe42201e750a7801c48fa
parent 486814 75d065357544a5564e20d80d52b52f2bda23656f
child 486816 b3a73061106471ca8b11746656a430161e9bbeca
push id246
push userfmarier@mozilla.com
push dateSat, 13 Oct 2018 00:15:40 +0000
reviewerstestonly
bugs1493384, 13158
milestone64.0a1
Bug 1493384 [wpt PR 13158] - Also ignore ERROR in slow test check, a=testonly Automatic update from web-platform-testsAlso ignore ERROR in slow test check Tweak the docstring accordingly to explain why. -- wpt-commits: 130669061c6f4b475e1189ef107010966d686324 wpt-pr: 13158
testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_stability.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/stability.py
@@ -110,27 +110,28 @@ def is_inconsistent(results_dict, iterat
         return False
     return len(results_dict) > 1 or sum(results_dict.values()) != iterations
 
 
 def find_slow_status(test):
     """Check if a single test almost times out.
 
     We are interested in tests that almost time out (i.e. likely to be flaky).
-    Therefore, timeout statuses are ignored, including (EXTERNAL-)TIMEOUT &
-    CRASH (tests that time out may be marked as CRASH if crashes are detected).
+    Therefore, timeout statuses are ignored, including (EXTERNAL-)TIMEOUT.
+    CRASH & ERROR are also ignored because the they override TIMEOUT; a test
+    that both crashes and times out is marked as CRASH, so it won't be flaky.
 
     Returns:
         A result status produced by a run that almost times out; None, if no
         runs almost time out.
     """
     if "timeout" not in test:
         return None
     threshold = test["timeout"] * FLAKY_THRESHOLD
-    for status in ['PASS', 'FAIL', 'OK', 'ERROR']:
+    for status in ['PASS', 'FAIL', 'OK']:
         if (status in test["longest_duration"] and
             test["longest_duration"][status] > threshold):
             return status
     return None
 
 
 def process_results(log, iterations):
     """Process test log and return overall results and list of inconsistent tests."""
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_stability.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_stability.py
@@ -16,16 +16,19 @@ def test_is_inconsistent():
 def test_find_slow_status():
     assert stability.find_slow_status({
         "longest_duration": {"TIMEOUT": 10},
         "timeout": 10}) is None
     assert stability.find_slow_status({
         "longest_duration": {"CRASH": 10},
         "timeout": 10}) is None
     assert stability.find_slow_status({
+        "longest_duration": {"ERROR": 10},
+        "timeout": 10}) is None
+    assert stability.find_slow_status({
         "longest_duration": {"PASS": 1},
         "timeout": 10}) is None
     assert stability.find_slow_status({
         "longest_duration": {"PASS": 81},
         "timeout": 100}) == "PASS"
     assert stability.find_slow_status({
         "longest_duration": {"TIMEOUT": 10, "FAIL": 81},
         "timeout": 100}) == "FAIL"