Bug 1558951 - Add support to wptrunner for allowed known_intermittent statuses r=webdriver-reviewers,maja_zf
authorNikki Sharpley <nsharpley@mozilla.com>
Thu, 04 Jul 2019 18:20:29 +0000
changeset 481316 e5c6fb94a7eba27d4928e56b388207fd7c60ba18
parent 481315 9e1f67f66b607808fa3c4a1a879e1d36abe491f3
child 481317 14faf5e9443e08f1c9ca8a010c7e4214783116a9
push id113608
push useropoprus@mozilla.com
push dateFri, 05 Jul 2019 06:58:54 +0000
treeherdermozilla-inbound@b7030ce607ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswebdriver-reviewers, maja_zf
bugs1558951
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1558951 - Add support to wptrunner for allowed known_intermittent statuses r=webdriver-reviewers,maja_zf This patch adds initial support for multiple allowed expected statuses in the test metadata. A `known_intermittent` property has been add to the ExpectedManifest, TestNode and SubtestNode objects. A `known_intermittent()`, similar to the `expected()` has been added to the Test class. This defines an expected status as the first status in the list if `expected` value is a list, simply the value of `expected` if it is a string. A test has been written for wpttest.py to reflect these changes. Where mozlog is used, `known_intermittent` has been included in the expected log. Differential Revision: https://phabricator.services.mozilla.com/D35363
testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_wpttest.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py
@@ -37,16 +37,27 @@ def bool_prop(name, node):
 def int_prop(name, node):
     """Boolean property"""
     try:
         return int(node.get(name))
     except KeyError:
         return None
 
 
+def list_prop(name, node):
+    """List property"""
+    try:
+        list_prop = node.get(name)
+        if isinstance(list_prop, basestring):
+            return [list_prop]
+        return list(list_prop)
+    except KeyError:
+        return []
+
+
 def tags(node):
     """Set of tags that have been applied to the test"""
     try:
         value = node.get("tags")
         if isinstance(value, (str, unicode)):
             return {value}
         return set(value)
     except KeyError:
@@ -283,16 +294,24 @@ class ExpectedManifest(ManifestItem):
     @property
     def lsan_max_stack_depth(self):
         return int_prop("lsan-max-stack-depth", self)
 
     @property
     def fuzzy(self):
         return fuzzy_prop(self)
 
+    @property
+    def expected(self):
+        return list_prop("expected", self)[0]
+
+    @property
+    def known_intermittent(self):
+        return list_prop("expected", self)[1:]
+
 
 class DirectoryManifest(ManifestItem):
     @property
     def disabled(self):
         return bool_prop("disabled", self)
 
     @property
     def restart_after(self):
@@ -410,16 +429,24 @@ class TestNode(ManifestItem):
     @property
     def lsan_max_stack_depth(self):
         return int_prop("lsan-max-stack-depth", self)
 
     @property
     def fuzzy(self):
         return fuzzy_prop(self)
 
+    @property
+    def expected(self):
+        return list_prop("expected", self)[0]
+
+    @property
+    def known_intermittent(self):
+        return list_prop("expected", self)[1:]
+
     def append(self, node):
         """Add a subtest to the current test
 
         :param node: AST Node associated with the subtest"""
         child = ManifestItem.append(self, node)
         self.subtests[child.name] = child
 
     def get_subtest(self, name):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py
@@ -328,20 +328,22 @@ class ExpectedUpdater(object):
         action_map = self.action_map
         action_map["suite_start"]({"run_info": data["run_info"]})
         for test in data["results"]:
             action_map["test_start"]({"test": test["test"]})
             for subtest in test["subtests"]:
                 action_map["test_status"]({"test": test["test"],
                                            "subtest": subtest["name"],
                                            "status": subtest["status"],
-                                           "expected": subtest.get("expected")})
+                                           "expected": subtest.get("expected"),
+                                           "known_intermittent": subtest.get("known_intermittent")})
             action_map["test_end"]({"test": test["test"],
                                     "status": test["status"],
-                                    "expected": test.get("expected")})
+                                    "expected": test.get("expected"),
+                                    "known_intermittent": test.get("known_intermittent")})
             if "asserts" in test:
                 asserts = test["asserts"]
                 action_map["assertion_count"]({"test": test["test"],
                                                "count": asserts["count"],
                                                "min_expected": asserts["min"],
                                                "max_expected": asserts["max"]})
         for item in data.get("lsan_leaks", []):
             action_map["lsan_leak"](item)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_wpttest.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_wpttest.py
@@ -37,16 +37,42 @@ test_1 = """\
     if os == 'win': FAIL
 """
 
 test_2 = """\
 [2.html]
   lsan-max-stack-depth: 42
 """
 
+test_3 = """\
+[3.html]
+  [subtest1]
+    expected: [PASS, FAIL]
+
+  [subtest2]
+    disabled: reason
+
+  [subtest3]
+    expected: FAIL
+"""
+
+test_4 = """\
+[4.html]
+  expected: FAIL
+"""
+
+test_5 = """\
+[5.html]
+"""
+
+test_6 = """\
+[6.html]
+  expected: [OK, FAIL]
+"""
+
 test_fuzzy = """\
 [fuzzy.html]
   fuzzy: fuzzy-ref.html:1;200
 """
 
 
 testharness_test = """<script src="/resources/testharness.js"></script>
 <script src="/resources/testharnessreport.js"></script>"""
@@ -60,103 +86,130 @@ def make_mock_manifest(*items):
     for test_type, dir_path, num_tests in items:
         for i in range(num_tests):
             filename = dir_path + "/%i.html" % i
             tests.append((test_type,
                           filename,
                           {TestharnessTest("/foo.bar", filename, "/", filename)}))
     return rv
 
+def make_test_object(test_name,
+                     test_path,
+                     index,
+                     items,
+                     inherit_metadata=None,
+                     iterate=False,
+                     condition=None):
+    inherit_metadata = inherit_metadata if inherit_metadata is not None else []
+    condition = condition if condition is not None else {}
+    tests = make_mock_manifest(*items) if isinstance(items, list) else make_mock_manifest(items)
+
+    test_metadata = manifestexpected.static.compile(BytesIO(test_name),
+                                                    condition,
+                                                    data_cls_getter=manifestexpected.data_cls_getter,
+                                                    test_path=test_path,
+                                                    url_base="/")
+
+    test = next(iter(tests[index][2])) if iterate else tests[index][2].pop()
+    return wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id))
+
 
 @pytest.mark.xfail(sys.version[0] == "3",
                    reason="bytes/text confusion in py3")
 def test_metadata_inherit():
-    tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
-                               ("test", "c", 10))
-
+    items = [("test", "a", 10), ("test", "a/b", 10), ("test", "c", 10)]
     inherit_metadata = [
         manifestexpected.static.compile(
             BytesIO(item),
             {},
             data_cls_getter=lambda x,y: manifestexpected.DirectoryManifest)
         for item in [dir_ini_0, dir_ini_1]]
-    test_metadata = manifestexpected.static.compile(BytesIO(test_0),
-                                                    {},
-                                                    data_cls_getter=manifestexpected.data_cls_getter,
-                                                    test_path="a/0.html",
-                                                    url_base="/")
 
-    test = next(iter(tests[0][2]))
-    test_obj = wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id))
+    test_obj = make_test_object(test_0, "a/0.html", 0, items, inherit_metadata, True)
+
     assert test_obj.max_assertion_count == 3
     assert test_obj.min_assertion_count == 1
     assert test_obj.prefs == {"b": "c", "c": "d"}
     assert test_obj.tags == {"a", "dir:a"}
 
 
 @pytest.mark.xfail(sys.version[0] == "3",
                    reason="bytes/text confusion in py3")
 def test_conditional():
-    tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
-                               ("test", "c", 10))
+    items = [("test", "a", 10), ("test", "a/b", 10), ("test", "c", 10)]
 
-    test_metadata = manifestexpected.static.compile(BytesIO(test_1),
-                                                    {"os": "win"},
-                                                    data_cls_getter=manifestexpected.data_cls_getter,
-                                                    test_path="a/1.html",
-                                                    url_base="/")
+    test_obj = make_test_object(test_1, "a/1.html", 1, items, None, True, {"os": "win"})
 
-    test = next(iter(tests[1][2]))
-    test_obj = wpttest.from_manifest(tests, test, [], test_metadata.get_test(test.id))
     assert test_obj.prefs == {"a": "b", "c": "d"}
     assert test_obj.expected() == "FAIL"
 
 
 @pytest.mark.xfail(sys.version[0] == "3",
                    reason="bytes/text confusion in py3")
 def test_metadata_lsan_stack_depth():
-    tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10))
+    items = [("test", "a", 10), ("test", "a/b", 10)]
 
-    test_metadata = manifestexpected.static.compile(BytesIO(test_2),
-                                                    {},
-                                                    data_cls_getter=manifestexpected.data_cls_getter,
-                                                    test_path="a/2.html",
-                                                    url_base="/")
-
-    test = next(iter(tests[2][2]))
-    test_obj = wpttest.from_manifest(tests, test, [], test_metadata.get_test(test.id))
+    test_obj = make_test_object(test_2, "a/2.html", 2, items, None, True)
 
     assert test_obj.lsan_max_stack_depth == 42
 
-    test = next(iter(tests[1][2]))
-    test_obj = wpttest.from_manifest(tests, test, [], test_metadata.get_test(test.id))
+    test_obj = make_test_object(test_2, "a/2.html", 1, items, None, True)
 
     assert test_obj.lsan_max_stack_depth is None
 
-    test_metadata = manifestexpected.static.compile(BytesIO(test_0),
-                                                    {},
-                                                    data_cls_getter=manifestexpected.data_cls_getter,
-                                                    test_path="a/0.html",
-                                                    url_base="/")
-
     inherit_metadata = [
         manifestexpected.static.compile(
             BytesIO(dir_ini_2),
             {},
             data_cls_getter=lambda x,y: manifestexpected.DirectoryManifest)
     ]
 
-    test = tests[0][2].pop()
-    test_obj = wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id))
+    test_obj = make_test_object(test_0, "a/0/html", 0, items, inherit_metadata, False)
 
     assert test_obj.lsan_max_stack_depth == 42
 
 
 @pytest.mark.xfail(sys.version[0] == "3",
                    reason="bytes/text confusion in py3")
+def test_subtests():
+    test_obj = make_test_object(test_3, "a/3.html", 3, ("test", "a", 4), None, False)
+    assert test_obj.expected("subtest1") == "PASS"
+    assert test_obj.known_intermittent("subtest1") == ["FAIL"]
+    assert test_obj.expected("subtest2") == "PASS"
+    assert test_obj.known_intermittent("subtest2") == []
+    assert test_obj.expected("subtest3") == "FAIL"
+    assert test_obj.known_intermittent("subtest3") == []
+
+
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
+def test_expected_fail():
+    test_obj = make_test_object(test_4, "a/4.html", 4, ("test", "a", 5), None, False)
+    assert test_obj.expected() == "FAIL"
+    assert test_obj.known_intermittent() == []
+
+
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
+def test_no_expected():
+    test_obj = make_test_object(test_5, "a/5.html", 5, ("test", "a", 6), None, False)
+    assert test_obj.expected() == "OK"
+    assert test_obj.known_intermittent() == []
+
+
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
+def test_known_intermittent():
+    test_obj = make_test_object(test_6, "a/6.html", 6, ("test", "a", 7), None, False)
+    assert test_obj.expected() == "OK"
+    assert test_obj.known_intermittent() == ["FAIL"]
+
+
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
 def test_metadata_fuzzy():
     manifest_data = {
         "items": {"reftest": {"a/fuzzy.html": [["a/fuzzy.html",
                                                 [["/a/fuzzy-ref.html", "=="]],
                                                 {"fuzzy": [[["/a/fuzzy.html", '/a/fuzzy-ref.html', '=='],
                                                             [[2, 3], [10, 15]]]]}]]}},
         "paths": {"a/fuzzy.html": ["0"*40, "reftest"]},
         "version": 6,
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py
@@ -5,38 +5,46 @@ from collections import defaultdict
 
 from .wptmanifest.parser import atoms
 
 atom_reset = atoms["Reset"]
 enabled_tests = {"testharness", "reftest", "wdspec"}
 
 
 class Result(object):
-    def __init__(self, status, message, expected=None, extra=None, stack=None):
+    def __init__(self,
+                 status,
+                 message,
+                 expected=None,
+                 extra=None,
+                 stack=None,
+                 known_intermittent=None):
         if status not in self.statuses:
             raise ValueError("Unrecognised status %s" % status)
         self.status = status
         self.message = message
         self.expected = expected
+        self.known_intermittent = known_intermittent if known_intermittent is not None else []
         self.extra = extra if extra is not None else {}
         self.stack = stack
 
     def __repr__(self):
         return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status)
 
 
 class SubtestResult(object):
-    def __init__(self, name, status, message, stack=None, expected=None):
+    def __init__(self, name, status, message, stack=None, expected=None, known_intermittent=None):
         self.name = name
         if status not in self.statuses:
             raise ValueError("Unrecognised status %s" % status)
         self.status = status
         self.message = message
         self.stack = stack
         self.expected = expected
+        self.known_intermittent = known_intermittent if known_intermittent is not None else []
 
     def __repr__(self):
         return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status)
 
 
 class TestharnessResult(Result):
     default_expected = "OK"
     statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"}
@@ -299,20 +307,39 @@ class Test(object):
         else:
             default = self.subtest_result_cls.default_expected
 
         metadata = self._get_metadata(subtest)
         if metadata is None:
             return default
 
         try:
-            return metadata.get("expected")
+            expected = metadata.get("expected")
+            if isinstance(expected, (basestring)):
+                return expected
+            elif isinstance(expected, list):
+                return expected[0]
+            elif expected is None:
+                return default
         except KeyError:
             return default
 
+    def known_intermittent(self, subtest=None):
+        metadata = self._get_metadata(subtest)
+        if metadata is None:
+            return []
+
+        try:
+            expected = metadata.get("expected")
+            if isinstance(expected, list):
+                return expected[1:]
+            return []
+        except KeyError:
+            return []
+
     def __repr__(self):
         return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
 
 
 class TestharnessTest(Test):
     result_cls = TestharnessResult
     subtest_result_cls = TestharnessSubtestResult
     test_type = "testharness"