author | Nikki Sharpley <nsharpley@mozilla.com> |
Thu, 04 Jul 2019 18:20:29 +0000 | |
changeset 481316 | e5c6fb94a7eba27d4928e56b388207fd7c60ba18 |
parent 481315 | 9e1f67f66b607808fa3c4a1a879e1d36abe491f3 |
child 481317 | 14faf5e9443e08f1c9ca8a010c7e4214783116a9 |
push id | 113608 |
push user | opoprus@mozilla.com |
push date | Fri, 05 Jul 2019 06:58:54 +0000 |
treeherder | mozilla-inbound@b7030ce607ec [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | webdriver-reviewers, maja_zf |
bugs | 1558951 |
milestone | 69.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py @@ -37,16 +37,27 @@ def bool_prop(name, node): def int_prop(name, node): """Boolean property""" try: return int(node.get(name)) except KeyError: return None +def list_prop(name, node): + """List property""" + try: + list_prop = node.get(name) + if isinstance(list_prop, basestring): + return [list_prop] + return list(list_prop) + except KeyError: + return [] + + def tags(node): """Set of tags that have been applied to the test""" try: value = node.get("tags") if isinstance(value, (str, unicode)): return {value} return set(value) except KeyError: @@ -283,16 +294,24 @@ class ExpectedManifest(ManifestItem): @property def lsan_max_stack_depth(self): return int_prop("lsan-max-stack-depth", self) @property def fuzzy(self): return fuzzy_prop(self) + @property + def expected(self): + return list_prop("expected", self)[0] + + @property + def known_intermittent(self): + return list_prop("expected", self)[1:] + class DirectoryManifest(ManifestItem): @property def disabled(self): return bool_prop("disabled", self) @property def restart_after(self): @@ -410,16 +429,24 @@ class TestNode(ManifestItem): @property def lsan_max_stack_depth(self): return int_prop("lsan-max-stack-depth", self) @property def fuzzy(self): return fuzzy_prop(self) + @property + def expected(self): + return list_prop("expected", self)[0] + + @property + def known_intermittent(self): + return list_prop("expected", self)[1:] + def append(self, node): """Add a subtest to the current test :param node: AST Node associated with the subtest""" child = ManifestItem.append(self, node) self.subtests[child.name] = child def get_subtest(self, name):
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py @@ -328,20 +328,22 @@ class ExpectedUpdater(object): action_map = self.action_map action_map["suite_start"]({"run_info": data["run_info"]}) for test in data["results"]: action_map["test_start"]({"test": test["test"]}) for subtest in test["subtests"]: action_map["test_status"]({"test": test["test"], "subtest": subtest["name"], "status": subtest["status"], - "expected": subtest.get("expected")}) + "expected": subtest.get("expected"), + "known_intermittent": subtest.get("known_intermittent")}) action_map["test_end"]({"test": test["test"], "status": test["status"], - "expected": test.get("expected")}) + "expected": test.get("expected"), + "known_intermittent": test.get("known_intermittent")}) if "asserts" in test: asserts = test["asserts"] action_map["assertion_count"]({"test": test["test"], "count": asserts["count"], "min_expected": asserts["min"], "max_expected": asserts["max"]}) for item in data.get("lsan_leaks", []): action_map["lsan_leak"](item)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_wpttest.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_wpttest.py @@ -37,16 +37,42 @@ test_1 = """\ if os == 'win': FAIL """ test_2 = """\ [2.html] lsan-max-stack-depth: 42 """ +test_3 = """\ +[3.html] + [subtest1] + expected: [PASS, FAIL] + + [subtest2] + disabled: reason + + [subtest3] + expected: FAIL +""" + +test_4 = """\ +[4.html] + expected: FAIL +""" + +test_5 = """\ +[5.html] +""" + +test_6 = """\ +[6.html] + expected: [OK, FAIL] +""" + test_fuzzy = """\ [fuzzy.html] fuzzy: fuzzy-ref.html:1;200 """ testharness_test = """<script src="/resources/testharness.js"></script> <script src="/resources/testharnessreport.js"></script>""" @@ -60,103 +86,130 @@ def make_mock_manifest(*items): for test_type, dir_path, num_tests in items: for i in range(num_tests): filename = dir_path + "/%i.html" % i tests.append((test_type, filename, {TestharnessTest("/foo.bar", filename, "/", filename)})) return rv +def make_test_object(test_name, + test_path, + index, + items, + inherit_metadata=None, + iterate=False, + condition=None): + inherit_metadata = inherit_metadata if inherit_metadata is not None else [] + condition = condition if condition is not None else {} + tests = make_mock_manifest(*items) if isinstance(items, list) else make_mock_manifest(items) + + test_metadata = manifestexpected.static.compile(BytesIO(test_name), + condition, + data_cls_getter=manifestexpected.data_cls_getter, + test_path=test_path, + url_base="/") + + test = next(iter(tests[index][2])) if iterate else tests[index][2].pop() + return wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id)) + @pytest.mark.xfail(sys.version[0] == "3", reason="bytes/text confusion in py3") def test_metadata_inherit(): - tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10), - ("test", "c", 10)) - + items = [("test", "a", 10), ("test", "a/b", 10), ("test", "c", 10)] inherit_metadata = [ manifestexpected.static.compile( BytesIO(item), {}, data_cls_getter=lambda x,y: manifestexpected.DirectoryManifest) for item in [dir_ini_0, dir_ini_1]] - test_metadata = manifestexpected.static.compile(BytesIO(test_0), - {}, - data_cls_getter=manifestexpected.data_cls_getter, - test_path="a/0.html", - url_base="/") - test = next(iter(tests[0][2])) - test_obj = wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id)) + test_obj = make_test_object(test_0, "a/0.html", 0, items, inherit_metadata, True) + assert test_obj.max_assertion_count == 3 assert test_obj.min_assertion_count == 1 assert test_obj.prefs == {"b": "c", "c": "d"} assert test_obj.tags == {"a", "dir:a"} @pytest.mark.xfail(sys.version[0] == "3", reason="bytes/text confusion in py3") def test_conditional(): - tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10), - ("test", "c", 10)) + items = [("test", "a", 10), ("test", "a/b", 10), ("test", "c", 10)] - test_metadata = manifestexpected.static.compile(BytesIO(test_1), - {"os": "win"}, - data_cls_getter=manifestexpected.data_cls_getter, - test_path="a/1.html", - url_base="/") + test_obj = make_test_object(test_1, "a/1.html", 1, items, None, True, {"os": "win"}) - test = next(iter(tests[1][2])) - test_obj = wpttest.from_manifest(tests, test, [], test_metadata.get_test(test.id)) assert test_obj.prefs == {"a": "b", "c": "d"} assert test_obj.expected() == "FAIL" @pytest.mark.xfail(sys.version[0] == "3", reason="bytes/text confusion in py3") def test_metadata_lsan_stack_depth(): - tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10)) + items = [("test", "a", 10), ("test", "a/b", 10)] - test_metadata = manifestexpected.static.compile(BytesIO(test_2), - {}, - data_cls_getter=manifestexpected.data_cls_getter, - test_path="a/2.html", - url_base="/") - - test = next(iter(tests[2][2])) - test_obj = wpttest.from_manifest(tests, test, [], test_metadata.get_test(test.id)) + test_obj = make_test_object(test_2, "a/2.html", 2, items, None, True) assert test_obj.lsan_max_stack_depth == 42 - test = next(iter(tests[1][2])) - test_obj = wpttest.from_manifest(tests, test, [], test_metadata.get_test(test.id)) + test_obj = make_test_object(test_2, "a/2.html", 1, items, None, True) assert test_obj.lsan_max_stack_depth is None - test_metadata = manifestexpected.static.compile(BytesIO(test_0), - {}, - data_cls_getter=manifestexpected.data_cls_getter, - test_path="a/0.html", - url_base="/") - inherit_metadata = [ manifestexpected.static.compile( BytesIO(dir_ini_2), {}, data_cls_getter=lambda x,y: manifestexpected.DirectoryManifest) ] - test = tests[0][2].pop() - test_obj = wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id)) + test_obj = make_test_object(test_0, "a/0/html", 0, items, inherit_metadata, False) assert test_obj.lsan_max_stack_depth == 42 @pytest.mark.xfail(sys.version[0] == "3", reason="bytes/text confusion in py3") +def test_subtests(): + test_obj = make_test_object(test_3, "a/3.html", 3, ("test", "a", 4), None, False) + assert test_obj.expected("subtest1") == "PASS" + assert test_obj.known_intermittent("subtest1") == ["FAIL"] + assert test_obj.expected("subtest2") == "PASS" + assert test_obj.known_intermittent("subtest2") == [] + assert test_obj.expected("subtest3") == "FAIL" + assert test_obj.known_intermittent("subtest3") == [] + + +@pytest.mark.xfail(sys.version[0] == "3", + reason="bytes/text confusion in py3") +def test_expected_fail(): + test_obj = make_test_object(test_4, "a/4.html", 4, ("test", "a", 5), None, False) + assert test_obj.expected() == "FAIL" + assert test_obj.known_intermittent() == [] + + +@pytest.mark.xfail(sys.version[0] == "3", + reason="bytes/text confusion in py3") +def test_no_expected(): + test_obj = make_test_object(test_5, "a/5.html", 5, ("test", "a", 6), None, False) + assert test_obj.expected() == "OK" + assert test_obj.known_intermittent() == [] + + +@pytest.mark.xfail(sys.version[0] == "3", + reason="bytes/text confusion in py3") +def test_known_intermittent(): + test_obj = make_test_object(test_6, "a/6.html", 6, ("test", "a", 7), None, False) + assert test_obj.expected() == "OK" + assert test_obj.known_intermittent() == ["FAIL"] + + +@pytest.mark.xfail(sys.version[0] == "3", + reason="bytes/text confusion in py3") def test_metadata_fuzzy(): manifest_data = { "items": {"reftest": {"a/fuzzy.html": [["a/fuzzy.html", [["/a/fuzzy-ref.html", "=="]], {"fuzzy": [[["/a/fuzzy.html", '/a/fuzzy-ref.html', '=='], [[2, 3], [10, 15]]]]}]]}}, "paths": {"a/fuzzy.html": ["0"*40, "reftest"]}, "version": 6,
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py @@ -5,38 +5,46 @@ from collections import defaultdict from .wptmanifest.parser import atoms atom_reset = atoms["Reset"] enabled_tests = {"testharness", "reftest", "wdspec"} class Result(object): - def __init__(self, status, message, expected=None, extra=None, stack=None): + def __init__(self, + status, + message, + expected=None, + extra=None, + stack=None, + known_intermittent=None): if status not in self.statuses: raise ValueError("Unrecognised status %s" % status) self.status = status self.message = message self.expected = expected + self.known_intermittent = known_intermittent if known_intermittent is not None else [] self.extra = extra if extra is not None else {} self.stack = stack def __repr__(self): return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status) class SubtestResult(object): - def __init__(self, name, status, message, stack=None, expected=None): + def __init__(self, name, status, message, stack=None, expected=None, known_intermittent=None): self.name = name if status not in self.statuses: raise ValueError("Unrecognised status %s" % status) self.status = status self.message = message self.stack = stack self.expected = expected + self.known_intermittent = known_intermittent if known_intermittent is not None else [] def __repr__(self): return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status) class TestharnessResult(Result): default_expected = "OK" statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"} @@ -299,20 +307,39 @@ class Test(object): else: default = self.subtest_result_cls.default_expected metadata = self._get_metadata(subtest) if metadata is None: return default try: - return metadata.get("expected") + expected = metadata.get("expected") + if isinstance(expected, (basestring)): + return expected + elif isinstance(expected, list): + return expected[0] + elif expected is None: + return default except KeyError: return default + def known_intermittent(self, subtest=None): + metadata = self._get_metadata(subtest) + if metadata is None: + return [] + + try: + expected = metadata.get("expected") + if isinstance(expected, list): + return expected[1:] + return [] + except KeyError: + return [] + def __repr__(self): return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id) class TestharnessTest(Test): result_cls = TestharnessResult subtest_result_cls = TestharnessSubtestResult test_type = "testharness"