Bug 1555014 - Update wptreport formatter to support new mozlog test field; known_intermittent r=jgraham
authorNikki Sharpley <nsharpley@mozilla.com>
Wed, 05 Jun 2019 20:27:34 +0000
changeset 480026 0e93c74963f6e8a0d48d5b79318fde292a80c676
parent 480025 aebef74ae4cf423ccc5b0a53a84584f3f839c7f4
child 480027 8c6b67d82222b5aabea4da0add5ceef2aff948ed
push id36200
push useraiakab@mozilla.com
push dateTue, 25 Jun 2019 21:58:14 +0000
treeherdermozilla-central@b23bd78e4d94 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1555014
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1555014 - Update wptreport formatter to support new mozlog test field; known_intermittent r=jgraham Added support for the new known_intermittent test field in mozlog.structured to the WptreportFormatter class. A test was added to check this field is recognised. The mozlog version was updated in the requirements.txt for wpt. Capture was added to the mozlog init file, so that the module can be used by mozlog consumers, such as wpt. Differential Revision: https://phabricator.services.mozilla.com/D33673
testing/web-platform/tests/tools/wptrunner/wptrunner/formatters/wptreport.py
testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_formatters.py
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/formatters/wptreport.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/formatters/wptreport.py
@@ -99,26 +99,30 @@ class WptreportFormatter(BaseFormatter):
 
         return subtest
 
     def test_status(self, data):
         subtest = self.create_subtest(data)
         subtest["status"] = data["status"]
         if "expected" in data:
             subtest["expected"] = data["expected"]
+        if "known_intermittent" in data:
+            subtest["known_intermittent"] = data["known_intermittent"]
         if "message" in data:
             subtest["message"] = replace_lone_surrogate(data["message"])
 
     def test_end(self, data):
         test = self.find_or_create_test(data)
         start_time = test.pop("start_time")
         test["duration"] = data["time"] - start_time
         test["status"] = data["status"]
         if "expected" in data:
             test["expected"] = data["expected"]
+        if "known_intermittent" in data:
+            test["known_intermittent"] = data["known_intermittent"]
         if "message" in data:
             test["message"] = replace_lone_surrogate(data["message"])
         if "reftest_screenshots" in data.get("extra", {}):
             test["screenshots"] = {
                 strip_server(item["url"]): "sha1:" + item["hash"]
                 for item in data["extra"]["reftest_screenshots"]
                 if type(item) == dict
             }
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_formatters.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_formatters.py
@@ -99,17 +99,19 @@ def test_wptreport_lone_surrogate(capfd)
 def test_wptreport_lone_surrogate_ucs2(capfd):
     # Since UCS4 is a superset of UCS2 we can meaningfully test the UCS2 code on a
     # UCS4 build, but not the reverse. However UCS2 is harder to handle and UCS4 is
     # the commonest (and sensible) configuration, so that's OK.
     output = StringIO()
     logger = structuredlog.StructuredLogger("test_a")
     logger.add_handler(handlers.StreamHandler(output, WptreportFormatter()))
 
-    with mock.patch.object(wptreport, 'surrogate_replacement', wptreport.SurrogateReplacementUcs2()):
+    with mock.patch.object(wptreport,
+                           'surrogate_replacement',
+                           wptreport.SurrogateReplacementUcs2()):
         # output a bunch of stuff
         logger.suite_start(["test-id-1"])  # no run_info arg!
         logger.test_start("test-id-1")
         logger.test_status("test-id-1",
                            subtest=u"Name with surrogate\uD800",
                            status="FAIL",
                            message=u"\U0001F601 \uDE0A\uD83D \uD83D\uDE0A")
         logger.test_end("test-id-1",
@@ -126,8 +128,41 @@ def test_wptreport_lone_surrogate_ucs2(c
     # check the actual output of the formatter
     output.seek(0)
     output_obj = json.load(output)
     test = output_obj["results"][0]
     assert test["message"] == u"U+de0aU+d83d \U0001f60a \U0001F601"
     subtest = test["subtests"][0]
     assert subtest["name"] == u"Name with surrogateU+d800"
     assert subtest["message"] == u"\U0001F601 U+de0aU+d83d \U0001f60a"
+
+
+def test_wptreport_known_intermittent(capfd):
+    output = StringIO()
+    logger = structuredlog.StructuredLogger("test_a")
+    logger.add_handler(handlers.StreamHandler(output, WptreportFormatter()))
+
+    # output a bunch of stuff
+    logger.suite_start(["test-id-1"])  # no run_info arg!
+    logger.test_start("test-id-1")
+    logger.test_status("test-id-1",
+                       "a-subtest",
+                       status="FAIL",
+                       expected="PASS",
+                       known_intermittent=["FAIL"])
+    logger.test_end("test-id-1",
+                    status="OK",)
+    logger.suite_end()
+
+    # check nothing got output to stdout/stderr
+    # (note that mozlog outputs exceptions during handling to stderr!)
+    captured = capfd.readouterr()
+    assert captured.out == ""
+    assert captured.err == ""
+
+    # check the actual output of the formatter
+    output.seek(0)
+    output_obj = json.load(output)
+    test = output_obj["results"][0]
+    assert test["status"] == u"OK"
+    subtest = test["subtests"][0]
+    assert subtest["expected"] == u"PASS"
+    assert subtest["known_intermittent"] == [u'FAIL']