Bug 1474569 - Improve output for failing wpt tests in jstests harness; r=bbouvier
authorMs2ger <Ms2ger@igalia.com>
Wed, 11 Jul 2018 17:52:36 +0200
changeset 426266 9d11c83f1d68ad7915d7bed042347c47bd5d54c1
parent 426265 72443d5dcf092154dee2955888f39d0eba19e265
child 426267 396805ee5355ec5909d856975019004d8d3a6a17
push id105188
push userMs2ger@gmail.com
push dateThu, 12 Jul 2018 08:12:08 +0000
treeherdermozilla-inbound@396805ee5355 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbbouvier
bugs1474569
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1474569 - Improve output for failing wpt tests in jstests harness; r=bbouvier
js/src/tests/lib/results.py
--- a/js/src/tests/lib/results.py
+++ b/js/src/tests/lib/results.py
@@ -65,49 +65,65 @@ class TestResult:
     @classmethod
     def from_wpt_output(cls, output):
         """Parse the output from a web-platform test that uses testharness.js.
         (The output is written to stdout in js/src/tests/testharnessreport.js.)
         """
         from wptrunner.executors.base import testharness_result_converter
 
         rc = output.rc
+        stdout = output.out.split("\n")
         if rc != 0:
             if rc == 3:
                 harness_status = "ERROR"
+                harness_message = "Exit code reported exception"
             else:
                 harness_status = "CRASH"
+                harness_message = "Exit code reported crash"
             tests = []
         else:
-            for line in output.out.split("\n"):
+            for (idx, line) in enumerate(stdout):
                 if line.startswith("WPT OUTPUT: "):
                     msg = line[len("WPT OUTPUT: "):]
                     data = [output.test.wpt.url] + json.loads(msg)
                     harness_status_obj, tests = testharness_result_converter(output.test.wpt, data)
                     harness_status = harness_status_obj.status
+                    harness_message = "Reported by harness: %s" % (harness_status_obj.message,)
+                    del stdout[idx]
                     break
             else:
                 harness_status = "ERROR"
+                harness_message = "No harness output found"
                 tests = []
+        stdout.append("Harness status: %s (%s)" % (harness_status, harness_message))
 
         result = cls.PASS
         results = []
         if harness_status != output.test.wpt.expected():
             if harness_status == "CRASH":
                 result = cls.CRASH
             else:
                 result = cls.FAIL
         else:
             for test in tests:
-                if test.status == output.test.wpt.expected(test.name):
+                test_output = "Subtest \"%s\": " % (test.name,)
+                expected = output.test.wpt.expected(test.name)
+                if test.status == expected:
                     test_result = (cls.PASS, "")
+                    test_output += "as expected: %s" % (test.status,)
                 else:
                     test_result = (cls.FAIL, test.message)
                     result = cls.FAIL
+                    test_output += "expected %s, found %s" % (expected, test.status)
+                    if test.message:
+                        test_output += " (with message: \"%s\")" % (test.message,)
                 results.append(test_result)
+                stdout.append(test_output)
+
+        output.out = "\n".join(stdout) + "\n"
 
         return cls(output.test, result, results)
 
     @classmethod
     def from_output(cls, output):
         test = output.test
         result = None          # str:      overall result, see class-level variables
         results = []           # (str,str) list: subtest results (pass/fail, message)