Bug 1531740 - make reftest-analyzer.xhtml stricter finding images r=dbaron
authorSam Sneddon <me@gsnedders.com>
Tue, 05 Mar 2019 00:23:49 +0000
changeset 520234 4bb1dd0270b6cf251823b865184d1dc0f3250eb6
parent 520233 a351f6e4fb9ca04267ef9e5e15d4d7a0e682ac9e
child 520235 654017ec58a77c584a0c4067a528f222e58c0f50
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdbaron
bugs1531740
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1531740 - make reftest-analyzer.xhtml stricter finding images r=dbaron Depends on D21688 Differential Revision: https://phabricator.services.mozilla.com/D21689
layout/tools/reftest/reftest-analyzer.xhtml
--- a/layout/tools/reftest/reftest-analyzer.xhtml
+++ b/layout/tools/reftest/reftest-analyzer.xhtml
@@ -220,55 +220,59 @@ var gTestItems;
 // in process_log.
 function test_parsing() {
   // Note that the logs in these testcases have been manually edited to strip
   // out stuff for brevity.
   var testcases = [
     { "name": "empty log",
       "log": "",
       "expected": { "pass": 0, "unexpected": 0, "random": 0, "skip": 0 },
+      "expected_images": 0,
     },
     { "name": "android log",
       "log": `[task 2018-12-28T10:36:45.718Z] 10:36:45     INFO -  REFTEST TEST-START | a == b
 [task 2018-12-28T10:36:45.719Z] 10:36:45     INFO -  REFTEST TEST-LOAD | a | 78 / 275 (28%)
 [task 2018-12-28T10:36:56.138Z] 10:36:56     INFO -  REFTEST TEST-LOAD | b | 78 / 275 (28%)
 [task 2018-12-28T10:37:06.559Z] 10:37:06     INFO -  REFTEST TEST-UNEXPECTED-FAIL | a == b | image comparison, max difference: 255, number of differing pixels: 5950
 [task 2018-12-28T10:37:06.568Z] 10:37:06     INFO -  REFTEST   IMAGE 1 (TEST): data:image/png;base64,
 [task 2018-12-28T10:37:06.577Z] 10:37:06     INFO -  REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,
 [task 2018-12-28T10:37:06.577Z] 10:37:06     INFO -  REFTEST INFO | Saved log: stuff trimmed here
 [task 2018-12-28T10:37:06.582Z] 10:37:06     INFO -  REFTEST TEST-END | a == b
 [task 2018-12-28T10:37:06.583Z] 10:37:06     INFO -  REFTEST TEST-START | a2 == b2
 [task 2018-12-28T10:37:06.583Z] 10:37:06     INFO -  REFTEST TEST-LOAD | a2 | 79 / 275 (28%)
 [task 2018-12-28T10:37:06.584Z] 10:37:06     INFO -  REFTEST TEST-LOAD | b2 | 79 / 275 (28%)
 [task 2018-12-28T10:37:16.982Z] 10:37:16     INFO -  REFTEST TEST-PASS | a2 == b2 | image comparison, max difference: 0, number of differing pixels: 0
 [task 2018-12-28T10:37:16.982Z] 10:37:16     INFO -  REFTEST TEST-END | a2 == b2`,
       "expected": { "pass": 1, "unexpected": 1, "random": 0, "skip": 0 },
+      "expected_images": 2,
     },
     { "name": "local reftest run (Linux)",
       "log": `REFTEST TEST-START | file:///a == file:///b
 REFTEST TEST-LOAD | file:///a | 73 / 86 (84%)
 REFTEST TEST-LOAD | file:///b | 73 / 86 (84%)
 REFTEST TEST-PASS | file:///a == file:///b | image comparison, max difference: 0, number of differing pixels: 0
 REFTEST TEST-END | file:///a == file:///b`,
       "expected": { "pass": 1, "unexpected": 0, "random": 0, "skip": 0 },
+      "expected_images": 0,
     },
     { "name": "wpt reftests (Linux automation)",
       "log": `16:50:43     INFO - TEST-START | /a
 16:50:43     INFO - PID 4276 | 1548694243694	Marionette	INFO	Testing http://web-platform.test:8000/a == http://web-platform.test:8000/b
 16:50:43     INFO - PID 4276 | 1548694243963	Marionette	INFO	No differences allowed
 16:50:44     INFO - TEST-PASS | /a | took 370ms
 16:50:44     INFO - TEST-START | /a2
 16:50:44     INFO - PID 4276 | 1548694244066	Marionette	INFO	Testing http://web-platform.test:8000/a2 == http://web-platform.test:8000/b2
 16:50:44     INFO - PID 4276 | 1548694244792	Marionette	INFO	No differences allowed
 16:50:44     INFO - PID 4276 | 1548694244792	Marionette	INFO	Found 28 pixels different, maximum difference per channel 14
 16:50:44     INFO - TEST-UNEXPECTED-FAIL | /a2 | Testing http://web-platform.test:8000/a2 == http://web-platform.test:8000/b2
 16:50:44     INFO - REFTEST   IMAGE 1 (TEST): data:image/png;base64,
 16:50:44     INFO - REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,
 16:50:44     INFO - TEST-INFO took 840ms`,
       "expected": { "pass": 1, "unexpected": 1, "random": 0, "skip": 0 },
+      "expected_images": 2,
     },
     { "name": "windows log",
       "log": `12:17:14     INFO - REFTEST TEST-START | a == b
 12:17:14     INFO - REFTEST TEST-LOAD | a | 1603 / 2053 (78%)
 12:17:14     INFO - REFTEST TEST-LOAD | b | 1603 / 2053 (78%)
 12:17:14     INFO - REFTEST TEST-PASS(EXPECTED RANDOM) | a == b | image comparison, max difference: 0, number of differing pixels: 0
 12:17:14     INFO - REFTEST TEST-END | a == b
 12:17:14     INFO - REFTEST TEST-START | a2 == b2
@@ -280,58 +284,94 @@ 12:17:14     INFO - REFTEST   IMAGE 2 (R
 12:17:14     INFO - REFTEST INFO | Saved log: stuff trimmed here
 12:17:14     INFO - REFTEST TEST-END | a2 == b2
 12:01:09     INFO - REFTEST TEST-START | a3 == b3
 12:01:09     INFO - REFTEST TEST-LOAD | a3 | 66 / 189 (34%)
 12:01:09     INFO - REFTEST TEST-LOAD | b3 | 66 / 189 (34%)
 12:01:09     INFO - REFTEST TEST-KNOWN-FAIL | a3 == b3 | image comparison, max difference: 255, number of differing pixels: 9654
 12:01:09     INFO - REFTEST TEST-END | a3 == b3`,
       "expected": { "pass": 1, "unexpected": 1, "random": 1, "skip": 0 },
+      "expected_images": 2,
     },
     { "name": "webrender wrench log (windows)",
       "log": `[task 2018-12-29T04:29:48.800Z] REFTEST a == b
 [task 2018-12-29T04:29:48.984Z] REFTEST a2 == b2
 [task 2018-12-29T04:29:49.053Z] REFTEST TEST-UNEXPECTED-FAIL | a2 == b2 | image comparison, max difference: 255, number of differing pixels: 3128
 [task 2018-12-29T04:29:49.053Z] REFTEST   IMAGE 1 (TEST): data:image/png;
 [task 2018-12-29T04:29:49.053Z] REFTEST   IMAGE 2 (REFERENCE): data:image/png;
 [task 2018-12-29T04:29:49.053Z] REFTEST TEST-END | a2 == b2`,
       "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+      "expected_images": 2,
     },
     { "name": "wpt reftests (Linux local; Bug 1530008)",
       "log": `SUITE-START | Running 1 tests
 TEST-START | /css/css-backgrounds/border-image-6.html
 TEST-UNEXPECTED-FAIL | /css/css-backgrounds/border-image-6.html | Testing http://web-platform.test:8000/css/css-backgrounds/border-image-6.html == http://web-platform.test:8000/css/css-backgrounds/border-image-6-ref.html
 REFTEST   IMAGE 1 (TEST): data:image/png;base64,
 REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,
 TEST-INFO took 425ms
 SUITE-END | took 2s`,
       "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+      "expected_images": 2,
+    },
+    { "name": "local reftest run with timestamps (Linux; Bug 1167712)",
+      "log": ` 0:05.21 REFTEST TEST-START | a
+ 0:05.21 REFTEST REFTEST TEST-LOAD | a | 0 / 1 (0%)
+ 0:05.27 REFTEST REFTEST TEST-LOAD | b | 0 / 1 (0%)
+ 0:05.66 REFTEST TEST-UNEXPECTED-FAIL | a | image comparison (==), max difference: 106, number of differing pixels: 800
+ 0:05.67 REFTEST REFTEST   IMAGE 1 (TEST): data:image/png;base64,
+ 0:05.67 REFTEST REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,
+ 0:05.73 REFTEST REFTEST TEST-END | a`,
+      "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+      "expected_images": 2,
+    },
+    { "name": "reftest run with whitespace compressed (Treeherder; Bug 1084322)",
+      "log": ` REFTEST TEST-START | a
+REFTEST TEST-LOAD | a | 0 / 1 (0%)
+REFTEST TEST-LOAD | b | 0 / 1 (0%)
+REFTEST TEST-UNEXPECTED-FAIL | a | image comparison (==), max difference: 106, number of differing pixels: 800
+REFTEST REFTEST IMAGE 1 (TEST): data:image/png;base64,
+REFTEST REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
+REFTEST REFTEST TEST-END | a`,
+      "expected": { "pass": 0, "unexpected": 1, "random": 0, "skip": 0 },
+      "expected_images": 2,
     },
   ];
 
   var current_test = 0;
 
   // Override the build_viewer function invoked at the end of process_log to
   // actually just check the results of parsing.
   build_viewer = function() {
     var expected = testcases[current_test].expected;
+    var expected_images = testcases[current_test].expected_images;
     for (var result of gTestItems) {
       for (let type in expected) { // type is "pass", "unexpected" etc.
         if (result[type]) {
           expected[type]--;
         }
       }
     }
     var failed = false;
     for (let type in expected) {
       if (expected[type] != 0) {
         console.log(`Failure: for testcase ${testcases[current_test].name} got ${expected[type]} fewer ${type} results than expected!`);
         failed = true;
       }
     }
+
+    let total_images = 0;
+    for (var result of gTestItems) {
+      total_images += result.images.length;
+    }
+    if (total_images !== expected_images) {
+      console.log(`Failure: for testcase ${testcases[current_test].name} got ${total_images} images, expected ${expected_images}`);
+      failed = true;
+    }
+
     if (!failed) {
       console.log(`Success for testcase ${testcases[current_test].name}`);
     }
   };
 
   while (current_test < testcases.length) {
     process_log(testcases[current_test].log);
     current_test++;
@@ -347,17 +387,17 @@ function process_log(contents) {
     // When making any changes to this code, please add a test to the
     // test_parsing function above, and ensure all existing tests pass.
     // !!!!!!
 
     var line = lines[j];
     // Ignore duplicated output in logcat.
     if (line.match(/I\/Gecko.*?REFTEST/))
       continue;
-    var match = line.match(/^.*?(?:REFTEST\s+)(.*)$/);
+    var match = line.match(/^.*?(?:REFTEST\s+)+(.*)$/);
     if (!match) {
       // WPT reftests don't always have the "REFTEST" prefix but do have
       // mozharness prefixing. Trying to match both prefixes optionally with a
       // single regex either makes an unreadable mess or matches everything so
       // we do them separately.
       match = line.match(/^(?:.*? (?:INFO|ERROR) -\s+)(.*)$/);
     }
     if (match)
@@ -376,17 +416,17 @@ function process_log(contents) {
           random: (random == "(EXPECTED RANDOM)"),
           skip: (extra == " (SKIP)"),
           url: url,
           images: [],
           imageLabels: []
         });
       continue;
     }
-    match = line.match(/IMAGE([^:]*): (data:.*)$/);
+    match = line.match(/^IMAGE([^:]*): (data:.*)$/);
     if (match) {
       var item = gTestItems[gTestItems.length - 1];
       item.images.push(match[2]);
       item.imageLabels.push(match[1]);
     }
   }
 
   build_viewer();