Bug 443090 - Unify unittest failures (reftest part) p=ted.mielczarek, r=dbaron
authorNick Thomas <nrthomas@gmail.com>
Wed, 16 Jul 2008 17:47:12 +1200
changeset 15963 03a87160d7a20bbb10d689fc1dfb3bcd8ca8ec4a
parent 15962 d3f2561990ab0bea2190664b9ddf7c548cca06fa
child 15964 2beee8ac413e0754ee47255dfb6c1c53196ae323
push idunknown
push userunknown
push dateunknown
reviewersdbaron
bugs443090
milestone1.9.1a1pre
Bug 443090 - Unify unittest failures (reftest part) p=ted.mielczarek, r=dbaron
layout/tools/reftest/clean-reftest-output.pl
layout/tools/reftest/reftest-analyzer.xhtml
layout/tools/reftest/reftest-to-html.pl
layout/tools/reftest/reftest.js
--- a/layout/tools/reftest/clean-reftest-output.pl
+++ b/layout/tools/reftest/clean-reftest-output.pl
@@ -51,17 +51,17 @@ print <<EOM
 <pre>
 EOM
 ;
 
 while (<>) {
     next unless /REFTEST/;
     chomp;
     chop if /\r$/;
-    s,(PASS|FAIL)([^:]*):( \(!=\))? (.*),\1\2:\3 <a href="\4">\4</a>,;
+    s,(TEST-)([^\|]*) \| ([^\|]*) \|(.*),\1\2: <a href="\3">\3</a>\4,;
     s,(IMAGE[^:]*): (data:.*),<a href="\2">\1</a>,;
     print;
     print "\n";
 }
 
 print <<EOM
 </pre>
 </body>
--- a/layout/tools/reftest/reftest-analyzer.xhtml
+++ b/layout/tools/reftest/reftest-analyzer.xhtml
@@ -136,27 +136,28 @@ function process_log(contents) {
 	var lines = contents.split(/[\r\n]+/);
 	gTestItems = [];
 	for (var j in lines) {
 		var line = lines[j];
 		var match = line.match(/^REFTEST (.*)$/);
 		if (!match)
 			continue;
 		line = match[1];
-		match = line.match(/^(PASS|UNEXPECTED PASS|KNOWN FAIL|UNEXPECTED FAIL)( \(RESULT EXPECTED TO BE RANDOM\)| \(SKIP\)|): (.*)/);
+		match = line.match(/^(TEST-PASS|TEST-UNEXPECTED-PASS|TEST-KNOWN-FAIL|TEST-UNEXPECTED-FAIL)(\(EXPECTED RANDOM\)|) \| ([^\|]+) \|(.*)/);
 		if (match) {
 			var state = match[1];
-			var extra = match[2];
+			var random = match[2];
 			var url = match[3];
+                        var extra = match[4];
 			gTestItems.push(
 				{
 					pass: !state.match(/FAIL$/),
 					// only one of the following three should ever be true
-					unexpected: !!state.match(/^UNEXPECTED/),
-					random: (extra == " (RESULT EXPECTED TO BE RANDOM)"),
+					unexpected: !!state.match(/^TEST-UNEXPECTED/),
+					random: (random == "(EXPECTED RANDOM)"),
 					skip: (extra == " (SKIP)"),
 					url: url,
 					images: []
 				});
 			continue;
 		}
 		match = line.match(/^  IMAGE[^:]*: (.*)$/);
 		if (match) {
--- a/layout/tools/reftest/reftest-to-html.pl
+++ b/layout/tools/reftest/reftest-to-html.pl
@@ -93,25 +93,25 @@ while (<>) {
   next unless /^REFTEST/;
 
   chomp;
   chop if /\r$/;
 
   s/^REFTEST *//;
 
   my $randomresult = 0;
-  if (/RESULT EXPECTED TO BE RANDOM/) {
-    s/\(RESULT EXPECTED TO BE RANDOM\) //;
+  if (/EXPECTED RANDOM/) {
+    s/\(EXPECTED RANDOM\)//;
     $randomresult = 1;
   }
 
-  if (/^PASS(.*)$/) {
+  if (/^TEST-PASS \| (.*)$/) {
     my $class = $randomresult ? "PASSRANDOM" : "PASS";
     print '<tr><td class="' . $class . '">' . do_html($1) . "</td></tr>\n";
-  } elsif (/^UNEXPECTED (....): (.*)$/) {
+  } elsif (/^TEST-UNEXPECTED-(....) \| (.*)$/) {
     if ($randomresult) {
       die "Error on line $l: UNEXPECTED with test marked random?!";
     }
     my $class = ($1 eq "PASS") ? "WEIRDPASS" : "FAIL";
     print '<tr><td class="' . $class . '">' . do_html($2) . "</td></tr>\n";
 
     # UNEXPECTED results can be followed by one or two images
     $testline = &readcleanline;
@@ -128,17 +128,17 @@ while (<>) {
         print '<a href="' . $1 . '"><img class="testref" src="' . $1 . '"></a>';
       }
 
     } else {
       die "Error on line $l";
     }
 
     print "</td></tr>\n";
-  } elsif (/^KNOWN FAIL(.*$)/) {
+  } elsif (/^TEST-KNOWN-FAIL \| (.*$)/) {
     my $class = $randomresult ? "XFAILRANDOM" : "XFAIL";
     print '<tr><td class="' . $class . '">' . do_html($1) . "</td></tr>\n";
   } else {
     print STDERR "Unknown Line: " . $_ . "\n";
     print "<tr><td><pre>" . $_ . "</pre></td></tr>\n";
   }
 }
 
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -105,17 +105,17 @@ function OnRefTestLoad()
         ReadTopManifest(window.arguments[0]);
         if (gServer) {
             gServer.registerContentType("sjs", "sjs");
             gServer.start(HTTP_SERVER_PORT);
         }
         StartCurrentTest();
     } catch (ex) {
         //gBrowser.loadURI('data:text/plain,' + ex);
-        dump("REFTEST EXCEPTION: " + ex + "\n");
+        dump("REFTEST TEST-FAIL | | EXCEPTION: " + ex + "\n");
         DoneTests();
     }
 }
 
 function OnRefTestUnload()
 {
     gBrowser.removeEventListener("load", OnDocumentLoad, true);
 }
@@ -276,17 +276,17 @@ function ServeFiles(manifestURL, directo
 
     return files.map(FileToURI);
 }
 
 function StartCurrentTest()
 {
     // make sure we don't run tests that are expected to kill the browser
     while (gURLs.length > 0 && gURLs[0].expected == EXPECTED_DEATH) {
-        dump("REFTEST KNOWN FAIL (SKIP): " + gURLs[0].url1.spec + "\n");
+        dump("REFTEST TEST-KNOWN-FAIL | " + gURLs[0].url1.spec + " | (SKIP)\n");
         gURLs.shift();
     }
 
     if (gURLs.length == 0)
         DoneTests();
     else
         StartCurrentURI(1);
 }
@@ -391,17 +391,17 @@ function DocumentLoaded()
     if (currentTestRunTime > gSlowestTestTime) {
         gSlowestTestTime = currentTestRunTime;
         gSlowestTestURL  = gURLs[0]["url" + gState].spec;
     }
 
     clearTimeout(gFailureTimeout);
 
     if (gURLs[0].expected == EXPECTED_LOAD) {
-        dump("REFTEST PASS (LOAD ONLY): " + gURLs[0].prettyPath + "\n");
+        dump("REFTEST TEST-PASS | " + gURLs[0].prettyPath + "| (LOAD ONLY)\n");
         gURLs.shift();
         StartCurrentTest();
         return;
     }
 
     var canvas;
 
     if (gState == 1)
@@ -444,29 +444,29 @@ function DocumentLoaded()
             }
 
             // whether the comparison result matches what is in the manifest
             var test_passed = (equal == gURLs[0].equal);
             // what is expected on this platform (PASS, FAIL, or RANDOM)
             var expected = gURLs[0].expected;
             
             var outputs = {};
-            const randomMsg = " (RESULT EXPECTED TO BE RANDOM)";
-            outputs[EXPECTED_PASS] = {true: "PASS",
-                                      false: "UNEXPECTED FAIL"};
-            outputs[EXPECTED_FAIL] = {true: "UNEXPECTED PASS",
-                                      false: "KNOWN FAIL"};
-            outputs[EXPECTED_RANDOM] = {true: "PASS" + randomMsg,
-                                        false: "KNOWN FAIL" + randomMsg};
+            const randomMsg = "(EXPECTED RANDOM)";
+            outputs[EXPECTED_PASS] = {true: "TEST-PASS",
+                                      false: "TEST-UNEXPECTED-FAIL"};
+            outputs[EXPECTED_FAIL] = {true: "TEST-UNEXPECTED-PASS",
+                                      false: "TEST-KNOWN-FAIL"};
+            outputs[EXPECTED_RANDOM] = {true: "TEST-PASS" + randomMsg,
+                                        false: "TEST-KNOWN-FAIL" + randomMsg};
             
-            var result = "REFTEST " + outputs[expected][test_passed] + ": ";
+            var result = "REFTEST " + outputs[expected][test_passed] + " | ";
+            result += gURLs[0].prettyPath + " | "; // the URL being tested
             if (!gURLs[0].equal) {
                 result += "(!=) ";
             }
-            result += gURLs[0].prettyPath; // the URL being tested
             dump(result + "\n");
             if (!test_passed && expected == EXPECTED_PASS ||
                 test_passed && expected == EXPECTED_FAIL) {
                 if (!equal) {
                     dump("REFTEST   IMAGE 1 (TEST): " + gCanvas1.toDataURL() + "\n");
                     dump("REFTEST   IMAGE 2 (REFERENCE): " + gCanvas2.toDataURL() + "\n");
                     dump("REFTEST number of differing pixels: " + differences + "\n");
                 } else {
@@ -479,13 +479,13 @@ function DocumentLoaded()
             break;
         default:
             throw "Unexpected state.";
     }
 }
 
 function LoadFailed()
 {
-    dump("REFTEST UNEXPECTED FAIL (LOADING): " +
-         gURLs[0]["url" + gState].spec + "\n");
+    dump("REFTEST TEST-UNEXPECTED-FAIL | " +
+         gURLs[0]["url" + gState].spec + "| Failed to load\n");
     gURLs.shift();
     StartCurrentTest();
 }