Bug 466372 - Add a global result summary to RefTest output; (Av2) Add dynamic summary to reftest.js; r=dbaron
authorSerge Gautherie <sgautherie.bz@free.fr>
Tue, 02 Dec 2008 13:35:24 +0100
changeset 22153 0f32649fe11b3c35f7cca4c673a4cd6b7f1ea227
parent 22152 182f85d0c344ddd5b1d8009458942d13e301f123
child 22154 90f89770b9b69551063c9688d98cec278a2f5da5
push id3829
push usersgautherie.bz@free.fr
push dateTue, 02 Dec 2008 12:36:07 +0000
treeherdermozilla-central@0f32649fe11b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdbaron
bugs466372
milestone1.9.2a1pre
Bug 466372 - Add a global result summary to RefTest output; (Av2) Add dynamic summary to reftest.js; r=dbaron
layout/tools/reftest/reftest.js
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -53,16 +53,30 @@ const NS_REFTESTHELPER_CONTRACTID =
 const NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX =
           "@mozilla.org/network/protocol;1?name=";
 
 const LOAD_FAILURE_TIMEOUT = 10000; // ms
 
 var gBrowser;
 var gCanvas1, gCanvas2;
 var gURLs;
+var gTestResults = {
+  // The errors...
+  Exception: 0,
+  FailedLoad: 0,
+  UnexpectedFail: 0,
+  UnexpectedPass: 0,
+  // The successes...
+  Pass: 0,
+  KnownFail : 0,
+  Random : 0,
+  LoadOnly: 0,
+  // The unknowns.
+  Skip: 0,
+};
 var gTotalTests = 0;
 var gState;
 var gCurrentURL;
 var gFailureTimeout;
 var gFailureReason;
 var gServer;
 var gCount = 0;
 
@@ -111,16 +125,17 @@ function OnRefTestLoad()
         if (gServer) {
             gServer.registerContentType("sjs", "sjs");
             gServer.start(HTTP_SERVER_PORT);
         }
         gTotalTests = gURLs.length;
         StartCurrentTest();
     } catch (ex) {
         //gBrowser.loadURI('data:text/plain,' + ex);
+        ++gTestResults.Exception;
         dump("REFTEST TEST-FAIL | | EXCEPTION: " + ex + "\n");
         DoneTests();
     }
 }
 
 function OnRefTestUnload()
 {
     /* Clear the sRGB forcing pref to leave the profile as we found it. */
@@ -318,16 +333,17 @@ function ServeFiles(manifestURL, depth, 
 
     return files.map(FileToURI);
 }
 
 function StartCurrentTest()
 {
     // make sure we don't run tests that are expected to kill the browser
     while (gURLs.length > 0 && gURLs[0].expected == EXPECTED_DEATH) {
+        ++gTestResults.Skip;
         dump("REFTEST TEST-KNOWN-FAIL | " + gURLs[0].url1.spec + " | (SKIP)\n");
         gURLs.shift();
     }
 
     if (gURLs.length == 0) {
         DoneTests();
     }
     else {
@@ -349,16 +365,21 @@ function StartCurrentURI(aState)
     gBrowser.loadURI(gCurrentURL);
 }
 
 function DoneTests()
 {
     dump("REFTEST FINISHED: Slowest test took " + gSlowestTestTime +
          "ms (" + gSlowestTestURL + ")\n");
 
+    dump("REFTEST INFO | Result summary:\n");
+    for (var result in gTestResults)
+      if (gTestResults[result] != 0)
+        dump("REFTEST INFO | " + result + ": " + gTestResults[result] + "\n");
+
     if (gServer)
         gServer.stop();
     goQuitApplication();
 }
 
 function setupZoom(contentRootElement) {
     if (!contentRootElement.hasAttribute('reftest-zoom'))
         return;
@@ -458,16 +479,17 @@ function DocumentLoaded()
         gSlowestTestTime = currentTestRunTime;
         gSlowestTestURL  = gURLs[0]["url" + gState].spec;
     }
 
     clearTimeout(gFailureTimeout);
     gFailureReason = null;
 
     if (gURLs[0].expected == EXPECTED_LOAD) {
+        ++gTestResults.LoadOnly;
         dump("REFTEST TEST-PASS | " + gURLs[0].prettyPath + " | (LOAD ONLY)\n");
         gURLs.shift();
         StartCurrentTest();
         return;
     }
 
     var canvas;
 
@@ -522,29 +544,38 @@ function DocumentLoaded()
 
             // whether the comparison result matches what is in the manifest
             var test_passed = (equal == gURLs[0].equal);
             // what is expected on this platform (PASS, FAIL, or RANDOM)
             var expected = gURLs[0].expected;
             
             var outputs = {};
             const randomMsg = "(EXPECTED RANDOM)";
-            outputs[EXPECTED_PASS] = {true: "TEST-PASS",
-                                      false: "TEST-UNEXPECTED-FAIL"};
-            outputs[EXPECTED_FAIL] = {true: "TEST-UNEXPECTED-PASS",
-                                      false: "TEST-KNOWN-FAIL"};
-            outputs[EXPECTED_RANDOM] = {true: "TEST-PASS" + randomMsg,
-                                        false: "TEST-KNOWN-FAIL" + randomMsg};
-            
-            var result = "REFTEST " + outputs[expected][test_passed] + " | ";
-            result += gURLs[0].prettyPath + " | "; // the URL being tested
+            outputs[EXPECTED_PASS] = {
+              true:  {s: "TEST-PASS"                  , n: "Pass"},
+              false: {s: "TEST-UNEXPECTED-FAIL"       , n: "UnexpectedFail"}
+            };
+            outputs[EXPECTED_FAIL] = {
+              true:  {s: "TEST-UNEXPECTED-PASS"       , n: "UnexpectedPass"},
+              false: {s: "TEST-KNOWN-FAIL"            , n: "KnownFail"}
+            };
+            outputs[EXPECTED_RANDOM] = {
+              true:  {s: "TEST-PASS" + randomMsg      , n: "Random"},
+              false: {s: "TEST-KNOWN-FAIL" + randomMsg, n: "Random"}
+            };
+
+            ++gTestResults[outputs[expected][test_passed].n];
+
+            var result = "REFTEST " + outputs[expected][test_passed].s + " | " +
+                         gURLs[0].prettyPath + " | "; // the URL being tested
             if (!gURLs[0].equal) {
                 result += "(!=) ";
             }
             dump(result + "\n");
+
             if (!test_passed && expected == EXPECTED_PASS ||
                 test_passed && expected == EXPECTED_FAIL) {
                 if (!equal) {
                     dump("REFTEST   IMAGE 1 (TEST): " + gCanvas1.toDataURL() + "\n");
                     dump("REFTEST   IMAGE 2 (REFERENCE): " + gCanvas2.toDataURL() + "\n");
                     dump("REFTEST number of differing pixels: " + differences + "\n");
                 } else {
                     dump("REFTEST   IMAGE: " + gCanvas1.toDataURL() + "\n");
@@ -556,13 +587,14 @@ function DocumentLoaded()
             break;
         default:
             throw "Unexpected state.";
     }
 }
 
 function LoadFailed()
 {
+    ++gTestResults.FailedLoad;
     dump("REFTEST TEST-UNEXPECTED-FAIL | " +
          gURLs[0]["url" + gState].spec + " | " + gFailureReason + "\n");
     gURLs.shift();
     StartCurrentTest();
 }