Bug 466372 - Add a global result summary to RefTest output; (Av2) Add dynamic summary to reftest.js; r=dbaron
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -53,16 +53,30 @@ const NS_REFTESTHELPER_CONTRACTID =
const NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX =
"@mozilla.org/network/protocol;1?name=";
const LOAD_FAILURE_TIMEOUT = 10000; // ms
var gBrowser;
var gCanvas1, gCanvas2;
var gURLs;
+var gTestResults = {
+ // The errors...
+ Exception: 0,
+ FailedLoad: 0,
+ UnexpectedFail: 0,
+ UnexpectedPass: 0,
+ // The successes...
+ Pass: 0,
+ KnownFail : 0,
+ Random : 0,
+ LoadOnly: 0,
+ // The unknowns.
+ Skip: 0,
+};
var gTotalTests = 0;
var gState;
var gCurrentURL;
var gFailureTimeout;
var gFailureReason;
var gServer;
var gCount = 0;
@@ -111,16 +125,17 @@ function OnRefTestLoad()
if (gServer) {
gServer.registerContentType("sjs", "sjs");
gServer.start(HTTP_SERVER_PORT);
}
gTotalTests = gURLs.length;
StartCurrentTest();
} catch (ex) {
//gBrowser.loadURI('data:text/plain,' + ex);
+ ++gTestResults.Exception;
dump("REFTEST TEST-FAIL | | EXCEPTION: " + ex + "\n");
DoneTests();
}
}
function OnRefTestUnload()
{
/* Clear the sRGB forcing pref to leave the profile as we found it. */
@@ -318,16 +333,17 @@ function ServeFiles(manifestURL, depth,
return files.map(FileToURI);
}
function StartCurrentTest()
{
// make sure we don't run tests that are expected to kill the browser
while (gURLs.length > 0 && gURLs[0].expected == EXPECTED_DEATH) {
+ ++gTestResults.Skip;
dump("REFTEST TEST-KNOWN-FAIL | " + gURLs[0].url1.spec + " | (SKIP)\n");
gURLs.shift();
}
if (gURLs.length == 0) {
DoneTests();
}
else {
@@ -349,16 +365,21 @@ function StartCurrentURI(aState)
gBrowser.loadURI(gCurrentURL);
}
function DoneTests()
{
dump("REFTEST FINISHED: Slowest test took " + gSlowestTestTime +
"ms (" + gSlowestTestURL + ")\n");
+ dump("REFTEST INFO | Result summary:\n");
+ for (var result in gTestResults)
+ if (gTestResults[result] != 0)
+ dump("REFTEST INFO | " + result + ": " + gTestResults[result] + "\n");
+
if (gServer)
gServer.stop();
goQuitApplication();
}
function setupZoom(contentRootElement) {
if (!contentRootElement.hasAttribute('reftest-zoom'))
return;
@@ -458,16 +479,17 @@ function DocumentLoaded()
gSlowestTestTime = currentTestRunTime;
gSlowestTestURL = gURLs[0]["url" + gState].spec;
}
clearTimeout(gFailureTimeout);
gFailureReason = null;
if (gURLs[0].expected == EXPECTED_LOAD) {
+ ++gTestResults.LoadOnly;
dump("REFTEST TEST-PASS | " + gURLs[0].prettyPath + " | (LOAD ONLY)\n");
gURLs.shift();
StartCurrentTest();
return;
}
var canvas;
@@ -522,29 +544,38 @@ function DocumentLoaded()
// whether the comparison result matches what is in the manifest
var test_passed = (equal == gURLs[0].equal);
// what is expected on this platform (PASS, FAIL, or RANDOM)
var expected = gURLs[0].expected;
var outputs = {};
const randomMsg = "(EXPECTED RANDOM)";
- outputs[EXPECTED_PASS] = {true: "TEST-PASS",
- false: "TEST-UNEXPECTED-FAIL"};
- outputs[EXPECTED_FAIL] = {true: "TEST-UNEXPECTED-PASS",
- false: "TEST-KNOWN-FAIL"};
- outputs[EXPECTED_RANDOM] = {true: "TEST-PASS" + randomMsg,
- false: "TEST-KNOWN-FAIL" + randomMsg};
-
- var result = "REFTEST " + outputs[expected][test_passed] + " | ";
- result += gURLs[0].prettyPath + " | "; // the URL being tested
+ outputs[EXPECTED_PASS] = {
+ true: {s: "TEST-PASS" , n: "Pass"},
+ false: {s: "TEST-UNEXPECTED-FAIL" , n: "UnexpectedFail"}
+ };
+ outputs[EXPECTED_FAIL] = {
+ true: {s: "TEST-UNEXPECTED-PASS" , n: "UnexpectedPass"},
+ false: {s: "TEST-KNOWN-FAIL" , n: "KnownFail"}
+ };
+ outputs[EXPECTED_RANDOM] = {
+ true: {s: "TEST-PASS" + randomMsg , n: "Random"},
+ false: {s: "TEST-KNOWN-FAIL" + randomMsg, n: "Random"}
+ };
+
+ ++gTestResults[outputs[expected][test_passed].n];
+
+ var result = "REFTEST " + outputs[expected][test_passed].s + " | " +
+ gURLs[0].prettyPath + " | "; // the URL being tested
if (!gURLs[0].equal) {
result += "(!=) ";
}
dump(result + "\n");
+
if (!test_passed && expected == EXPECTED_PASS ||
test_passed && expected == EXPECTED_FAIL) {
if (!equal) {
dump("REFTEST IMAGE 1 (TEST): " + gCanvas1.toDataURL() + "\n");
dump("REFTEST IMAGE 2 (REFERENCE): " + gCanvas2.toDataURL() + "\n");
dump("REFTEST number of differing pixels: " + differences + "\n");
} else {
dump("REFTEST IMAGE: " + gCanvas1.toDataURL() + "\n");
@@ -556,13 +587,14 @@ function DocumentLoaded()
break;
default:
throw "Unexpected state.";
}
}
function LoadFailed()
{
+ ++gTestResults.FailedLoad;
dump("REFTEST TEST-UNEXPECTED-FAIL | " +
gURLs[0]["url" + gState].spec + " | " + gFailureReason + "\n");
gURLs.shift();
StartCurrentTest();
}