Bug 499315 - add support of "script" tests to reftests, r=dbaron.
authorBob Clary <bclary@bclary.com>
Thu, 20 Aug 2009 00:56:22 -0700
changeset 31676 c01c8bea9eadbf55f85593c6b428fd5e17c4236c
parent 31675 5730f4c1d01658bb7c1b3f1ef190532719a81e63
child 31677 8cb946fd4488dd806dd646d4265b4270bb5ed862
push id8662
push userbclary@mozilla.com
push dateThu, 20 Aug 2009 08:36:27 +0000
treeherdermozilla-central@c01c8bea9ead [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdbaron
bugs499315
milestone1.9.3a1pre
Bug 499315 - add support of "script" tests to reftests, r=dbaron.
layout/tools/reftest/README.txt
layout/tools/reftest/print-manifest-dirs.py
layout/tools/reftest/reftest.js
--- a/layout/tools/reftest/README.txt
+++ b/layout/tools/reftest/README.txt
@@ -159,24 +159,41 @@ 2. A test item
       }
 
       For more details on exactly which functions and properties are available
       on request/response in handleRequest, see the nsIHttpRe(quest|sponse)
       definitions in <netwerk/test/httpserver/nsIHttpServer.idl>.
 
    c. <type> is one of the following:
 
-      ==    The test passes if the images of the two renderings are the
-            SAME.
-      !=    The test passes if the images of the two renderings are 
-            DIFFERENT.
-      load  The test passes unconditionally if the page loads.  url_ref
-            must be omitted, and the test cannot be marked as fails or
-            random.  (Used to test for crashes, hangs, assertions, and
-            leaks.)
+      ==     The test passes if the images of the two renderings are the
+             SAME.
+      !=     The test passes if the images of the two renderings are 
+             DIFFERENT.
+      load   The test passes unconditionally if the page loads.  url_ref
+             must be omitted, and the test cannot be marked as fails or
+             random.  (Used to test for crashes, hangs, assertions, and
+             leaks.)
+      script The loaded page records the test's pass or failure status
+             in a JavaScript data structure accessible through the following
+             API.
+
+             getTestCases() returns an array of test result objects
+             representing the results of the tests performed by the page.
+
+             Each test result object has two methods:
+
+             testPassed() returns true if the test result object passed,
+             otherwise it returns false.
+
+             testDescription() returns a string describing the test
+             result.
+
+             url_ref must be omitted. The test may be marked as fails or
+             random. (Used to test the JavaScript Engine.)
 
    d. <url> is either a relative file path or an absolute URL for the
       test page
 
    e. <url_ref> is either a relative file path or an absolute URL for
       the reference page
 
    The only difference between <url> and <url_ref> is that results of
--- a/layout/tools/reftest/print-manifest-dirs.py
+++ b/layout/tools/reftest/print-manifest-dirs.py
@@ -67,17 +67,17 @@ def parseManifest(manifest, dirs):
       # need to package the dir referenced here
       d = os.path.normpath(os.path.join(manifestdir, m.group(1)))
       dirs.add(d)
       del items[0]
 
     if items[0] == "include":
       parseManifest(os.path.join(manifestdir, items[1]), dirs)
       continue
-    elif items[0] == "load":
+    elif items[0] == "load" or items[0] == "script":
       testURLs = [items[1]]
     elif items[0] == "==" or items[0] == "!=":
       testURLs = items[1:3]
     for u in testURLs:
       if u.startswith("about:") or u.startswith("data:"):
         # can't very well package about: or data: URIs
         continue
       d = os.path.dirname(os.path.normpath(os.path.join(manifestdir, u)))
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -102,22 +102,26 @@ var gIOService;
 var gDebug;
 var gWindowUtils;
 
 var gCurrentTestStartTime;
 var gSlowestTestTime = 0;
 var gSlowestTestURL;
 var gClearingForAssertionCheck = false;
 
+const TYPE_REFTEST_EQUAL = '==';
+const TYPE_REFTEST_NOTEQUAL = '!=';
+const TYPE_LOAD = 'load';     // test without a reference (just test that it does
+                              // not assert, crash, hang, or leak)
+const TYPE_SCRIPT = 'script'; // test contains individual test results
+
 const EXPECTED_PASS = 0;
 const EXPECTED_FAIL = 1;
 const EXPECTED_RANDOM = 2;
 const EXPECTED_DEATH = 3;  // test must be skipped to avoid e.g. crash/hang
-const EXPECTED_LOAD = 4; // test without a reference (just test that it does
-                         // not assert, crash, hang, or leak)
 
 var HTTP_SERVER_PORT = 4444;
 const HTTP_SERVER_PORTS_TO_TRY = 50;
 
 var gRecycledCanvases = new Array();
 
 function AllocateCanvas()
 {
@@ -141,17 +145,17 @@ function OnRefTestLoad()
 {
     gBrowser = document.getElementById("browser");
 
     /* set the gLoadTimeout */
     try {
       var prefs = Components.classes["@mozilla.org/preferences-service;1"].
                   getService(Components.interfaces.nsIPrefBranch2);
       gLoadTimeout = prefs.getIntPref("reftest.timeout");
-    }                  
+    }
     catch(e) {
       gLoadTimeout = 5 * 60 * 1000; //5 minutes as per bug 479518
     }
 
     gBrowser.addEventListener("load", OnDocumentLoad, true);
 
     try {
         gWindowUtils = window.QueryInterface(CI.nsIInterfaceRequestor).getInterface(CI.nsIDOMWindowUtils);
@@ -364,63 +368,80 @@ function ReadManifest(aURL)
 
         if (items[0] == "include") {
             if (items.length != 2 || runHttp)
                 throw "Error 2 in manifest file " + aURL.spec + " line " + lineNo;
             var incURI = gIOService.newURI(items[1], null, listURL);
             secMan.checkLoadURI(aURL, incURI,
                                 CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
             ReadManifest(incURI);
-        } else if (items[0] == "load") {
-            if (expected_status == EXPECTED_PASS)
-                expected_status = EXPECTED_LOAD;
+        } else if (items[0] == TYPE_LOAD) {
             if (items.length != 2 ||
-                (expected_status != EXPECTED_LOAD &&
+                (expected_status != EXPECTED_PASS &&
                  expected_status != EXPECTED_DEATH))
                 throw "Error 3 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI] = runHttp
                             ? ServeFiles(aURL, httpDepth,
                                          listURL.file.parent, [items[1]])
                             : [gIOService.newURI(items[1], null, listURL)];
             var prettyPath = runHttp
                            ? gIOService.newURI(items[1], null, listURL).spec
                            : testURI.spec;
             secMan.checkLoadURI(aURL, testURI,
                                 CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
-            gURLs.push( { equal: true /* meaningless */,
+            gURLs.push( { type: TYPE_LOAD,
                           expected: expected_status,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           url1: testURI,
                           url2: null } );
-        } else if (items[0] == "==" || items[0] == "!=") {
+        } else if (items[0] == TYPE_SCRIPT) {
+            if (items.length != 2)
+                throw "Error 4 in manifest file " + aURL.spec + " line " + lineNo;
+            var [testURI] = runHttp
+                            ? ServeFiles(aURL, httpDepth,
+                                         listURL.file.parent, [items[1]])
+                            : [gIOService.newURI(items[1], null, listURL)];
+            var prettyPath = runHttp
+                           ? gIOService.newURI(items[1], null, listURL).spec
+                           : testURI.spec;
+            secMan.checkLoadURI(aURL, testURI,
+                                CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
+            gURLs.push( { type: TYPE_SCRIPT,
+                          expected: expected_status,
+                          prettyPath: prettyPath,
+                          minAsserts: minAsserts,
+                          maxAsserts: maxAsserts,
+                          url1: testURI,
+                          url2: null } );
+        } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
             if (items.length != 3)
-                throw "Error 4 in manifest file " + aURL.spec + " line " + lineNo;
+                throw "Error 5 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI, refURI] = runHttp
                                   ? ServeFiles(aURL, httpDepth,
                                                listURL.file.parent, [items[1], items[2]])
                                   : [gIOService.newURI(items[1], null, listURL),
                                      gIOService.newURI(items[2], null, listURL)];
             var prettyPath = runHttp
                            ? gIOService.newURI(items[1], null, listURL).spec
                            : testURI.spec;
             secMan.checkLoadURI(aURL, testURI,
                                 CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
             secMan.checkLoadURI(aURL, refURI,
                                 CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
-            gURLs.push( { equal: (items[0] == "=="),
+            gURLs.push( { type: items[0],
                           expected: expected_status,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           url1: testURI,
                           url2: refURI } );
         } else {
-            throw "Error 5 in manifest file " + aURL.spec + " line " + lineNo;
+            throw "Error 6 in manifest file " + aURL.spec + " line " + lineNo;
         }
     } while (more);
 }
 
 function AddURIUseCount(uri)
 {
     if (uri == null)
         return;
@@ -432,18 +453,20 @@ function AddURIUseCount(uri)
         gURIUseCounts[spec] = 1;
     }
 }
 
 function BuildUseCounts()
 {
     gURIUseCounts = {};
     for (var i = 0; i < gURLs.length; ++i) {
-        var expected = gURLs[i].expected;
-        if (expected != EXPECTED_DEATH && expected != EXPECTED_LOAD) {
+        var url = gURLs[i];
+        if (url.expected != EXPECTED_DEATH &&
+            (url.type == TYPE_REFTEST_EQUAL ||
+             url.type == TYPE_REFTEST_NOTEQUAL)) {
             AddURIUseCount(gURLs[i].url1);
             AddURIUseCount(gURLs[i].url2);
         }
     }
 }
 
 function ServeFiles(manifestURL, depth, directory, files)
 {
@@ -512,17 +535,19 @@ function StartCurrentURI(aState)
         ++gTestResults.Exception;
     }
     gFailureTimeout = setTimeout(LoadFailed, gLoadTimeout);
     gFailureReason = "timed out waiting for onload to fire";
 
     gState = aState;
     gCurrentURL = gURLs[0]["url" + aState].spec;
 
-    if (gURICanvases[gCurrentURL] && gURLs[0].expected != EXPECTED_LOAD &&
+    if (gURICanvases[gCurrentURL] &&
+        (gURLs[0].type == TYPE_REFTEST_EQUAL ||
+         gURLs[0].type == TYPE_REFTEST_NOTEQUAL) &&
         gURLs[0].maxAsserts == 0) {
         // Pretend the document loaded --- DocumentLoaded will notice
         // there's already a canvas for this URL
         setTimeout(DocumentLoaded, 0);
     } else {
         gBrowser.loadURI(gCurrentURL);
     }
 }
@@ -835,22 +860,106 @@ function DocumentLoaded()
         gSlowestTestTime = currentTestRunTime;
         gSlowestTestURL  = gCurrentURL;
     }
 
     clearTimeout(gFailureTimeout);
     gFailureReason = null;
     gFailureTimeout = null;
 
-    if (gURLs[0].expected == EXPECTED_LOAD) {
+    // Not 'const ...' because of 'EXPECTED_*' value dependency.
+    var outputs = {};
+    const randomMsg = "(EXPECTED RANDOM)";
+    outputs[EXPECTED_PASS] = {
+        true:  {s: "TEST-PASS"                  , n: "Pass"},
+        false: {s: "TEST-UNEXPECTED-FAIL"       , n: "UnexpectedFail"}
+    };
+    outputs[EXPECTED_FAIL] = {
+        true:  {s: "TEST-UNEXPECTED-PASS"       , n: "UnexpectedPass"},
+        false: {s: "TEST-KNOWN-FAIL"            , n: "KnownFail"}
+    };
+    outputs[EXPECTED_RANDOM] = {
+        true:  {s: "TEST-PASS" + randomMsg      , n: "Random"},
+        false: {s: "TEST-KNOWN-FAIL" + randomMsg, n: "Random"}
+    };
+    var output;
+
+    if (gURLs[0].type == TYPE_LOAD) {
         ++gTestResults.LoadOnly;
         dump("REFTEST TEST-PASS | " + gURLs[0].prettyPath + " | (LOAD ONLY)\n");
         FinishTestItem();
         return;
     }
+    if (gURLs[0].type == TYPE_SCRIPT) {
+        var missing_msg = false;
+        var testwindow = gBrowser.contentWindow;
+        expected = gURLs[0].expected;
+
+        if (testwindow.wrappedJSObject)
+            testwindow = testwindow.wrappedJSObject;
+
+        var testcases;
+
+        if (!testwindow.getTestCases || typeof testwindow.getTestCases != "function") {
+            // Force an unexpected failure to alert the test author to fix the test.
+            expected = EXPECTED_PASS;
+            missing_msg = "test must provide a function getTestCases(). (SCRIPT)\n";
+        }
+        else if (!(testcases = testwindow.getTestCases())) {
+            // Force an unexpected failure to alert the test author to fix the test.
+            expected = EXPECTED_PASS;
+            missing_msg = "test's getTestCases() must return an Array-like Object. (SCRIPT)\n";
+        }
+        else if (testcases.length == 0) {
+            // This failure may be due to a JavaScript Engine bug causing
+            // early termination of the test.
+            missing_msg = "No test results reported. (SCRIPT)\n";
+        }
+
+        if (missing_msg) {
+            output = outputs[expected][false];
+            ++gTestResults[output.n];
+            var result = "REFTEST " + output.s + " | " +
+                gURLs[0].prettyPath + " | " + // the URL being tested
+                missing_msg;
+
+            dump(result);
+            FinishTestItem();
+            return;
+        }
+
+        var results = testcases.map(function(test) {
+                return { passed: test.testPassed(), description: test.testDescription()};
+            });
+        var anyFailed = results.some(function(result) { return !result.passed; });
+        var outputPair;
+        if (anyFailed && expected == EXPECTED_FAIL) {
+            // If we're marked as expected to fail, and some (but not all) tests
+            // passed, treat those tests as though they were marked random
+            // (since we can't tell whether they were really intended to be
+            // marked failing or not).
+            outputPair = { true: outputs[EXPECTED_RANDOM][true],
+                           false: outputs[expected][false] };
+        } else {
+            outputPair = outputs[expected];
+        }
+        var index = 0;
+        results.forEach(function(result) {
+                var output = outputPair[result.passed];
+
+                ++gTestResults[output.n];
+                result = "REFTEST " + output.s + " | " +
+                    gURLs[0].prettyPath + " | " + // the URL being tested
+                    result.description + " item " + (++index) + "\n";
+                dump(result);
+            });
+
+        FinishTestItem();
+        return;
+    }
 
     if (gURICanvases[gCurrentURL]) {
         gCurrentCanvas = gURICanvases[gCurrentURL];
     } else if (gCurrentCanvas == null) {
         InitCurrentCanvasWithSnapshot();
     }
     if (gState == 1) {
         gCanvas1 = gCurrentCanvas;
@@ -884,41 +993,26 @@ function DocumentLoaded()
             } else {
                 differences = -1;
                 var k1 = gCanvas1.toDataURL();
                 var k2 = gCanvas2.toDataURL();
                 equal = (k1 == k2);
             }
 
             // whether the comparison result matches what is in the manifest
-            var test_passed = (equal == gURLs[0].equal);
+            var test_passed = (equal == (gURLs[0].type == TYPE_REFTEST_EQUAL));
             // what is expected on this platform (PASS, FAIL, or RANDOM)
             var expected = gURLs[0].expected;
+            output = outputs[expected][test_passed];
 
-            // Not 'const ...' because of 'EXPECTED_*' value dependency.
-            var outputs = {};
-            const randomMsg = "(EXPECTED RANDOM)";
-            outputs[EXPECTED_PASS] = {
-              true:  {s: "TEST-PASS"                  , n: "Pass"},
-              false: {s: "TEST-UNEXPECTED-FAIL"       , n: "UnexpectedFail"}
-            };
-            outputs[EXPECTED_FAIL] = {
-              true:  {s: "TEST-UNEXPECTED-PASS"       , n: "UnexpectedPass"},
-              false: {s: "TEST-KNOWN-FAIL"            , n: "KnownFail"}
-            };
-            outputs[EXPECTED_RANDOM] = {
-              true:  {s: "TEST-PASS" + randomMsg      , n: "Random"},
-              false: {s: "TEST-KNOWN-FAIL" + randomMsg, n: "Random"}
-            };
+            ++gTestResults[output.n];
 
-            ++gTestResults[outputs[expected][test_passed].n];
-
-            var result = "REFTEST " + outputs[expected][test_passed].s + " | " +
+            var result = "REFTEST " + output.s + " | " +
                          gURLs[0].prettyPath + " | "; // the URL being tested
-            if (!gURLs[0].equal) {
+            if (gURLs[0].type == TYPE_REFTEST_NOTEQUAL) {
                 result += "(!=) ";
             }
             dump(result + "\n");
 
             if (!test_passed && expected == EXPECTED_PASS ||
                 test_passed && expected == EXPECTED_FAIL) {
                 if (!equal) {
                     dump("REFTEST   IMAGE 1 (TEST): " + gCanvas1.toDataURL() + "\n");