Bug 629595. (Av3b) test_webgl_conformance_test_suite.html: Remove dump()s, Add ok()s and todo()s, Do a few related rewrites, Fix totalFailed calculation, Remove (obsolete) workaround for Windows 2000. r=bjacob.
authorSerge Gautherie <sgautherie.bz@free.fr>
Wed, 29 Feb 2012 10:41:42 +0100
changeset 87912 cb01e23f83cffb4ef934c3aa1ab7c17757928826
parent 87911 30b4f99a137c72345bb0401fa1970598687d80c7
child 87984 3812d0ce274e8ad01f579efcd2dffd4d83be0295
child 89275 3fe06bb3b9f246f05f16d0749a5076af79105f97
push id22163
push usersgautherie.bz@free.fr
push dateWed, 29 Feb 2012 09:46:13 +0000
treeherdermozilla-central@cb01e23f83cf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbjacob
bugs629595
milestone13.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 629595. (Av3b) test_webgl_conformance_test_suite.html: Remove dump()s, Add ok()s and todo()s, Do a few related rewrites, Fix totalFailed calculation, Remove (obsolete) workaround for Windows 2000. r=bjacob.
content/canvas/test/webgl/test_webgl_conformance_test_suite.html
--- a/content/canvas/test/webgl/test_webgl_conformance_test_suite.html
+++ b/content/canvas/test/webgl/test_webgl_conformance_test_suite.html
@@ -59,41 +59,33 @@ function start() {
   var kIsWindowsVistaOrHigher = false;
   if (kIsWindows) {
     // code borrowed from browser/modules/test/browser_taskbar_preview.js
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
     var version = Components.classes["@mozilla.org/system-info;1"]
                             .getService(Components.interfaces.nsIPropertyBag2)
                             .getProperty("version");
     kIsWindowsVistaOrHigher = (parseFloat(version) >= 6.0);
-    // Workaround for Windows 2000 (driver?) which may crash itself.
-    if (parseFloat(version) <= 5.0) {
-      todo(false, "Test disabled on Windows 2000 and older. (To prevent possible system crash.)");
-      SimpleTest.finish();
-      return;
-    }
   }
 
   // we currently disable this test on version of Mac OSX older than 10.6,
   // due to various weird failures, including one making getRenderbufferParameter tests
   // on DEPTH_STENCIL fail
   var kDarwinVersion = 0;
   if (kIsMac) {
     // code borrowed from browser/modules/test/browser_taskbar_preview.js
-    var is106orHigher = false;
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
     kDarwinVersion = parseFloat(Components.classes["@mozilla.org/system-info;1"]
                                           .getService(Components.interfaces.nsIPropertyBag2)
                                           .getProperty("version"));
-    // the next line is correct: Mac OS 10.6 corresponds to Darwin version 10 !
-    // Mac OS 10.5 would be Darwin version 9. the |version| string we've got here
+    // the next line is correct: Mac OSX 10.6 corresponds to Darwin version 10 !
+    // Mac OSX 10.5 would be Darwin version 9. the |version| string we've got here
     // is the Darwin version.
-    is106orHigher = (kDarwinVersion >= 10.0);
-    if (!is106orHigher) {
-      dump("WebGL mochitest disabled on Mac OSX versions older than 10.6\n");
+    if (kDarwinVersion < 10.0) {
+      todo(false, "Test disabled on Mac OSX versions older than 10.6.");
       SimpleTest.finish();
       return;
     }
   }
 
   function getEnv(env) {
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
     var envsvc = Components.classes["@mozilla.org/process/environment;1"].getService(Components.interfaces.nsIEnvironment);
@@ -125,45 +117,72 @@ function start() {
     var node = reporter.localDoc.createTextNode('');
     li.appendChild(ul);
     div.appendChild(node);
     this.totalsElem = node;
     this.resultElem = ul;
     this.elem = li;
   };
 
-  Page.prototype.isExpectedToFullyPass = function() {
-    return testsExpectedToFail.indexOf(this.url) == -1 && testsToIgnore.indexOf(this.url) == -1;
+  /**
+   * Indicates whether this test page results are not to be ignored.
+   */
+  Page.prototype.shouldBeAccountedFor = function() {
+    return testsToIgnore.indexOf(this.url) == -1;
   }
 
-  Page.prototype.errormsg = function(msg) {
-    return msg + ' (URL: ' + this.url + ')';
+  /**
+   * Indicates whether all this test page results are expected not to fail,
+   * if not ignored.
+   */
+  Page.prototype.isExpectedToFullyPass = function() {
+    return this.shouldBeAccountedFor() &&
+           testsExpectedToFail.indexOf(this.url) == -1;
   }
 
+  /**
+   * Returns log message with added test page url.
+   */
+  Page.prototype.logMsg = function(msg) {
+    return '[' + this.url + '] ' + msg;
+  }
+
+  /**
+   * Reports an individual test result of test page.
+   */
   Page.prototype.addResult = function(msg, success) {
     ++this.totalTests;
     if (success === undefined) {
       ++this.totalTimeouts;
       var result = "timeout";
       var css = "timeout";
       // only few timeouts are actually caught here --- most are caught in finishPage().
       if (this.isExpectedToFullyPass()) {
-        ok(false, this.errormsg('Test timed out, "' + msg + '"'));
+        ok(false, this.logMsg('Test timed out'), msg);
+      } else {
+        todo(false, this.logMsg('Test timed out'), msg);
       }
     } else if (success) {
       ++this.totalSuccessful;
       var result = "success";
       var css = "success";
-      // don't report success.
+      if (this.shouldBeAccountedFor()) {
+        ok(true, this.logMsg('Test passed'), msg);
+      } else {
+        todo(false, this.logMsg('Test passed, but is ignored'), msg);
+      }
+      // Don't report individual success to UI, to keep it light.
       return;
     } else {
       var result = "failed";
       var css = "fail";
       if (this.isExpectedToFullyPass()) {
-        ok(false, this.errormsg('Test failed, "' + msg + '"'));
+        ok(false, this.logMsg('Test failed'), msg);
+      } else {
+        todo(false, this.logMsg('Test failed'), msg);
       }
     }
 
     var node = this.reporter.localDoc.createTextNode(result + ': ' + msg);
     var li = this.reporter.localDoc.createElement('li');
     li.appendChild(node);
     li.setAttribute('class', css);
     this.resultElem.appendChild(li);
@@ -176,41 +195,52 @@ function start() {
     // remove previous results.
     while (this.resultElem.hasChildNodes()) {
       this.resultElem.removeChild(this.resultElem.childNodes[0]);
     }
     this.totalsElem.textContent = '';
     return true;
   };
 
+  /**
+   * Reports test page result summary.
+   */
   Page.prototype.finishPage = function(success) {
     var msg = ' (' + this.totalSuccessful + ' of ' +
               this.totalTests + ' passed)';
     if (success === undefined) {
       var css = 'testpagetimeout';
       msg = '(*timeout*)';
       ++this.totalTests;
       ++this.totalTimeouts;
+      // Most timeouts are only caught here --- though a few are (already) caught in addResult().
       if (this.isExpectedToFullyPass()) {
-        ok(false, this.errormsg('Unexpected timeout in this test page'));
-        window.dump('WebGL test error: test page timeout: ' + this.url + '\n');
+        ok(false, this.logMsg('Timeout in this test page'));
+      } else {
+        todo(false, this.logMsg('Timeout in this test page'));
       }
     } else if (this.totalSuccessful != this.totalTests) {
       var css = 'testpagefail';
+      var totalFailed = this.totalTests - this.totalTimeouts - this.totalSuccessful;
       if (this.isExpectedToFullyPass()) {
-        window.dump('WebGL test error: test page failure: ' + this.url + '\n');
+        ok(false, this.logMsg("(WebGL test error) " + totalFailed + ' failure(s) and ' + this.totalTimeouts + ' timeout(s)'));
+      } else {
+        todo(false, this.logMsg("(WebGL test error) " + totalFailed + ' failure(s) and ' + this.totalTimeouts + ' timeout(s)'));
       }
-      // failures have already been reported for the sub-tests
     } else {
       var css = 'testpagesuccess';
       if (this.isExpectedToFullyPass()) {
-        ok(true, this.errormsg('Successful test page'));
+        ok(true, this.logMsg('All ' + this.totalSuccessful + ' test(s) passed'));
+      } else {
+        if (this.shouldBeAccountedFor()) {
+          todo(true, this.logMsg('Test page expected to fail, but all ' + this.totalSuccessful + ' tests passed'));
+        } else {
+          todo(false, this.logMsg('All ' + this.totalSuccessful + ' test(s) passed, but test page is ignored'));
+        }
       }
-      window.dump('WebGL test page successful: ' + this.url + '\n');
-      testsSuccessful.push(this.url);
     }
     this.elem.setAttribute('class', css);
     this.totalsElem.textContent = msg;
   };
 
   var Reporter = function() {
     this.localDoc = document;
 
@@ -241,17 +271,17 @@ function start() {
   Reporter.prototype.addPage = function(url) {
     this.currentPage = new Page(this, url, this.resultElem);
     this.resultElem.appendChild(this.currentPage.elem);
     ++this.totalPages;
     this.pagesByURL[url] = this.currentPage;
   };
 
   Reporter.prototype.startPage = function(url) {
-    dump('WebGL mochitest: starting page ' + url + '\n');
+    info("[" + url + "] (WebGL mochitest) Starting test page");
 
     // Calling garbageCollect before each test page fixes intermittent failures with
     // out-of-memory errors, often failing to create a WebGL context.
     // The explanation is that the JS engine keeps unreferenced WebGL contexts around
     // for too long before GCing (bug 617453), so that during this mochitest dozens of unreferenced
     // WebGL contexts can accumulate at a given time.
     SpecialPowers.DOMWindowUtils.garbageCollect();
 
@@ -261,24 +291,21 @@ function start() {
     expectedtofailTextNode.textContent = testsExpectedToFail.length +
                                          ' test pages are expected to fail out of ' +
                                          this.totalPages;
     ignoredtestsTextNode.textContent = testsToIgnore.length +
                                          ' test pages have their results ignored';
     return page.startPage();
   };
 
-  Reporter.prototype.totalFailed = function() {
-    return this.totalTests - this.totalSuccessful;
-  };
-
   Reporter.prototype.displayStats = function() {
+    var totalFailed = this.totalTests - this.totalTimeouts - this.totalSuccessful;
     this.fullResultsNode.textContent =
       this.totalSuccessful + ' passed, ' +
-      this.totalFailed() + ' failed, ' +
+      totalFailed + ' failed, ' +
       this.totalTimeouts + ' timed out';
   };
 
   Reporter.prototype.addResult = function(msg, success) {
     if (this.currentPage != null) {
       this.currentPage.addResult(msg, success);
     }
   };
@@ -290,19 +317,16 @@ function start() {
       this.totalSuccessful += this.currentPage.totalSuccessful;
       this.totalTimeouts += this.currentPage.totalTimeouts;
       this.currentPage = null;
       this.displayStats();
     }
   };
 
   Reporter.prototype.finishedTestSuite = function() {
-      for (var i = 0; i < testsExpectedToFail.length; ++i)
-        if (testsSuccessful.indexOf(testsExpectedToFail[i]) != -1)
-          todo(true, 'Test expected to fail, but passed: ' + testsExpectedToFail[i]);
       statusTextNode.textContent = 'Finished';
       SimpleTest.finish();
   }
 
   Reporter.prototype.ready = function() {
     statusTextNode.textContent = 'Loaded test lists. Starting tests...';
     window.webglTestHarness.runTests();
   }
@@ -348,20 +372,22 @@ function start() {
   getURLOptions(OPTIONS);
 
   function runTestSuite() {
     var reporter = new Reporter();
 
     // try to create a dummy WebGL context, just to catch context creation failures once here,
     // rather than having them result in 100's of failures (one in each test page)
     var canvas = document.getElementById("webglcheck-default");
-    var ctx = null;
+    var ctx;
     try {
         ctx = canvas.getContext("experimental-webgl");
-    } catch(e) {}
+    } catch(e) {
+        ok(false, "canvas.getContext() failed", e);
+    }
 
     if (ctx) {
         statusTextNode.textContent = 'Loading test lists...';
         var iframe = document.getElementById("testframe");
         var testHarness = new WebGLTestHarnessModule.TestHarness(
             iframe,
             '00_test_list.txt',
             function(type, msg, success) {
@@ -373,17 +399,16 @@ function start() {
     } else {
         var errmsg = "Can't create a WebGL context";
         reporter.fullResultsNode.textContent = errmsg;
         // Workaround for SeaMonkey tinderboxes which don't support WebGL.
         if (navigator.userAgent.match(/ SeaMonkey\//))
           todo(false, errmsg + " (This is expected on SeaMonkey (tinderboxes).)");
         else
           ok(false, errmsg);
-        dump("WebGL mochitest failed: " + errmsg + "\n");
         reporter.finishedTestSuite();
     }
   };
 
   SimpleTest.requestLongerTimeout(3);
 
   var statusElem = document.getElementById("status");
   var statusTextNode = document.createTextNode('');
@@ -430,18 +455,16 @@ function start() {
 
   if (kIsMac && kDarwinVersion >= 11.0) {
     testsExpectedToFail.push('conformance/textures/texture-mips.html');
     testsExpectedToFail.push('conformance/textures/texture-npot.html');
   }
 
   var testsToIgnore = [];
 
-  var testsSuccessful = [];
-
   runTestSuite();
 }
 
 </script>
 </head>
 <body onload="start();">
 <p id="display"></p>
 <div id="content" style="display: none">