Bug 582053 - Integrate WebGL conformance suite as a mochitest - r=vlad, a=blocking2.0
authorBenoit Jacob <bjacob@mozilla.com>
Fri, 14 Jan 2011 13:40:05 -0500
changeset 60605 2793d141b019024fd4c9e5f7273f0727c5c1f585
parent 60604 4e0501a0c5e5eae34fd3e8f754cff5e4c66b5260
child 60606 9caa011615ff3def0e33205f3aacb990fb301445
push idunknown
push userunknown
push dateunknown
reviewersvlad, blocking2
bugs582053
milestone2.0b10pre
Bug 582053 - Integrate WebGL conformance suite as a mochitest - r=vlad, a=blocking2.0
content/canvas/test/webgl/Makefile.in
content/canvas/test/webgl/failing_tests.txt
content/canvas/test/webgl/failing_tests_linux.txt
content/canvas/test/webgl/failing_tests_mac.txt
content/canvas/test/webgl/failing_tests_windows.txt
content/canvas/test/webgl/test_webgl_conformance_test_suite.html
--- a/content/canvas/test/webgl/Makefile.in
+++ b/content/canvas/test/webgl/Makefile.in
@@ -39,20 +39,21 @@ DEPTH		= ../../../..
 topsrcdir	= @top_srcdir@
 srcdir		= @srcdir@
 VPATH		= @srcdir@
 relativesrcdir  = content/canvas/test/webgl
 
 include $(DEPTH)/config/autoconf.mk
 include $(topsrcdir)/config/rules.mk
 _TEST_FILES = \
-	test_webgl_conformance_test_suite.html \
-	00_testFIXME_list.txt \
-	failing_tests.txt \
-	$(NULL)
+  test_webgl_conformance_test_suite.html \
+  00_testFIXME_list.txt \
+  failing_tests_linux.txt \
+  failing_tests_windows.txt \
+  failing_tests_mac.txt \
+  $(NULL)
 
 libs:: $(_TEST_FILES)
 	$(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)
 	$(TAR) cC $(srcdir) \
-		resources \
-		conformance \
-		more \
-		| $(TAR) xC $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)
+	  resources \
+	  conformance \
+	  | $(TAR) xC $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)
rename from content/canvas/test/webgl/failing_tests.txt
rename to content/canvas/test/webgl/failing_tests_linux.txt
--- a/content/canvas/test/webgl/failing_tests.txt
+++ b/content/canvas/test/webgl/failing_tests_linux.txt
@@ -1,33 +1,10 @@
-conformance/canvas-test.html
-conformance/constants.html
-conformance/context-attributes-alpha-depth-stencil-antialias.html
 conformance/context-attributes.html
-conformance/context-type-test.html
+conformance/gl-enum-tests.html
+conformance/gl-get-active-attribute.html
 conformance/gl-get-calls.html
 conformance/gl-teximage.html
-conformance/glsl-conformance.html
-conformance/methods.html
-conformance/program-test.html
-conformance/read-pixels-pack-alignment.html
-conformance/tex-image-and-sub-image-2d-with-video.html
-conformance/tex-image-with-invalid-data.html
-conformance/tex-input-validation.html
-more/conformance/constants.html
-more/conformance/getContext.html
-more/conformance/quickCheckAPI.html
-more/conformance/quickCheckAPIBadArgs.html
-more/functions/bufferDataBadArgs.html
-more/functions/copyTexImage2D.html
-more/functions/copyTexImage2DBadArgs.html
-more/functions/copyTexSubImage2D.html
-more/functions/copyTexSubImage2DBadArgs.html
-more/functions/deleteBufferBadArgs.html
-more/functions/drawElementsBadArgs.html
-more/functions/getImageData.html
-more/functions/readPixelsBadArgs.html
-more/functions/texImage2DBadArgs.html
-more/functions/texSubImage2DBadArgs.html
-more/functions/texSubImage2DHTMLBadArgs.html
-more/functions/uniformiBadArgs.html
-more/functions/uniformMatrixBadArgs.html
-more/glsl/arrayOutOfBounds.html
+conformance/gl-uniform-bool.html
+conformance/gl-vertexattribpointer.html
+conformance/glsl-2types-of-textures-on-same-unit.html
+conformance/is-object.html
+conformance/uninitialized-test.html
new file mode 100644
--- /dev/null
+++ b/content/canvas/test/webgl/failing_tests_mac.txt
@@ -0,0 +1,9 @@
+conformance/canvas-test.html
+conformance/canvas-test.html
+conformance/context-attributes.html
+conformance/gl-enum-tests.html
+conformance/gl-get-calls.html
+conformance/gl-object-get-calls.html
+conformance/gl-teximage.html
+conformance/gl-vertexattribpointer.html
+conformance/texture-npot.html
new file mode 100644
--- /dev/null
+++ b/content/canvas/test/webgl/failing_tests_windows.txt
@@ -0,0 +1,10 @@
+conformance/context-attributes.html
+conformance/gl-enum-tests.html
+conformance/gl-get-active-attribute.html
+conformance/gl-get-calls.html
+conformance/gl-teximage.html
+conformance/gl-uniform-bool.html
+conformance/gl-vertexattribpointer.html
+conformance/glsl-2types-of-textures-on-same-unit.html
+conformance/is-object.html
+conformance/uninitialized-test.html
--- a/content/canvas/test/webgl/test_webgl_conformance_test_suite.html
+++ b/content/canvas/test/webgl/test_webgl_conformance_test_suite.html
@@ -13,34 +13,34 @@ Mochitest version of the WebGL Conforman
 
 /**
  * This is copied from webgl-test-harness.js where it is defined as a private function, not accessible to us (argh!)
  *
  * Loads text from an external file. This function is synchronous.
  * @param {string} url The url of the external file.
  * @return {string} the loaded text if the request is synchronous.
  */
-var loadTextFileSynchronous = function(url) {
-  var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
-  var request;
-  if (window.XMLHttpRequest) {
-    request = new XMLHttpRequest();
-    if (request.overrideMimeType) {
-      request.overrideMimeType('text/plain');
+  var loadTextFileSynchronous = function (url) {
+    var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
+    var request;
+    if (window.XMLHttpRequest) {
+      request = new XMLHttpRequest();
+      if (request.overrideMimeType) {
+        request.overrideMimeType('text/plain');
+      }
+    } else {
+      throw 'XMLHttpRequest is disabled';
     }
-  } else {
-    throw 'XMLHttpRequest is disabled';
-  }
-  request.open('GET', url, false);
-  request.send(null);
-  if (request.readyState != 4) {
-    throw error;
-  }
-  return request.responseText;
-};
+    request.open('GET', url, false);
+    request.send(null);
+    if (request.readyState != 4) {
+      throw error;
+    }
+    return request.responseText;
+  };
 
 function start() {
 
   function prefSvc() {
     var svc = Components.classes["@mozilla.org/preferences-service;1"]
                         .getService(Components.interfaces.nsIPrefService).getBranch("");
     return svc;
   }
@@ -50,62 +50,33 @@ function start() {
     prefSvc().setBoolPref("webgl.enabled_for_all_sites", value);
   }
 
   function getEnabledForAllSites() {
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
     return prefSvc().getBoolPref("webgl.enabled_for_all_sites");
   }
 
-  function setAnglePath(path) {
+  function setPreferGL(value) {
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
-    if (path == null) {
-      try {
-        prefSvc().clearUserPref("gfx.angle.egl.path");
-      } catch (e) {
-        // ignore
-      }
-      return;
-    }
-    try {
-      var f = Components.classes["@mozilla.org/file/local;1"].createInstance(Components.interfaces.nsILocalFile);
-      f.initWithPath(path);
-      prefSvc().setComplexValue("gfx.angle.egl.path", Components.interfaces.nsILocalFile, f);
-    } catch (e) {
-      // ignore -- the path is likely not valid, e.g. trying to use a path of "C:\libs" on non-windows boxes.
-    }
+    prefSvc().setBoolPref("webgl.prefer_gl", value);
   }
 
-  function getAnglePath() {
-    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
-    if (prefSvc().getPrefType("gfx.angle.egl.path") == Components.interfaces.nsIPrefBranch.PREF_INVALID)
-      return null;
-    return prefSvc().getComplexValue("gfx.angle.egl.path", Components.interfaces.nsILocalFile);
-  }
-
-  function getPreferEgl() {
+  function getPreferGL() {
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
-    if (prefSvc().getPrefType("webgl.prefer_egl") == Components.interfaces.nsIPrefBranch.PREF_INVALID)
-      return null;
-    return prefSvc().getBoolPref("webgl.prefer_egl");
-  }
-
-  function setPreferEgl(val) {
-    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
-    if (val == null) {
-      prefSvc().clearUserPref("webgl.prefer_egl");
-      return;
-    }
-    prefSvc().setBoolPref("webgl.prefer_egl", val);
+    var retval = false;
+    try {
+      retval = prefSvc().getBoolPref("webgl.prefer_gl");
+    } catch (ex) { }
+    return retval;
   }
 
   function restoreOldPrefs() {
     setEnabledForAllSites(saved_enabled_for_all_sites);
-    setAnglePath(saved_angle_path);
-    setPreferEgl(saved_prefer_egl);
+    setPreferGL(saved_prefer_gl);
   }
 
   function getEnv(env) {
     netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
     var envsvc = Components.classes["@mozilla.org/process/environment;1"].getService(Components.interfaces.nsIEnvironment);
     var val = envsvc.get(env);
     if (val == "")
       return null;
@@ -250,16 +221,17 @@ function start() {
   Reporter.prototype.addPage = function(url) {
     this.currentPage = new Page(this, url, this.resultElem);
     this.resultElem.appendChild(this.currentPage.elem);
     ++this.totalPages;
     this.pagesByURL[url] = this.currentPage;
   };
 
   Reporter.prototype.startPage = function(url) {
+    dump('WebGL mochitest: starting page ' + url + '\n');
     var page = this.pagesByURL[url];
     this.currentPage = page;
     statusTextNode.textContent = 'Running URL: ' + url;
     expectedtofailTextNode.textContent = testsExpectedToFail.length +
                                          ' test pages are expected to fail out of ' +
                                          this.totalPages;
     return page.startPage();
   };
@@ -287,54 +259,54 @@ function start() {
       this.totalTests += this.currentPage.totalTests;
       this.totalSuccessful += this.currentPage.totalSuccessful;
       this.totalTimeouts += this.currentPage.totalTimeouts;
       this.currentPage = null;
       this.displayStats();
     }
   };
 
-  Reporter.prototype.completedCurrentTestSuiteRun = function() {
-        timesTestSuiteAlreadyRun++;
-        runTestSuite();
-  };
+  Reporter.prototype.finishedTestSuite = function() {
+      for (var i = 0; i < testsExpectedToFail.length; ++i)
+        if (testsSuccessful.indexOf(testsExpectedToFail[i]) != -1)
+          ok(false, 'Test expected to fail, but passed: ' + testsExpectedToFail[i]);
+      restoreOldPrefs();
+      statusTextNode.textContent = 'Finished';
+      SimpleTest.finish();
+  }
 
   Reporter.prototype.reportFunc = function(type, msg, success) {
     switch (type) {
       case reportType.ADD_PAGE:
         return this.addPage(msg);
       case reportType.START_PAGE:
         return this.startPage(msg);
       case reportType.TEST_RESULT:
         return this.addResult(msg, success);
       case reportType.FINISH_PAGE:
         return this.finishPage(success);
       case reportType.FINISHED_ALL_TESTS:
-        this.completedCurrentTestSuiteRun();
+        this.finishedTestSuite();
         return true;
       default:
         throw 'unhandled';
         break;
     };
   };
 
-  function doRunTests() {
+  function runTestSuite() {
     var reporter = new Reporter();
 
     // set WebGL-related preferences needed to run WebGL tests
     setEnabledForAllSites(true);
 
-    // if this wasn't specified, then set it to our expected dir for the build
-    // machines
-    if (!saved_angle_path) {
-      // For windows test runners.  For everyone else, it doesn't matter,
-      // we'll use ANGLE if it's installed according to the prefs.
-      setAnglePath("C:\\libs");
-      setPreferEgl(true);
-    }
+    // XXX temporary hack. Until the DirectX runtime is deployed on test slaves, we just use OpenGL,
+    // so that rel-eng can safely deploy DirectX runtime without getting sudden mochitest failures
+    // (e.g. if there's a ANGLE bug).
+    setPreferGL(true);
 
     // try to create a dummy WebGL context, just to catch context creation failures once here,
     // rather than having them result in 100's of failures (one in each test page)
     var canvas = document.getElementById("webglcheck-default");
     var ctx = null;
     try {
         ctx = canvas.getContext("experimental-webgl");
     } catch(e) {}
@@ -346,70 +318,103 @@ function start() {
             '00_testFIXME_list.txt',
             function(type, msg, success) {
                 return reporter.reportFunc(type, msg, success);
             });
         testHarness.setTimeoutDelay(10000); // and make it much higher when running under valgrind.
         window.webglTestHarness = testHarness;
         testHarness.runTests();
     } else {
-        // if the webgl context is null, we agreed that we wouldn't fail the test, we only add a TODO mentioning
-        // that the GL setup needs to be fixed in order for these tests to be actually run. The reason why we don't
-        // currently want to fail the test is that we want to be able to land this mochitest as soon as possible and
-        // just take care of the GL setups on the test machines afterwards.
-        var errmsg = "Disabled, because we were unable to create a WebGL context anyhow.";
+        var errmsg = "Can't create a WebGL context";
         reporter.fullResultsNode.textContent = errmsg;
-        todo(false, errmsg);
-        reporter.completedCurrentTestSuiteRun();
+        ok(false, errmsg);
+        dump("WebGL mochitest failed: " + errmsg + "\n");
+        reporter.finishedTestSuite();
     }
   };
 
-  function runTestSuite() {
-    if (timesTestSuiteAlreadyRun == 1) {
-      for (var i = 0; i < testsExpectedToFail.length; ++i) {
-        if (testsSuccessful.indexOf(testsExpectedToFail[i]) != -1) {
-          todo(false, 'Test expected to fail, but passed: ' + testsExpectedToFail[i]);
-          dump('Test expected to fail, but passed: ' + testsExpectedToFail[i] + '\n');
-        }
-      }
-
-      restoreOldPrefs();
-      statusTextNode.textContent = 'Finished';
-      SimpleTest.finish();
-      return;
-    }
-
-    doRunTests();
-  };
-
   SimpleTest.waitForExplicitFinish();
   SimpleTest.requestLongerTimeout(2);
 
-  if (getEnv("MOZ_WEBGL_TEST_SUITE") == null) {
-    // webgl test suite is disabled unless the env var is set
-    todo(false, 'WebGL test suite is disabled!');
-    SimpleTest.finish();
-    return;
+  var kIsWindows = false;
+  var kIsMac = false;
+  var kIsLinux = false;
+  if (navigator.platform.indexOf("Win") == 0)
+    kIsWindows = true;
+  else if (navigator.platform.indexOf("Linux") == 0)
+    kIsLinux = true;
+  else if (navigator.platform.indexOf("Mac") == 0)
+    kIsMac = true;
+
+  // we currently disable this test on versions of Windows older than Windows Vista,
+  // due to failure to load ANGLE on win 2003 test slaves, see bug 582053.
+  if (kIsWindows) {
+    // code borrowed from browser/components/wintaskbar/test/browser_taskbar_preview.js
+    var isWinVistaOrHigher = false;
+    try {
+      netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+      var version = Components.classes["@mozilla.org/system-info;1"]
+                              .getService(Components.interfaces.nsIPropertyBag2)
+                              .getProperty("version");
+      isWinVistaOrHigher = (parseFloat(version) >= 6.0);
+    } catch (ex) {}
+    if (!isWinVistaOrHigher) {
+      dump("WebGL mochitest disabled on Windows versions older than Vista");
+      SimpleTest.finish();
+      return;
+    }
   }
 
-  var timesTestSuiteAlreadyRun = 0;
+  // we currently disable this test on version of Mac OSX older than 10.6,
+  // due to various weird failures, including one making getRenderbufferParameter tests
+  // on DEPTH_STENCIL fail
+  if (kIsMac) {
+    // code borrowed from browser/components/wintaskbar/test/browser_taskbar_preview.js
+    var is106orHigher = false;
+    try {
+      netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+      var version = Components.classes["@mozilla.org/system-info;1"]
+                              .getService(Components.interfaces.nsIPropertyBag2)
+                              .getProperty("version");
+      is106orHigher = (parseFloat(version) >= 10.6);
+    } catch (ex) { }
+    if (!is106orHigher) {
+      dump("WebGL mochitest disabled on Mac OSX versions older than 10.6");
+      SimpleTest.finish();
+      return;
+    }
+  }
 
   var saved_enabled_for_all_sites = getEnabledForAllSites();
-  var saved_angle_path = getAnglePath();
-  var saved_prefer_egl = getPreferEgl();
+  var saved_prefer_gl = getPreferGL();
 
   var statusElem = document.getElementById("status");
   var statusTextNode = document.createTextNode('');
   statusElem.appendChild(statusTextNode);
 
   var expectedtofailElem = document.getElementById("expectedtofail");
   var expectedtofailTextNode = document.createTextNode('');
   expectedtofailElem.appendChild(expectedtofailTextNode);
 
-  var testsExpectedToFail = loadTextFileSynchronous('failing_tests.txt').split('\n');
+  // Windows uses the ANGLE library for rendering. Until everything is perfect, this means a different set of
+  // failing tests. It's easier to do a platform check for Windows than for ANGLE itself.
+  // Moreover, we currently also have different tests failing on Mac and on Linux,
+  // presumably due to differences in the drivers.
+  var failingTestsFilename;
+  if (kIsWindows)
+    failingTestsFilename = 'failing_tests_windows.txt';
+  else if (kIsLinux)
+    failingTestsFilename = 'failing_tests_linux.txt';
+  else if (kIsMac)
+    failingTestsFilename = 'failing_tests_mac.txt';
+
+  var testsExpectedToFail = loadTextFileSynchronous(failingTestsFilename)
+                            .replace(/\r/g, '') // convert to unix line breaks
+                            .split('\n');
+
   var testsSuccessful = [];
 
   runTestSuite();
 }
 
 </script>
 </head>
 <body onload="start()">