Bug 693703: added additional logging information for mochitests, incl. image reference differences r=bjacob
authorDoug Sherk <dsherk@mozilla.com>
Thu, 03 Nov 2011 10:50:40 -0400
changeset 79679 ce1330a5c9cb4a3d590dc81b378e5236279aa76f
parent 79678 7da669483434e13e0cf1c8557cd7de6d8803287b
child 79680 ad1f423c7755a2da364024f9b4665bf54e43f66c
push id3092
push userbjacob@mozilla.com
push dateThu, 03 Nov 2011 14:51:53 +0000
treeherdermozilla-inbound@ce1330a5c9cb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbjacob
bugs693703
milestone10.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 693703: added additional logging information for mochitests, incl. image reference differences r=bjacob Added some code to print to dump output of WebGL mochitest failures. Also added special code to handle incorrect reference images. It will now provide the user with a way to compare the reference and actual drawings.
content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html
content/canvas/test/webgl/conformance/misc/uninitialized-test.html
content/canvas/test/webgl/conformance/resources/webgl-test-utils.js
content/canvas/test/webgl/log-more-info-about-test-failures.patch
content/canvas/test/webgl/resources/js-test-pre.js
--- a/content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html
+++ b/content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html
@@ -117,17 +117,18 @@ function init()
     var bufFunction = new Uint8Array(width * height * 4);
     var bufMacro = new Uint8Array(width * height * 4);
 
     if (drawAndRead("canvasFunction", "vshaderFunction", bufFunction) == false ||
         drawAndRead("canvasMacro", "vshaderMacro", bufMacro) == false) {
         testFailed("Setup failed");
     } else {
         if (compareRendering(bufFunction, bufMacro, 4) == false)
-            testFailed("Rendering results are different");
+            testFailedRender("Rendering results are different", bufMacro,
+                             bufFunction, width, height);
         else
             testPassed("Rendering results are the same");
     }
 }
 
 init();
 successfullyParsed = true;
 </script>
--- a/content/canvas/test/webgl/conformance/misc/uninitialized-test.html
+++ b/content/canvas/test/webgl/conformance/misc/uninitialized-test.html
@@ -56,17 +56,19 @@ function checkNonZeroPixels(texture, tex
     gl.readPixels(0, 0, texWidth, texHeight, gl.RGBA, gl.UNSIGNED_BYTE, data);
 
     var k = 0;
     for (var y = 0; y < texHeight; ++y) {
         for (var x = 0; x < texWidth; ++x) {
             var index = (y * texWidth + x) * 4;
             if (x >= skipX && x < skipX + skipWidth && y >= skipY && y < skipY + skipHeight) {
                 if (data[index] != skipR || data[index + 1] != skipG || data[index + 2] != skipB || data[index + 3] != skipA) {
-                    testFailed("non-zero pixel values are wrong");
+                    testFailed("non-zero pixel values are wrong at (" + x + ", " + y + "), data was (" +
+                               data[index] + "," + data[index + 1] + "," + data[index + 2] + "," + data[index + 3] +
+                               ") should have been (" + skipR + "," + skipG + "," + skipB + "," + skipA + ")");
                     return;
                 }
             } else {
                 for (var i = 0; i < 4; ++i) {
                     if (data[index + i] != 0)
                         k++;
                 }
             }
--- a/content/canvas/test/webgl/conformance/resources/webgl-test-utils.js
+++ b/content/canvas/test/webgl/conformance/resources/webgl-test-utils.js
@@ -389,21 +389,29 @@ var drawQuad = function(gl, opt_color) {
 var checkCanvasRect = function(gl, x, y, width, height, color, msg, errorRange) {
   errorRange = errorRange || 0;
   var buf = new Uint8Array(width * height * 4);
   gl.readPixels(x, y, width, height, gl.RGBA, gl.UNSIGNED_BYTE, buf);
   for (var i = 0; i < width * height; ++i) {
     var offset = i * 4;
     for (var j = 0; j < color.length; ++j) {
       if (Math.abs(buf[offset + j] - color[j]) > errorRange) {
-        testFailed(msg);
         var was = buf[offset + 0].toString();
         for (j = 1; j < color.length; ++j) {
           was += "," + buf[offset + j];
         }
+
+        var cv = document.createElement('canvas');
+        cv.height = height;
+        cv.width = width;
+        var ctx = cv.getContext('2d');
+        ctx.fillStyle="rgba(" + color[0] + ", " + color[1] + ", " + color[2] + ", 255)";
+        ctx.fillRect(0, 0, width, height);
+        testFailedRender(msg, ctx, buf, width, height);
+
         debug('at (' + (i % width) + ', ' + Math.floor(i / width) +
               ') expected: ' + color + ' was ' + was);
         return;
       }
     }
   }
   testPassed(msg);
 };
new file mode 100644
--- /dev/null
+++ b/content/canvas/test/webgl/log-more-info-about-test-failures.patch
@@ -0,0 +1,181 @@
+# HG changeset patch
+# Parent 2dbd71999fe8a8da476ab6cc97716f7b7c294fb8
+# User Doug Sherk <dsherk@mozilla.com>
+Bug 693703: added additional logging information for mochitests, incl. image reference differences r=bjacob
+
+Added some code to print to dump output of WebGL mochitest failures. Also added
+special code to handle incorrect reference images. It will now provide the user
+with a way to compare the reference and actual drawings.
+
+diff --git a/content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html b/content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html
+--- a/content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html
++++ b/content/canvas/test/webgl/conformance/glsl/misc/glsl-function-nodes.html
+@@ -115,17 +115,18 @@ function init()
+     var bufFunction = new Uint8Array(width * height * 4);
+     var bufMacro = new Uint8Array(width * height * 4);
+ 
+     if (drawAndRead("canvasFunction", "vshaderFunction", bufFunction) == false ||
+         drawAndRead("canvasMacro", "vshaderMacro", bufMacro) == false) {
+         testFailed("Setup failed");
+     } else {
+         if (compareRendering(bufFunction, bufMacro, 4) == false)
+-            testFailed("Rendering results are different");
++            testFailedRender("Rendering results are different", bufMacro,
++                             bufFunction, width, height);
+         else
+             testPassed("Rendering results are the same");
+     }
+ }
+ 
+ init();
+ successfullyParsed = true;
+ </script>
+diff --git a/content/canvas/test/webgl/conformance/misc/uninitialized-test.html b/content/canvas/test/webgl/conformance/misc/uninitialized-test.html
+--- a/content/canvas/test/webgl/conformance/misc/uninitialized-test.html
++++ b/content/canvas/test/webgl/conformance/misc/uninitialized-test.html
+@@ -56,17 +56,19 @@ function checkNonZeroPixels(texture, tex
+     gl.readPixels(0, 0, texWidth, texHeight, gl.RGBA, gl.UNSIGNED_BYTE, data);
+ 
+     var k = 0;
+     for (var y = 0; y < texHeight; ++y) {
+         for (var x = 0; x < texWidth; ++x) {
+             var index = (y * texWidth + x) * 4;
+             if (x >= skipX && x < skipX + skipWidth && y >= skipY && y < skipY + skipHeight) {
+                 if (data[index] != skipR || data[index + 1] != skipG || data[index + 2] != skipB || data[index + 3] != skipA) {
+-                    testFailed("non-zero pixel values are wrong");
++                    testFailed("non-zero pixel values are wrong at (" + x + ", " + y + "), data was (" +
++                               data[index] + "," + data[index + 1] + "," + data[index + 2] + "," + data[index + 3] +
++                               ") should have been (" + skipR + "," + skipG + "," + skipB + "," + skipA + ")");
+                     return;
+                 }
+             } else {
+                 for (var i = 0; i < 4; ++i) {
+                     if (data[index + i] != 0)
+                         k++;
+                 }
+             }
+diff --git a/content/canvas/test/webgl/conformance/resources/webgl-test-utils.js b/content/canvas/test/webgl/conformance/resources/webgl-test-utils.js
+--- a/content/canvas/test/webgl/conformance/resources/webgl-test-utils.js
++++ b/content/canvas/test/webgl/conformance/resources/webgl-test-utils.js
+@@ -389,21 +389,29 @@ var drawQuad = function(gl, opt_color) {
+ var checkCanvasRect = function(gl, x, y, width, height, color, msg, errorRange) {
+   errorRange = errorRange || 0;
+   var buf = new Uint8Array(width * height * 4);
+   gl.readPixels(x, y, width, height, gl.RGBA, gl.UNSIGNED_BYTE, buf);
+   for (var i = 0; i < width * height; ++i) {
+     var offset = i * 4;
+     for (var j = 0; j < color.length; ++j) {
+       if (Math.abs(buf[offset + j] - color[j]) > errorRange) {
+-        testFailed(msg);
+         var was = buf[offset + 0].toString();
+         for (j = 1; j < color.length; ++j) {
+           was += "," + buf[offset + j];
+         }
++
++        var cv = document.createElement('canvas');
++        cv.height = height;
++        cv.width = width;
++        var ctx = cv.getContext('2d');
++        ctx.fillStyle="rgba(" + color[0] + ", " + color[1] + ", " + color[2] + ", 255)";
++        ctx.fillRect(0, 0, width, height);
++        testFailedRender(msg, ctx, buf, width, height);
++
+         debug('at (' + (i % width) + ', ' + Math.floor(i / width) +
+               ') expected: ' + color + ' was ' + was);
+         return;
+       }
+     }
+   }
+   testPassed(msg);
+ };
+diff --git a/content/canvas/test/webgl/resources/js-test-pre.js b/content/canvas/test/webgl/resources/js-test-pre.js
+--- a/content/canvas/test/webgl/resources/js-test-pre.js
++++ b/content/canvas/test/webgl/resources/js-test-pre.js
+@@ -81,16 +81,87 @@ function testPassed(msg)
+     reportTestResultsToHarness(true, msg);
+     debug('<span><span class="pass">PASS</span> ' + escapeHTML(msg) + '</span>');
+ }
+ 
+ function testFailed(msg)
+ {
+     reportTestResultsToHarness(false, msg);
+     debug('<span><span class="fail">FAIL</span> ' + escapeHTML(msg) + '</span>');
++    dump('FAIL: ' + msg + '\n');
++
++    var stack = (new Error).stack.split('\n');
++    if (!stack.length) {
++        return;
++    }
++
++    dump('STACK TRACE: \n');
++
++    stack.pop();
++    var index = 0, frame, messages = new Array();
++    // Match all .html files and print out the line in them.
++    while (stack.length && index != -1) {
++        frame = stack.pop();
++        index = frame.indexOf(".html:");
++        if (index != -1) {
++            messages.unshift(frame);
++        }
++    }
++
++    // Print out the first stack frame in JS and then stop.
++    if (stack.length) {
++        messages.unshift(stack.pop());
++    }
++
++    for (message in messages) {
++        dump(messages[message] + '\n');
++    }
++}
++
++function testFailedRender(msg, ref, test, width, height) 
++{
++    var refData;
++    if (typeof ref.getImageData == 'function') {
++        refData = ref.canvas.toDataURL();
++    } else {
++        refData = arrayToURLData(ref, width, height);
++    }
++
++    var testData;
++    if (typeof test.getImageData == 'function') {
++        testData = test.canvas.toDataURL();
++    } else {
++        testData = arrayToURLData(test, width, height);
++    }
++    
++    testFailed(msg);
++
++    var data = 'REFTEST TEST-UNEXPECTED-FAIL | ' + msg + ' | image comparison (==)\n' +
++               'REFTEST   IMAGE 1 (TEST): ' + testData + '\n' +
++               'REFTEST   IMAGE 2 (REFERENCE): ' + refData;
++    dump('FAIL: ' + data + '\n');
++    dump('To view the differences between these image renderings, go to the following link: https://hg.mozilla.org/mozilla-central/raw-file/tip/layout/tools/reftest/reftest-analyzer.xhtml#log=' +
++    encodeURIComponent(encodeURIComponent(data)) + '\n');
++}
++
++function arrayToURLData(buf, width, height)
++{
++    var cv = document.createElement('canvas');
++    cv.height = height;
++    cv.width = width;
++    var ctx = cv.getContext('2d');
++    var imgd = ctx.getImageData(0, 0, width, height);
++    for (i = 0; i < height * width; ++i) {
++        offset = i * 4;
++        for (j = 0; j < 4; j++) {
++            imgd.data[offset + j] = buf[offset + j];
++        }
++    }
++    ctx.putImageData(imgd, 0, 0);
++    return cv.toDataURL();
+ }
+ 
+ function areArraysEqual(_a, _b)
+ {
+     try {
+         if (_a.length !== _b.length)
+             return false;
+         for (var i = 0; i < _a.length; i++)
--- a/content/canvas/test/webgl/resources/js-test-pre.js
+++ b/content/canvas/test/webgl/resources/js-test-pre.js
@@ -81,16 +81,87 @@ function testPassed(msg)
     reportTestResultsToHarness(true, msg);
     debug('<span><span class="pass">PASS</span> ' + escapeHTML(msg) + '</span>');
 }
 
 function testFailed(msg)
 {
     reportTestResultsToHarness(false, msg);
     debug('<span><span class="fail">FAIL</span> ' + escapeHTML(msg) + '</span>');
+    dump('FAIL: ' + msg + '\n');
+
+    var stack = (new Error).stack.split('\n');
+    if (!stack.length) {
+        return;
+    }
+
+    dump('STACK TRACE: \n');
+
+    stack.pop();
+    var index = 0, frame, messages = new Array();
+    // Match all .html files and print out the line in them.
+    while (stack.length && index != -1) {
+        frame = stack.pop();
+        index = frame.indexOf(".html:");
+        if (index != -1) {
+            messages.unshift(frame);
+        }
+    }
+
+    // Print out the first stack frame in JS and then stop.
+    if (stack.length) {
+        messages.unshift(stack.pop());
+    }
+
+    for (message in messages) {
+        dump(messages[message] + '\n');
+    }
+}
+
+function testFailedRender(msg, ref, test, width, height) 
+{
+    var refData;
+    if (typeof ref.getImageData == 'function') {
+        refData = ref.canvas.toDataURL();
+    } else {
+        refData = arrayToURLData(ref, width, height);
+    }
+
+    var testData;
+    if (typeof test.getImageData == 'function') {
+        testData = test.canvas.toDataURL();
+    } else {
+        testData = arrayToURLData(test, width, height);
+    }
+    
+    testFailed(msg);
+
+    var data = 'REFTEST TEST-UNEXPECTED-FAIL | ' + msg + ' | image comparison (==)\n' +
+               'REFTEST   IMAGE 1 (TEST): ' + testData + '\n' +
+               'REFTEST   IMAGE 2 (REFERENCE): ' + refData;
+    dump('FAIL: ' + data + '\n');
+    dump('To view the differences between these image renderings, go to the following link: https://hg.mozilla.org/mozilla-central/raw-file/tip/layout/tools/reftest/reftest-analyzer.xhtml#log=' +
+    encodeURIComponent(encodeURIComponent(data)) + '\n');
+}
+
+function arrayToURLData(buf, width, height)
+{
+    var cv = document.createElement('canvas');
+    cv.height = height;
+    cv.width = width;
+    var ctx = cv.getContext('2d');
+    var imgd = ctx.getImageData(0, 0, width, height);
+    for (i = 0; i < height * width; ++i) {
+        offset = i * 4;
+        for (j = 0; j < 4; j++) {
+            imgd.data[offset + j] = buf[offset + j];
+        }
+    }
+    ctx.putImageData(imgd, 0, 0);
+    return cv.toDataURL();
 }
 
 function areArraysEqual(_a, _b)
 {
     try {
         if (_a.length !== _b.length)
             return false;
         for (var i = 0; i < _a.length; i++)