Bug 718329 - Make fuzzy reftests take max tolerance values. r=roc
authorMatt Woodrow <mwoodrow@mozilla.com>
Mon, 23 Jan 2012 20:56:12 +1300
changeset 86326 05df0cddc299aac74bc6645a7264019012df0ce3
parent 86325 e29b78989aeb91b60b2030baf19f70ffd28b6484
child 86327 e170001f3e9b29b17e1ff32c3c7dfc6b19ccda9f
push id805
push userakeybl@mozilla.com
push dateWed, 01 Feb 2012 18:17:35 +0000
treeherdermozilla-aurora@6fb3bf232436 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersroc
bugs718329
milestone12.0a1
Bug 718329 - Make fuzzy reftests take max tolerance values. r=roc
layout/reftests/bugs/reftest.list
layout/reftests/css-gradients/reftest.list
layout/reftests/reftest-sanity/reftest.list
layout/tools/reftest/reftest.js
--- a/layout/reftests/bugs/reftest.list
+++ b/layout/reftests/bugs/reftest.list
@@ -1683,9 +1683,9 @@ needs-focus != 703186-1.html 703186-2.ht
 == 711359-1.html 711359-1-ref.html
 == 712849-1.html 712849-1-ref.html
 == 713856-static.html  713856-ref.html
 == 713856-dynamic.html 713856-ref.html
 == 714519-1-as.html 714519-1-ref.html
 == 714519-1-q.html 714519-1-ref.html
 == 714519-2-as.html 714519-2-ref.html
 == 714519-2-q.html 714519-2-ref.html
-fuzzy == 718521.html 718521-ref.html
+fuzzy-if(cocoaWidget,1,170) == 718521.html 718521-ref.html
--- a/layout/reftests/css-gradients/reftest.list
+++ b/layout/reftests/css-gradients/reftest.list
@@ -1,14 +1,14 @@
-fuzzy-if(azureQuartz) fails-if(Android) == linear-1a.html linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-1b.html linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-keywords-1a.html linear-keywords-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-keywords-1b.html linear-keywords-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-percent.html linear-percent-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-mix.html linear-mix-ref.html
+fuzzy-if(azureQuartz,1,34919) fails-if(Android) == linear-1a.html linear-1-ref.html
+fuzzy-if(azureQuartz,1,34919) fails-if(Android) == linear-1b.html linear-1-ref.html
+fuzzy-if(azureQuartz,1,11483) fails-if(Android) == linear-keywords-1a.html linear-keywords-1-ref.html
+fuzzy-if(azureQuartz,1,11483) fails-if(Android) == linear-keywords-1b.html linear-keywords-1-ref.html
+fuzzy-if(azureQuartz,1,10230) fails-if(Android) == linear-percent.html linear-percent-ref.html
+fuzzy-if(azureQuartz,1,17068) fails-if(Android) == linear-mix.html linear-mix-ref.html
 == linear-diagonal-1a.html linear-diagonal-1-ref.html
 == linear-diagonal-1b.html linear-diagonal-1-ref.html
 == linear-diagonal-1c.html linear-diagonal-1-ref.html
 == linear-diagonal-2a.html linear-diagonal-2-ref.html
 == linear-diagonal-2b.html linear-diagonal-2-ref.html
 == linear-diagonal-2c.html linear-diagonal-2-ref.html
 == linear-diagonal-3a.html linear-diagonal-3-ref.html
 == linear-diagonal-3b.html linear-diagonal-3-ref.html
@@ -31,30 +31,30 @@ fails-if(d2d) == linear-repeat-1f.html l
 fails-if(d2d) == linear-repeat-1g.html linear-repeat-1-ref.html # bug 582236
 == linear-size-1a.html linear-size-1-ref.html
 == linear-stops-1a.html linear-stops-1-ref.html
 == linear-stops-1b.html linear-stops-1-ref.html
 == linear-stops-1c.html linear-stops-1-ref.html
 == linear-stops-1d.html linear-stops-1-ref.html
 == linear-stops-1e.html linear-stops-1-ref.html
 == linear-stops-1f.html linear-stops-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-vertical-1a.html linear-vertical-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-vertical-1b.html linear-vertical-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-vertical-1c.html linear-vertical-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-vertical-1d.html linear-vertical-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == linear-vertical-1e.html linear-vertical-1-ref.html
+fuzzy-if(azureQuartz,1,9674) fails-if(Android) == linear-vertical-1a.html linear-vertical-1-ref.html
+fuzzy-if(azureQuartz,1,9674) fails-if(Android) == linear-vertical-1b.html linear-vertical-1-ref.html
+fuzzy-if(azureQuartz,1,9674) fails-if(Android) == linear-vertical-1c.html linear-vertical-1-ref.html
+fuzzy-if(azureQuartz,1,9674) fails-if(Android) == linear-vertical-1d.html linear-vertical-1-ref.html
+fuzzy-if(azureQuartz,1,9674) fails-if(Android) == linear-vertical-1e.html linear-vertical-1-ref.html
 == linear-viewport.html linear-viewport-ref.html
 == linear-zero-length-1a.html linear-zero-length-1-ref.html
 == linear-zero-length-1b.html linear-zero-length-1-ref.html
 == linear-zero-length-1c.html linear-zero-length-1-ref.html
 == nostops.html about:blank
 == onestop.html about:blank
-fuzzy-if(azureQuartz) fails-if(Android) random-if(d2d) == radial-1a.html radial-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == radial-2a.html radial-2-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == radial-2b.html radial-2-ref.html
+fuzzy-if(azureQuartz,1,1997) fails-if(Android) random-if(d2d) == radial-1a.html radial-1-ref.html
+fuzzy-if(azureQuartz,1,1926) fails-if(Android) == radial-2a.html radial-2-ref.html
+fuzzy-if(azureQuartz,1,1926) fails-if(Android) == radial-2b.html radial-2-ref.html
 == radial-position-1a.html radial-position-1-ref.html
 == radial-shape-closest-corner-1a.html radial-shape-closest-corner-1-ref.html
 == radial-shape-closest-side-1a.html radial-shape-closest-side-1-ref.html
 == radial-shape-farthest-corner-1a.html radial-shape-farthest-corner-1-ref.html
 == radial-shape-farthest-side-1a.html radial-shape-farthest-side-1-ref.html
 == radial-size-1a.html radial-size-1-ref.html
 == radial-zero-length-1a.html radial-zero-length-1-ref.html
 == radial-zero-length-1b.html radial-zero-length-1-ref.html
@@ -72,33 +72,33 @@ fails-if(d2d) == repeating-linear-1b.htm
 == twostops-1b.html twostops-1-ref.html
 fails-if(/Mac\x20OS\x20X\x2010\.[56]/.test(http.oscpu)) == twostops-1c.html twostops-1-ref.html # bug 524173
 == twostops-1d.html twostops-1-ref.html
 == twostops-1e.html twostops-1-ref.html
 == twostops-1f.html twostops-1-ref.html
 == twostops-1g.html twostops-1-ref.html
 
 # from http://www.xanthir.com/:4bhipd by way of http://a-ja.net/newgrad.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-1a.html aja-linear-1-ref.html
+fuzzy-if(azureQuartz,1,4646) fails-if(Android) == aja-linear-1a.html aja-linear-1-ref.html
 fails-if(!d2d) == aja-linear-1b.html aja-linear-1-ref.html # bug 526694
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-1c.html aja-linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-1d.html aja-linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-1e.html aja-linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-1f.html aja-linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-1g.html aja-linear-1-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-2a.html aja-linear-2-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-2b.html aja-linear-2-ref.html
+fuzzy-if(azureQuartz,1,4646) fails-if(Android) == aja-linear-1c.html aja-linear-1-ref.html
+fuzzy-if(azureQuartz,1,4646) fails-if(Android) == aja-linear-1d.html aja-linear-1-ref.html
+fuzzy-if(azureQuartz,1,4646) fails-if(Android) == aja-linear-1e.html aja-linear-1-ref.html
+fuzzy-if(azureQuartz,1,4646) fails-if(Android) == aja-linear-1f.html aja-linear-1-ref.html
+fuzzy-if(azureQuartz,1,4646) fails-if(Android) == aja-linear-1g.html aja-linear-1-ref.html
+fuzzy-if(azureQuartz,1,4675) fails-if(Android) == aja-linear-2a.html aja-linear-2-ref.html
+fuzzy-if(azureQuartz,1,4675) fails-if(Android) == aja-linear-2b.html aja-linear-2-ref.html
 fails == aja-linear-2c.html aja-linear-2-ref.html # bug 522607
 fails-if(!d2d) == aja-linear-2d.html aja-linear-2-ref.html # bug 526694
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-3a.html aja-linear-3-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-3b.html aja-linear-3-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-4a.html aja-linear-4-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-4b.html aja-linear-4-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) == aja-linear-5a.html aja-linear-5-ref.html
-fuzzy-if(azureQuartz) fails-if(Android) fails-if(/Mac\x20OS\x20X\x2010\.5/.test(http.oscpu)) == aja-linear-6a.html aja-linear-6-ref.html # bug 526708
+fuzzy-if(azureQuartz,1,4900) fails-if(Android) == aja-linear-3a.html aja-linear-3-ref.html
+fuzzy-if(azureQuartz,1,4900) fails-if(Android) == aja-linear-3b.html aja-linear-3-ref.html
+fuzzy-if(azureQuartz,1,8655) fails-if(Android) == aja-linear-4a.html aja-linear-4-ref.html
+fuzzy-if(azureQuartz,1,8655) fails-if(Android) == aja-linear-4b.html aja-linear-4-ref.html
+fuzzy-if(azureQuartz,2,7878) fails-if(Android) == aja-linear-5a.html aja-linear-5-ref.html
+fuzzy-if(azureQuartz,1,3910) fails-if(Android) fails-if(/Mac\x20OS\x20X\x2010\.5/.test(http.oscpu)) == aja-linear-6a.html aja-linear-6-ref.html # bug 526708
 fails == aja-linear-6b.html aja-linear-6-ref.html # bug 522607
 == height-dependence-1.html height-dependence-1-ref.html
 fails-if(cocoaWidget) == height-dependence-2.html height-dependence-2-ref.html # bug 535007
 == height-dependence-3.html height-dependence-3-ref.html
 
 fails-if(d2d) == linear-onestopposition-1.html linear-onestopposition-1-ref.html # bug 638664
 == linear-onestopposition-1.html linear-onestopposition-1-ref2.html
 fails-if(d2d) fails-if(cocoaWidget) == radial-onestopposition-1.html radial-onestopposition-1-ref.html # bug 638664
--- a/layout/reftests/reftest-sanity/reftest.list
+++ b/layout/reftests/reftest-sanity/reftest.list
@@ -124,15 +124,15 @@ fails pref(font.size.variable.x-western,
 fails pref(font.size.variable.x-western,"foo") == font-size-16.html font-default.html
 # a string pref
 pref(font.default.x-western,"serif") == font-serif.html font-default.html
 pref(font.default.x-western,"serif") != font-sans-serif.html font-default.html
 pref(font.default.x-western,"sans-serif") == font-sans-serif.html font-default.html
 pref(font.default.x-western,"sans-serif") != font-serif.html font-default.html
 fails pref(font.default.x-western,true) == font-serif.html font-default.html
 fails pref(font.default.x-western,0) == font-serif.html font-default.html
-# reftest syntax: fuzzy
-fuzzy == fuzzy.html fuzzy-ref.html
-fuzzy != too-fuzzy.html fuzzy-ref.html
-fuzzy-if(true) == fuzzy.html fuzzy-ref.html
-fuzzy-if(false) == fuzzy-ref.html fuzzy-ref.html
+# reftest syntax: fuzzy(maxPixelDifference,maxNumberDifferingPixels)
+fuzzy(1,250000) == fuzzy.html fuzzy-ref.html
+fuzzy(1,250000) != too-fuzzy.html fuzzy-ref.html
+fuzzy-if(true,1,250000) == fuzzy.html fuzzy-ref.html
+fuzzy-if(false,2,1) == fuzzy-ref.html fuzzy-ref.html
 # When using 565 fuzzy.html and fuzzy-ref.html will compare as equal
-fails fuzzy-if(false) random-if(Android) == fuzzy.html fuzzy-ref.html
+fails fuzzy-if(false,2,1) random-if(Android) == fuzzy.html fuzzy-ref.html
--- a/layout/tools/reftest/reftest.js
+++ b/layout/tools/reftest/reftest.js
@@ -683,27 +683,29 @@ function ReadManifest(aURL, inherited_st
 
         var expected_status = EXPECTED_PASS;
         var allow_silent_fail = false;
         var minAsserts = 0;
         var maxAsserts = 0;
         var needs_focus = false;
         var slow = false;
         var prefSettings = [];
+        var fuzzy_max_delta = 2;
+        var fuzzy_max_pixels = 1;
         
         while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail|pref|fuzzy)/)) {
             var item = items.shift();
             var stat;
             var cond;
-            var m = item.match(/^(fails|random|skip|silentfail|fuzzy)-if(\(.*\))$/);
+            var m = item.match(/^(fails|random|skip|silentfail)-if(\(.*\))$/);
             if (m) {
                 stat = m[1];
                 // Note: m[2] contains the parentheses, and we want them.
                 cond = Components.utils.evalInSandbox(m[2], sandbox);
-            } else if (item.match(/^(fails|random|skip|fuzzy)$/)) {
+            } else if (item.match(/^(fails|random|skip)$/)) {
                 stat = item;
                 cond = true;
             } else if (item == "needs-focus") {
                 needs_focus = true;
                 cond = false;
             } else if ((m = item.match(/^asserts\((\d+)(-\d+)?\)$/))) {
                 cond = false;
                 minAsserts = Number(m[1]);
@@ -764,29 +766,39 @@ function ReadManifest(aURL, inherited_st
                 } else if (valType == "number" && (parseInt(prefVal) == prefVal)) {
                     prefType = PREF_INTEGER;
                 } else {
                     throw "Error in pref value in manifest file " + aURL.spec + " line " + lineNo;
                 }
                 prefSettings.push( { name: prefName,
                                      type: prefType,
                                      value: prefVal } );
+            } else if ((m = item.match(/^fuzzy\((\d+),(\d+)\)$/))) {
+              cond = false;
+              expected_status = EXPECTED_FUZZY;
+              fuzzy_max_delta = Number(m[1]);
+              fuzzy_max_pixels = Number(m[2]);
+            } else if ((m = item.match(/^fuzzy-if\((.*?),(\d+),(\d+)\)$/))) {
+              cond = false;
+              if (Components.utils.evalInSandbox("(" + m[1] + ")", sandbox)) {
+                expected_status = EXPECTED_FUZZY;
+                fuzzy_max_delta = Number(m[2]);
+                fuzzy_max_pixels = Number(m[3]);
+              }
             } else {
                 throw "Error 1 in manifest file " + aURL.spec + " line " + lineNo;
             }
 
             if (cond) {
                 if (stat == "fails") {
                     expected_status = EXPECTED_FAIL;
                 } else if (stat == "random") {
                     expected_status = EXPECTED_RANDOM;
                 } else if (stat == "skip") {
                     expected_status = EXPECTED_DEATH;
-                } else if (stat == "fuzzy") {
-                    expected_status = EXPECTED_FUZZY;
                 } else if (stat == "silentfail") {
                     allow_silent_fail = true;
                 }
             }
         }
 
         expected_status = Math.max(expected_status, inherited_status);
 
@@ -845,16 +857,18 @@ function ReadManifest(aURL, inherited_st
                           expected: expected_status,
                           allowSilentFail: allow_silent_fail,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
                           prefSettings: prefSettings,
+                          fuzzyMaxDelta: fuzzy_max_delta,
+                          fuzzyMaxPixels: fuzzy_max_pixels,
                           url1: testURI,
                           url2: null } );
         } else if (items[0] == TYPE_SCRIPT) {
             if (items.length != 2)
                 throw "Error 4 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI] = runHttp
                             ? ServeFiles(aURL, httpDepth,
                                          listURL, [items[1]])
@@ -868,16 +882,18 @@ function ReadManifest(aURL, inherited_st
                           expected: expected_status,
                           allowSilentFail: allow_silent_fail,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
                           prefSettings: prefSettings,
+                          fuzzyMaxDelta: fuzzy_max_delta,
+                          fuzzyMaxPixels: fuzzy_max_pixels,
                           url1: testURI,
                           url2: null } );
         } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
             if (items.length != 3)
                 throw "Error 5 in manifest file " + aURL.spec + " line " + lineNo;
             var [testURI, refURI] = runHttp
                                   ? ServeFiles(aURL, httpDepth,
                                                listURL, [items[1], items[2]])
@@ -894,16 +910,18 @@ function ReadManifest(aURL, inherited_st
                           expected: expected_status,
                           allowSilentFail: allow_silent_fail,
                           prettyPath: prettyPath,
                           minAsserts: minAsserts,
                           maxAsserts: maxAsserts,
                           needsFocus: needs_focus,
                           slow: slow,
                           prefSettings: prefSettings,
+                          fuzzyMaxDelta: fuzzy_max_delta,
+                          fuzzyMaxPixels: fuzzy_max_pixels,
                           url1: testURI,
                           url2: refURI } );
         } else {
             throw "Error 6 in manifest file " + aURL.spec + " line " + lineNo;
         }
     }
 }
 
@@ -1408,17 +1426,18 @@ function RecordResult(testRunTime, error
             var maxDifference = {};
 
             differences = gWindowUtils.compareCanvases(gCanvas1, gCanvas2, maxDifference);
             equal = (differences == 0);
 
             // what is expected on this platform (PASS, FAIL, or RANDOM)
             var expected = gURLs[0].expected;
 
-            if (maxDifference.value > 0 && maxDifference.value <= 2) {
+            if (maxDifference.value > 0 && maxDifference.value <= gURLs[0].fuzzyMaxDelta &&
+                differences <= gURLs[0].fuzzyMaxPixels) {
                 if (equal) {
                     throw "Inconsistent result from compareCanvases.";
                 }
                 equal = expected == EXPECTED_FUZZY;
                 gDumpLog("REFTEST fuzzy match\n");
             }
 
             // whether the comparison result matches what is in the manifest