Bug 1525314 - Auto-fuzz WR on Android with maxDifference<=2. r=jmaher
☠☠ backed out by 2e7a7f345b27 ☠ ☠
authorKartikaya Gupta <kgupta@mozilla.com>
Wed, 03 Jul 2019 17:59:55 +0000
changeset 544154 95790a07a93c5a9faf1ededcb0c4c05191e784b4
parent 544153 28f52fd3934ed3f724aa132253c1f25cc49ae67d
child 544155 02399933ac4b1466f44099766aed5c3a76f69a2e
push id2131
push userffxbld-merge
push dateMon, 26 Aug 2019 18:30:20 +0000
treeherdermozilla-release@b19ffb3ca153 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1525314
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1525314 - Auto-fuzz WR on Android with maxDifference<=2. r=jmaher Due to the sheer number of tests that exhibit a random fuzz with maxDifference=1 and maxDifference=2 with WR on Android, it's easier to just tweak the harness to autofuzz these away. This adds machinery to do so, and also adds a new annotation that can be used to disable the autofuzzing on specific tests. Depends on D36794 Differential Revision: https://phabricator.services.mozilla.com/D36796
layout/reftests/reftest-sanity/reftest.list
layout/tools/reftest/README.txt
layout/tools/reftest/manifest.jsm
layout/tools/reftest/reftest.jsm
--- a/layout/reftests/reftest-sanity/reftest.list
+++ b/layout/reftests/reftest-sanity/reftest.list
@@ -144,31 +144,31 @@ test-pref(font.size.variable.x-western,2
 test-pref(font.size.variable.x-western,24) != font-default.html font-size-16.html
 fails test-pref(font.size.variable.x-western,false) == font-default.html font-size-16.html
 fails test-pref(font.size.variable.x-western,"foo") == font-default.html font-size-16.html
 ref-pref(font.size.variable.x-western,16) test-pref(font.size.variable.x-western,24) != font-default.html font-default.html
 ref-pref(font.size.variable.x-western,24) test-pref(font.size.variable.x-western,16) != font-default.html font-default.html
 ref-pref(font.size.variable.x-western,24) test-pref(font.size.variable.x-western,24) == font-default.html font-default.html
 
 # reftest syntax: fuzzy(0-maxPixelDifference,0-maxNumberDifferingPixels)
-fuzzy(0-1,0-250000) == fuzzy.html fuzzy-ref.html
-fuzzy(0-1,0-250000) != too-fuzzy.html fuzzy-ref.html
-fuzzy-if(true,0-1,0-250000) == fuzzy.html fuzzy-ref.html
-fuzzy-if(false,0-2,0-1) == fuzzy-ref.html fuzzy-ref.html
+noautofuzz fuzzy(0-1,0-250000) == fuzzy.html fuzzy-ref.html
+noautofuzz fuzzy(0-1,0-250000) != too-fuzzy.html fuzzy-ref.html
+noautofuzz fuzzy-if(true,0-1,0-250000) == fuzzy.html fuzzy-ref.html
+noautofuzz fuzzy-if(false,0-2,0-1) == fuzzy-ref.html fuzzy-ref.html
 # test some ranged fuzzes
-fuzzy(1-10,1-250000) fuzzy-if(false,5-10,0-250000) == fuzzy.html fuzzy-ref.html
-fuzzy(0-0,0-250000) != fuzzy.html fuzzy-ref.html
-fuzzy(0-1,0-2) != fuzzy.html fuzzy-ref.html
+noautofuzz fuzzy(1-10,1-250000) fuzzy-if(false,5-10,0-250000) == fuzzy.html fuzzy-ref.html
+noautofuzz fuzzy(0-0,0-250000) != fuzzy.html fuzzy-ref.html
+noautofuzz fuzzy(0-1,0-2) != fuzzy.html fuzzy-ref.html
 # If enabled, the following two should result in UNEXPECTED-PASS because
 # they are both overfuzzed
-# fuzzy(3-4,0-250000) == fuzzy.html fuzzy-ref.html
-# fuzzy(0-1,250001-250002) == fuzzy.html fuzzy-ref.html
+# noautofuzz fuzzy(3-4,0-250000) == fuzzy.html fuzzy-ref.html
+# noautofuzz fuzzy(0-1,250001-250002) == fuzzy.html fuzzy-ref.html
 #
 # When using 565 fuzzy.html and fuzzy-ref.html will compare as equal
-fails fuzzy-if(false,0-2,0-1) random-if(Android) == fuzzy.html fuzzy-ref.html
+noautofuzz fails fuzzy-if(false,0-2,0-1) random-if(Android) == fuzzy.html fuzzy-ref.html
 
 # Test that reftest-no-paint fails correctly.
 fails-if(layerChecksEnabled) == reftest-no-paint.html reftest-no-paint-ref.html
 
 skip-if(!asyncPan||!browserIsRemote) == async-scroll-1a.html async-scroll-1-ref.html
 
 # Disable low-res painting for this test as it will cause more to
 # be drawn than we want.
--- a/layout/tools/reftest/README.txt
+++ b/layout/tools/reftest/README.txt
@@ -174,16 +174,24 @@ 2. A test item
           Loading the test and reference is known to assert between
           minCount and maxCount times, inclusive.
           NOTE: See above regarding canvas caching.
 
       asserts-if(condition,count)
       asserts-if(condition,minCount-maxCount)
           Same as above, but only if condition is true.
 
+      noautofuzz
+          Disables the autofuzzing behaviour hard-coded in the reftest harness
+          for specific platform configurations. The autofuzzing is intended to
+          compensate for inherent nondeterminism that results in intermittently
+          fuzzy results (with small amounts of fuzz) across many/all tests on
+          a given platform. Specifying 'noautofuzz' on the test will disable
+          the autofuzzing for that test and require an exact match.
+
       Conditions are JavaScript expressions *without spaces* in them.
       They are evaluated in a sandbox in which a limited set of
       variables are defined.  See the BuildConditionSandbox function in
       layout/tools/reftest.js for details.
 
       Examples of using conditions:
           fails-if(winWidget) == test reference
           asserts-if(cocoaWidget,2) load crashtest
--- a/layout/tools/reftest/manifest.jsm
+++ b/layout/tools/reftest/manifest.jsm
@@ -117,18 +117,19 @@ function ReadManifest(aURL, aFilter)
         var slow = false;
         var testPrefSettings = defaultTestPrefSettings.concat();
         var refPrefSettings = defaultRefPrefSettings.concat();
         var fuzzy_delta = { min: 0, max: 2 };
         var fuzzy_pixels = { min: 0, max: 1 };
         var chaosMode = false;
         var wrCapture = { test: false, ref: false };
         var nonSkipUsed = false;
+        var noAutoFuzz = false;
 
-        while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail|pref|test-pref|ref-pref|fuzzy|chaos-mode|wr-capture|wr-capture-ref)/)) {
+        while (items[0].match(/^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail|pref|test-pref|ref-pref|fuzzy|chaos-mode|wr-capture|wr-capture-ref|noautofuzz)/)) {
             var item = items.shift();
             var stat;
             var cond;
             var m = item.match(/^(fails|random|skip|silentfail)-if(\(.*\))$/);
             if (m) {
                 stat = m[1];
                 // Note: m[2] contains the parentheses, and we want them.
                 cond = Cu.evalInSandbox(m[2], sandbox);
@@ -206,16 +207,19 @@ function ReadManifest(aURL, aFilter)
                 cond = false;
                 chaosMode = true;
             } else if (item == "wr-capture") {
                 cond = false;
                 wrCapture.test = true;
             } else if (item == "wr-capture-ref") {
                 cond = false;
                 wrCapture.ref = true;
+            } else if (item == "noautofuzz") {
+                cond = false;
+                noAutoFuzz = true;
             } else {
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unexpected item " + item;
             }
 
             if (stat != "skip") {
                 nonSkipUsed = true;
             }
 
@@ -311,17 +315,18 @@ function ReadManifest(aURL, aFilter)
                           fuzzyMaxDelta: fuzzy_delta.max,
                           fuzzyMinPixels: fuzzy_pixels.min,
                           fuzzyMaxPixels: fuzzy_pixels.max,
                           runHttp: runHttp,
                           httpDepth: httpDepth,
                           url1: items[1],
                           url2: null,
                           chaosMode: chaosMode,
-                          wrCapture: wrCapture }, aFilter);
+                          wrCapture: wrCapture,
+                          noAutoFuzz: noAutoFuzz }, aFilter);
         } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL || items[0] == TYPE_PRINT) {
             if (items.length != 3)
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
 
             if (items[0] == TYPE_REFTEST_NOTEQUAL &&
                 expected_status == EXPECTED_FUZZY &&
                 (fuzzy_delta.min > 0 || fuzzy_pixels.min > 0)) {
                 throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": minimum fuzz must be zero for tests of type " + items[0];
@@ -359,17 +364,18 @@ function ReadManifest(aURL, aFilter)
                           fuzzyMaxDelta: fuzzy_delta.max,
                           fuzzyMinPixels: fuzzy_pixels.min,
                           fuzzyMaxPixels: fuzzy_pixels.max,
                           runHttp: runHttp,
                           httpDepth: httpDepth,
                           url1: items[1],
                           url2: items[2],
                           chaosMode: chaosMode,
-                          wrCapture: wrCapture }, aFilter);
+                          wrCapture: wrCapture,
+                          noAutoFuzz: noAutoFuzz }, aFilter);
         } else {
             throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0];
         }
     }
 }
 
 // Read all available data from an input stream and return it
 // as a string.
--- a/layout/tools/reftest/reftest.jsm
+++ b/layout/tools/reftest/reftest.jsm
@@ -64,16 +64,25 @@ g.logger = new StructuredLogger('reftest
 var logger = g.logger;
 
 function TestBuffer(str)
 {
   logger.debug(str);
   g.testLog.push(str);
 }
 
+function isWebRenderOnAndroidDevice() {
+  var xr = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
+  // This is the best we can do for now; maybe in the future we'll have
+  // more correct detection of this case.
+  return xr.OS == "Android" &&
+      g.browserIsRemote &&
+      g.windowUtils.layerManagerType == "WebRender";
+}
+
 function FlushTestBuffer()
 {
   // In debug mode, we've dumped all these messages already.
   if (g.logLevel !== 'debug') {
     for (var i = 0; i < g.testLog.length; ++i) {
       logger.info("Saved log: " + g.testLog[i]);
     }
   }
@@ -1100,26 +1109,43 @@ function RecordResult(testRunTime, error
             var differences;
             // whether the two renderings match:
             var equal;
             var maxDifference = {};
             // whether the allowed fuzziness from the annotations is exceeded
             // by the actual comparison results
             var fuzz_exceeded = false;
 
+            // what is expected on this platform (PASS, FAIL, RANDOM, or FUZZY)
+            var expected = g.urls[0].expected;
+
             differences = g.windowUtils.compareCanvases(g.canvas1, g.canvas2, maxDifference);
+
+            if (g.urls[0].noAutoFuzz) {
+                // Autofuzzing is disabled
+            } else if (isWebRenderOnAndroidDevice() && maxDifference.value <= 2 && differences > 0) {
+                // Autofuzz for WR on Android physical devices: Reduce any
+                // maxDifference of 2 to 0, because we get a lot of off-by-ones
+                // and off-by-twos that are very random and hard to annotate.
+                // In cases where the difference on any pixel component is more
+                // than 2 we require manual annotation. Note that this applies
+                // to both == tests and != tests, so != tests don't
+                // inadvertently pass due to a random off-by-one pixel
+                // difference.
+                logger.info(`REFTEST wr-on-android dropping fuzz of (${maxDifference.value}, ${differences}) to (0, 0)`);
+                maxDifference.value = 0;
+                differences = 0;
+            }
+
             equal = (differences == 0);
 
             if (maxDifference.value > 0 && equal) {
                 throw "Inconsistent result from compareCanvases.";
             }
 
-            // what is expected on this platform (PASS, FAIL, or RANDOM)
-            var expected = g.urls[0].expected;
-
             if (expected == EXPECTED_FUZZY) {
                 logger.info(`REFTEST fuzzy test ` +
                             `(${g.urls[0].fuzzyMinDelta}, ${g.urls[0].fuzzyMinPixels}) <= ` +
                             `(${maxDifference.value}, ${differences}) <= ` +
                             `(${g.urls[0].fuzzyMaxDelta}, ${g.urls[0].fuzzyMaxPixels})`);
                 fuzz_exceeded = maxDifference.value > g.urls[0].fuzzyMaxDelta ||
                                 differences > g.urls[0].fuzzyMaxPixels;
                 equal = !fuzz_exceeded &&