Bug 1510696 - Make width/height of reftests configurable over Marionette r=jgraham,ato
authorGeoffrey Sneddon <me@gsnedders.com>
Fri, 11 Jan 2019 18:11:30 +0000
changeset 453525 0a9c477577dec9ab7b8f4bf6e424e1c19f4db256
parent 453524 0655bfda512a0f795754f830dbe6a37e1f3bbbab
child 453527 ab971e61cf793a8a7ed3918aa2a86724fbbb4cde
push id111104
push usernerli@mozilla.com
push dateFri, 11 Jan 2019 22:00:51 +0000
treeherdermozilla-inbound@1ddef6cf6c7e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham, ato
bugs1510696
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1510696 - Make width/height of reftests configurable over Marionette r=jgraham,ato Depends on D13282 Differential Revision: https://phabricator.services.mozilla.com/D13283
testing/marionette/driver.js
testing/marionette/harness/marionette_harness/tests/unit/test_reftest.py
testing/marionette/harness/marionette_harness/www/reftest/mostly-teal-700x700.html
testing/marionette/harness/marionette_harness/www/reftest/teal-700x700.html
testing/marionette/reftest.js
testing/web-platform/meta/css/CSS2/generated-content/content-175.xht.ini
testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-collapse-001.xht.ini
testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-103.xht.ini
testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-104.xht.ini
testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-113.xht.ini
testing/web-platform/meta/css/CSS2/normal-flow/blocks-020.xht.ini
testing/web-platform/meta/css/CSS2/normal-flow/blocks-021.xht.ini
testing/web-platform/meta/css/CSS2/normal-flow/blocks-022.xht.ini
testing/web-platform/meta/css/CSS2/normal-flow/inline-replaced-width-014.xht.ini
testing/web-platform/meta/css/CSS2/positioning/absolute-replaced-width-003c.xht.ini
testing/web-platform/meta/css/CSS2/positioning/left-offset-percentage-002.xht.ini
testing/web-platform/meta/css/CSS2/positioning/top-offset-percentage-002.xht.ini
testing/web-platform/meta/css/CSS2/tables/separated-border-model-007.xht.ini
testing/web-platform/meta/css/CSS2/tables/separated-border-model-008.xht.ini
testing/web-platform/meta/css/CSS2/tables/separated-border-model-009.xht.ini
testing/web-platform/meta/css/CSS2/visuren/top-114.xht.ini
testing/web-platform/meta/css/css-backgrounds/border-image-outset-003.html.ini
testing/web-platform/meta/css/css-flexbox/flexbox_visibility-collapse-line-wrapping.html.ini
testing/web-platform/meta/css/css-images/tiled-radial-gradients.html.ini
testing/web-platform/meta/css/css-multicol/multicol-fill-auto-001.xht.ini
testing/web-platform/meta/css/css-multicol/multicol-inherit-001.xht.ini
testing/web-platform/meta/css/css-multicol/multicol-overflow-000.xht.ini
testing/web-platform/meta/css/css-transforms/matrix/svg-matrix-005.html.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-007.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-013.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-019.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-025.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-031.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-037.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-043.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-049.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-055.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-061.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-067.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-073.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-079.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-085.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-091.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-097.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-107.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-115.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-123.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-131.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-139.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-147.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-155.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-163.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-171.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-179.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-187.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-195.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-203.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-211.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-219.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-227.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-006.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-012.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-018.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-024.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-030.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-036.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-042.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-048.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-054.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-060.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-066.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-072.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-078.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-084.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-090.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-096.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-106.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-114.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-122.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-130.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-138.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-146.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-154.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-162.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-170.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-178.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-186.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-194.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-202.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-210.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-218.xht.ini
testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-226.xht.ini
testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vlr-007.xht.ini
testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vlr-008.xht.ini
testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vrl-008.xht.ini
testing/web-platform/meta/css/css-writing-modes/sizing-orthog-prct-htb-in-vrl-001.xht.ini
testing/web-platform/meta/css/css-writing-modes/text-indent-vlr-011.xht.ini
testing/web-platform/meta/css/css-writing-modes/text-indent-vlr-013.xht.ini
testing/web-platform/meta/css/css-writing-modes/text-indent-vrl-010.xht.ini
testing/web-platform/meta/css/css-writing-modes/text-indent-vrl-012.xht.ini
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
--- a/testing/marionette/driver.js
+++ b/testing/marionette/driver.js
@@ -3501,35 +3501,35 @@ GeckoDriver.prototype.setupReftest = asy
 
   let {urlCount = {}, screenshot = "unexpected"} = cmd.parameters;
   if (!["always", "fail", "unexpected"].includes(screenshot)) {
     throw new InvalidArgumentError(
         "Value of `screenshot` should be 'always', 'fail' or 'unexpected'");
   }
 
   this._reftest = new reftest.Runner(this);
-  await this._reftest.setup(urlCount, screenshot);
+  this._reftest.setup(urlCount, screenshot);
 };
 
 
 /** Run a reftest. */
 GeckoDriver.prototype.runReftest = async function(cmd) {
-  let {test, references, expected, timeout} = cmd.parameters;
+  let {test, references, expected, timeout, width, height} = cmd.parameters;
 
   if (!this._reftest) {
     throw new UnsupportedOperationError(
         "Called reftest:run before reftest:start");
   }
 
   assert.string(test);
   assert.string(expected);
   assert.array(references);
 
   return {value: await this._reftest.run(
-      test, references, expected, timeout)};
+      test, references, expected, timeout, width, height)};
 };
 
 /**
  * End a reftest run.
  *
  * Closes the reftest window (without changing the current window handle),
  * and removes cached canvases.
  */
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_reftest.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_reftest.py
@@ -53,8 +53,32 @@ class TestReftest(MarionetteTestCase):
         self.marionette._send_message("reftest:setup", {"screenshot": "unexpected"})
         rv = self.marionette._send_message("reftest:run",
                                            {"test": test_page,
                                             "references": [[test_page_2, [], "=="]],
                                             "expected": "PASS",
                                             "timeout": 10 * 1000})
         self.marionette._send_message("reftest:teardown", {})
         self.assertEqual(u"PASS", rv[u"value"][u"status"])
+
+    def test_cache_multiple_sizes(self):
+        teal = self.fixtures.where_is("reftest/teal-700x700.html")
+        mostly_teal = self.fixtures.where_is("reftest/mostly-teal-700x700.html")
+
+        self.marionette._send_message("reftest:setup", {"screenshot": "unexpected"})
+        rv = self.marionette._send_message("reftest:run",
+                                           {"test": teal,
+                                            "references": [[mostly_teal, [], "=="]],
+                                            "expected": "PASS",
+                                            "timeout": 10 * 1000,
+                                            "width": 600,
+                                            "height": 600})
+        self.assertEqual(u"PASS", rv[u"value"][u"status"])
+
+        rv = self.marionette._send_message("reftest:run",
+                                           {"test": teal,
+                                            "references": [[mostly_teal, [], "=="]],
+                                            "expected": "PASS",
+                                            "timeout": 10 * 1000,
+                                            "width": 700,
+                                            "height": 700})
+        self.assertEqual(u"FAIL", rv[u"value"][u"status"])
+        self.marionette._send_message("reftest:teardown", {})
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/www/reftest/mostly-teal-700x700.html
@@ -0,0 +1,21 @@
+<!doctype html>
+<style>
+  * {
+      margin: 0;
+      padding: 0;
+  }
+
+  html {
+      background: teal;
+  }
+
+  div {
+      position: absolute;
+      top: 600px;
+      left: 600px;
+      height: 100px;
+      width: 100px;
+      background: orange;
+  }
+</style>
+<div></div>
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/www/reftest/teal-700x700.html
@@ -0,0 +1,21 @@
+<!doctype html>
+<style>
+  * {
+      margin: 0;
+      padding: 0;
+  }
+
+  html {
+      background: teal;
+  }
+
+  div {
+      position: absolute;
+      top: 600px;
+      left: 600px;
+      height: 100px;
+      width: 100px;
+      background: transparent;
+  }
+</style>
+<div></div>
--- a/testing/marionette/reftest.js
+++ b/testing/marionette/reftest.js
@@ -28,35 +28,35 @@ const SCREENSHOT_MODE = {
 
 const STATUS = {
   PASS: "PASS",
   FAIL: "FAIL",
   ERROR: "ERROR",
   TIMEOUT: "TIMEOUT",
 };
 
-const REFTEST_WIDTH = 600;
-const REFTEST_HEIGHT = 600;
+const DEFAULT_REFTEST_WIDTH = 600;
+const DEFAULT_REFTEST_HEIGHT = 600;
 
 /**
  * Implements an fast runner for web-platform-tests format reftests
  * c.f. http://web-platform-tests.org/writing-tests/reftests.html.
  *
  * @namespace
  */
 this.reftest = {};
 
 /**
  * @memberof reftest
  * @class Runner
  */
 reftest.Runner = class {
   constructor(driver) {
     this.driver = driver;
-    this.canvasCache = new Map([[null, []]]);
+    this.canvasCache = new DefaultMap(undefined, () => new Map([[null, []]]));
     this.windowUtils = null;
     this.lastURL = null;
     this.remote = Preferences.get(PREF_E10S);
   }
 
   /**
    * Setup the required environment for running reftests.
    *
@@ -65,71 +65,83 @@ reftest.Runner = class {
    *
    * @param {Object.<Number>} urlCount
    *     Object holding a map of URL: number of times the URL
    *     will be opened during the reftest run, where that's
    *     greater than 1.
    * @param {string} screenshotMode
    *     String enum representing when screenshots should be taken
    */
-  async setup(urlCount, screenshotMode) {
+  setup(urlCount, screenshotMode) {
     this.parentWindow =  assert.open(this.driver.getCurrentWindow());
 
     this.screenshotMode = SCREENSHOT_MODE[screenshotMode] ||
         SCREENSHOT_MODE.unexpected;
 
     this.urlCount = Object.keys(urlCount || {})
         .reduce((map, key) => map.set(key, urlCount[key]), new Map());
-
-    await this.ensureWindow();
   }
 
-  async ensureWindow(timeout) {
+  async ensureWindow(timeout, width, height) {
+    logger.debug(`ensuring we have a window ${width}x${height}`);
+
     if (this.reftestWin && !this.reftestWin.closed) {
-      return this.reftestWin;
+      let browserRect = this.reftestWin.gBrowser.getBoundingClientRect();
+      if (browserRect.width === width && browserRect.height === height) {
+        return this.reftestWin;
+      }
+      logger.debug(`current: ${browserRect.width}x${browserRect.height}`);
     }
 
     let reftestWin;
     if (Services.appinfo.OS == "Android") {
       logger.debug("Using current window");
       reftestWin = this.parentWindow;
       await this.driver.listener.get({
         commandID: this.driver.listener.activeMessageId,
         pageTimeout: timeout,
         url: "about:blank",
         loadEventExpected: false});
-
     } else {
       logger.debug("Using separate window");
-      reftestWin = await this.openWindow();
+      if (this.reftestWin && !this.reftestWin.closed) {
+        this.reftestWin.close();
+      }
+      reftestWin = await this.openWindow(width, height);
     }
 
-    this.setupWindow(reftestWin);
+    this.setupWindow(reftestWin, width, height);
     this.windowUtils = reftestWin.windowUtils;
     this.reftestWin = reftestWin;
 
     let found = this.driver.findWindow([reftestWin], () => true);
     await this.driver.setWindowHandle(found, true);
 
+    let browserRect = reftestWin.gBrowser.getBoundingClientRect();
+    logger.debug(`new: ${browserRect.width}x${browserRect.height}`);
+
     return reftestWin;
   }
 
-  async openWindow() {
+  async openWindow(width, height) {
+    assert.positiveInteger(width);
+    assert.positiveInteger(height);
+
     let reftestWin = this.parentWindow.open(
         "chrome://marionette/content/reftest.xul",
         "reftest",
-        `chrome,height=${REFTEST_WIDTH},width=${REFTEST_HEIGHT}`);
+        `chrome,height=${width},width=${height}`);
 
     await new Promise(resolve => {
       reftestWin.addEventListener("load", resolve, {once: true});
     });
     return reftestWin;
   }
 
-  setupWindow(reftestWin) {
+  setupWindow(reftestWin, width, height) {
     let browser;
     if (Services.appinfo.OS === "Android") {
       browser = reftestWin.document.getElementsByTagName("browser")[0];
       browser.setAttribute("remote", "false");
     } else {
       browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
       browser.permanentKey = {};
       browser.setAttribute("id", "browser");
@@ -138,21 +150,21 @@ reftest.Runner = class {
       browser.setAttribute("primary", "true");
       if (this.remote) {
         browser.setAttribute("remote", "true");
         browser.setAttribute("remoteType", "web");
       } else {
         browser.setAttribute("remote", "false");
       }
     }
-    // Make sure the browser element is exactly 600x600, no matter
+    // Make sure the browser element is exactly the right size, no matter
     // what size our window is
     const windowStyle = `padding: 0px; margin: 0px; border:none;
-min-width: ${REFTEST_WIDTH}px; min-height: ${REFTEST_HEIGHT}px;
-max-width: ${REFTEST_WIDTH}px; max-height: ${REFTEST_HEIGHT}px`;
+min-width: ${width}px; min-height: ${height}px;
+max-width: ${width}px; max-height: ${height}px`;
     browser.setAttribute("style", windowStyle);
 
     if (Services.appinfo.OS !== "Android") {
       let doc = reftestWin.document.documentElement;
       while (doc.firstChild) {
         doc.firstChild.remove();
       }
       doc.appendChild(browser);
@@ -208,30 +220,33 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
    * @param {string} expected
    *     Expected test outcome (e.g. <tt>PASS</tt>, <tt>FAIL</tt>).
    * @param {number} timeout
    *     Test timeout in milliseconds.
    *
    * @return {Object}
    *     Result object with fields status, message and extra.
    */
-  async run(testUrl, references, expected, timeout) {
+  async run(testUrl, references, expected, timeout,
+      width = DEFAULT_REFTEST_WIDTH,
+      height = DEFAULT_REFTEST_HEIGHT) {
 
     let timeoutHandle;
 
     let timeoutPromise = new Promise(resolve => {
       timeoutHandle = this.parentWindow.setTimeout(() => {
         resolve({status: STATUS.TIMEOUT, message: null, extra: {}});
       }, timeout);
     });
 
     let testRunner = (async () => {
       let result;
       try {
-        result = await this.runTest(testUrl, references, expected, timeout);
+        result = await this.runTest(testUrl, references, expected, timeout,
+            width, height);
       } catch (e) {
         result = {
           status: STATUS.ERROR,
           message: String(e),
           stack: e.stack,
           extra: {},
         };
       }
@@ -242,18 +257,18 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
     this.parentWindow.clearTimeout(timeoutHandle);
     if (result.status === STATUS.TIMEOUT) {
       this.abort();
     }
 
     return result;
   }
 
-  async runTest(testUrl, references, expected, timeout) {
-    let win = await this.ensureWindow(timeout);
+  async runTest(testUrl, references, expected, timeout, width, height) {
+    let win = await this.ensureWindow(timeout, width, height);
 
     function toBase64(screenshot) {
       let dataURL = screenshot.canvas.toDataURL();
       return dataURL.split(",")[1];
     }
 
     let result = {
       status: STATUS.FAIL,
@@ -322,23 +337,24 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
         let isFail = this.screenshotMode === SCREENSHOT_MODE.fail;
         let isUnexpected = this.screenshotMode === SCREENSHOT_MODE.unexpected;
         if (isFail || (isUnexpected && expected != result.status)) {
           recordScreenshot();
         }
       }
 
       // Return any reusable canvases to the pool
-      let canvasPool = this.canvasCache.get(null);
+      let cacheKey = width + "x" + height;
+      let canvasPool = this.canvasCache.get(cacheKey).get(null);
       [comparison.lhs, comparison.rhs].map(screenshot => {
         if (screenshot !== null && screenshot.reuseCanvas) {
           canvasPool.push(screenshot.canvas);
         }
       });
-      logger.debug(`Canvas pool is of length ${canvasPool.length}`);
+      logger.debug(`Canvas pool (${cacheKey}) is of length ${canvasPool.length}`);
 
     }
 
     if (screenshotData.length) {
       // For now the tbpl formatter only accepts one screenshot, so just
       // return the last one we took.
       let lastScreenshot = screenshotData[screenshotData.length - 1];
       // eslint-disable-next-line camelcase
@@ -412,40 +428,41 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
     const focusManager = Services.focus;
     if (focusManager.activeWindow != win) {
       focusManager.activeWindow = win;
     }
     this.driver.curBrowser.contentBrowser.focus();
   }
 
   async screenshot(win, url, timeout) {
-    win.innerWidth = REFTEST_WIDTH;
-    win.innerHeight = REFTEST_HEIGHT;
-
     // On windows the above doesn't *actually* set the window to be the
     // reftest size; but *does* set the content area to be the right size;
     // the window is given some extra borders that aren't explicable from CSS
     let browserRect = win.gBrowser.getBoundingClientRect();
     let canvas = null;
     let remainingCount = this.urlCount.get(url) || 1;
     let cache = remainingCount > 1;
+    let cacheKey = browserRect.width + "x" + browserRect.height;
     logger.debug(`screenshot ${url} remainingCount: ` +
-        `${remainingCount} cache: ${cache}`);
+        `${remainingCount} cache: ${cache} cacheKey: ${cacheKey}`);
     let reuseCanvas = false;
-    if (this.canvasCache.has(url)) {
+    let sizedCache = this.canvasCache.get(cacheKey);
+    if (sizedCache.has(url)) {
       logger.debug(`screenshot ${url} taken from cache`);
-      canvas = this.canvasCache.get(url);
+      canvas = sizedCache.get(url);
       if (!cache) {
-        this.canvasCache.delete(url);
+        sizedCache.delete(url);
       }
     } else {
-      let canvases = this.canvasCache.get(null);
-      if (canvases.length) {
-        canvas = canvases.pop();
+      let canvasPool = sizedCache.get(null);
+      if (canvasPool.length) {
+        logger.debug("reusing canvas from canvas pool");
+        canvas = canvasPool.pop();
       } else {
+        logger.debug("using new canvas");
         canvas = null;
       }
       reuseCanvas = !cache;
 
       let ctxInterface = win.CanvasRenderingContext2D;
       let flags = ctxInterface.DRAWWINDOW_DRAW_CARET |
           ctxInterface.DRAWWINDOW_DRAW_VIEW;
 
@@ -481,20 +498,38 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
       canvas = capture.canvas(
           win,
           0, // left
           0, // top
           browserRect.width,
           browserRect.height,
           {canvas, flags});
     }
-    if (canvas.width !== REFTEST_WIDTH || canvas.height !== REFTEST_HEIGHT) {
+    if (canvas.width !== browserRect.width ||
+        canvas.height !== browserRect.height) {
       logger.warn(`Canvas dimensions changed to ${canvas.width}x${canvas.height}`);
       reuseCanvas = false;
       cache = false;
     }
     if (cache) {
-      this.canvasCache.set(url, canvas);
+      sizedCache.set(url, canvas);
     }
     this.urlCount.set(url, remainingCount - 1);
     return {canvas, reuseCanvas};
   }
 };
+
+class DefaultMap extends Map {
+  constructor(iterable, defaultFactory) {
+    super(iterable);
+    this.defaultFactory = defaultFactory;
+  }
+
+  get(key) {
+    if (this.has(key)) {
+      return super.get(key);
+    }
+
+    let v = this.defaultFactory();
+    this.set(key, v);
+    return v;
+  }
+}
--- a/testing/web-platform/meta/css/CSS2/generated-content/content-175.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/generated-content/content-175.xht.ini
@@ -1,4 +1,2 @@
 [content-175.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-collapse-001.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-collapse-001.xht.ini
@@ -1,4 +1,2 @@
 [margin-collapse-001.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-103.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-103.xht.ini
@@ -1,4 +1,2 @@
 [margin-right-103.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-104.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-104.xht.ini
@@ -1,4 +1,2 @@
 [margin-right-104.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-113.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/margin-padding-clear/margin-right-113.xht.ini
@@ -1,4 +1,2 @@
 [margin-right-113.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/normal-flow/blocks-020.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/normal-flow/blocks-020.xht.ini
@@ -1,4 +1,2 @@
 [blocks-020.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/normal-flow/blocks-021.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/normal-flow/blocks-021.xht.ini
@@ -1,4 +1,2 @@
 [blocks-021.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/normal-flow/blocks-022.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/normal-flow/blocks-022.xht.ini
@@ -1,4 +1,2 @@
 [blocks-022.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/normal-flow/inline-replaced-width-014.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/normal-flow/inline-replaced-width-014.xht.ini
@@ -1,4 +1,2 @@
 [inline-replaced-width-014.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/positioning/absolute-replaced-width-003c.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/positioning/absolute-replaced-width-003c.xht.ini
@@ -1,4 +1,2 @@
 [absolute-replaced-width-003c.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/positioning/left-offset-percentage-002.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/positioning/left-offset-percentage-002.xht.ini
@@ -1,4 +1,2 @@
 [left-offset-percentage-002.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/positioning/top-offset-percentage-002.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/positioning/top-offset-percentage-002.xht.ini
@@ -1,4 +1,2 @@
 [top-offset-percentage-002.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/tables/separated-border-model-007.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/tables/separated-border-model-007.xht.ini
@@ -1,4 +1,2 @@
 [separated-border-model-007.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/tables/separated-border-model-008.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/tables/separated-border-model-008.xht.ini
@@ -1,4 +1,2 @@
 [separated-border-model-008.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/tables/separated-border-model-009.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/tables/separated-border-model-009.xht.ini
@@ -1,4 +1,2 @@
 [separated-border-model-009.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/CSS2/visuren/top-114.xht.ini
+++ b/testing/web-platform/meta/css/CSS2/visuren/top-114.xht.ini
@@ -1,6 +1,7 @@
 [top-114.xht]
   expected:
     if os == "win": FAIL
     if os == "linux": FAIL
     if not debug and not webrender and e10s and (os == "mac") and (version == "OS X 10.10.5") and (processor == "x86_64") and (bits == 64): FAIL
     if debug and not webrender and e10s and (os == "mac") and (version == "OS X 10.10.5") and (processor == "x86_64") and (bits == 64): FAIL
+    if not debug and not webrender and not e10s and (os == "android") and (version == "Ubuntu 16.04") and (processor == "x86") and (bits == 32): FAIL
--- a/testing/web-platform/meta/css/css-backgrounds/border-image-outset-003.html.ini
+++ b/testing/web-platform/meta/css/css-backgrounds/border-image-outset-003.html.ini
@@ -1,4 +1,2 @@
 [border-image-outset-003.html]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-flexbox/flexbox_visibility-collapse-line-wrapping.html.ini
+++ b/testing/web-platform/meta/css/css-flexbox/flexbox_visibility-collapse-line-wrapping.html.ini
@@ -1,4 +1,2 @@
 [flexbox_visibility-collapse-line-wrapping.html]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-images/tiled-radial-gradients.html.ini
+++ b/testing/web-platform/meta/css/css-images/tiled-radial-gradients.html.ini
@@ -1,4 +1,2 @@
 [tiled-radial-gradients.html]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-multicol/multicol-fill-auto-001.xht.ini
+++ b/testing/web-platform/meta/css/css-multicol/multicol-fill-auto-001.xht.ini
@@ -1,4 +1,2 @@
 [multicol-fill-auto-001.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-multicol/multicol-inherit-001.xht.ini
+++ b/testing/web-platform/meta/css/css-multicol/multicol-inherit-001.xht.ini
@@ -1,4 +1,2 @@
 [multicol-inherit-001.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-multicol/multicol-overflow-000.xht.ini
+++ b/testing/web-platform/meta/css/css-multicol/multicol-overflow-000.xht.ini
@@ -1,4 +1,2 @@
 [multicol-overflow-000.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-transforms/matrix/svg-matrix-005.html.ini
+++ b/testing/web-platform/meta/css/css-transforms/matrix/svg-matrix-005.html.ini
@@ -1,4 +1,2 @@
 [svg-matrix-005.html]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-007.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-007.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-007.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-013.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-013.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-013.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-019.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-019.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-019.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-025.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-025.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-025.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-031.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-031.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-031.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-037.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-037.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-037.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-043.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-043.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-043.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-049.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-049.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-049.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-055.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-055.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-055.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-061.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-061.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-061.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-067.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-067.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-067.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-073.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-073.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-073.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-079.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-079.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-079.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-085.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-085.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-085.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-091.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-091.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-091.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-097.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-097.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-097.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-107.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-107.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-107.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-115.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-115.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-115.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-123.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-123.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-123.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-131.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-131.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-131.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-139.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-139.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-139.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-147.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-147.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-147.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-155.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-155.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-155.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-163.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-163.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-163.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-171.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-171.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-171.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-179.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-179.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-179.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-187.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-187.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-187.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-195.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-195.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-195.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-203.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-203.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-203.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-211.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-211.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-211.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-219.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-219.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-219.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-227.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vlr-227.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vlr-227.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-006.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-006.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-006.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-012.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-012.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-012.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-018.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-018.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-018.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-024.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-024.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-024.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-030.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-030.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-030.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-036.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-036.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-036.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-042.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-042.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-042.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-048.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-048.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-048.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-054.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-054.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-054.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-060.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-060.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-060.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-066.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-066.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-066.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-072.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-072.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-072.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-078.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-078.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-078.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-084.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-084.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-084.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-090.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-090.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-090.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-096.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-096.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-096.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-106.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-106.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-106.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-114.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-114.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-114.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-122.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-122.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-122.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-130.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-130.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-130.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-138.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-138.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-138.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-146.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-146.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-146.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-154.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-154.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-154.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-162.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-162.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-162.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-170.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-170.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-170.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-178.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-178.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-178.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-186.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-186.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-186.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-194.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-194.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-194.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-202.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-202.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-202.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-210.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-210.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-210.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-218.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-218.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-218.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-226.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/abs-pos-non-replaced-vrl-226.xht.ini
@@ -1,4 +1,2 @@
 [abs-pos-non-replaced-vrl-226.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vlr-007.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vlr-007.xht.ini
@@ -1,4 +1,2 @@
 [sizing-orthog-htb-in-vlr-007.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vlr-008.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vlr-008.xht.ini
@@ -1,4 +1,2 @@
 [sizing-orthog-htb-in-vlr-008.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vrl-008.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-htb-in-vrl-008.xht.ini
@@ -1,4 +1,2 @@
 [sizing-orthog-htb-in-vrl-008.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-prct-htb-in-vrl-001.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/sizing-orthog-prct-htb-in-vrl-001.xht.ini
@@ -1,4 +1,2 @@
 [sizing-orthog-prct-htb-in-vrl-001.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/text-indent-vlr-011.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/text-indent-vlr-011.xht.ini
@@ -1,4 +1,2 @@
 [text-indent-vlr-011.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/text-indent-vlr-013.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/text-indent-vlr-013.xht.ini
@@ -1,4 +1,2 @@
 [text-indent-vlr-013.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/text-indent-vrl-010.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/text-indent-vrl-010.xht.ini
@@ -1,4 +1,2 @@
 [text-indent-vrl-010.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/meta/css/css-writing-modes/text-indent-vrl-012.xht.ini
+++ b/testing/web-platform/meta/css/css-writing-modes/text-indent-vrl-012.xht.ini
@@ -1,4 +1,2 @@
 [text-indent-vrl-012.xht]
-  expected:
-    if os == "android": PASS
-    FAIL
+  expected: FAIL
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
@@ -144,17 +144,17 @@ class FennecBrowser(FirefoxBrowser):
             self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
                                           # Disable high DPI
                                           "layout.css.devPixelsPerPx": "1.0",
                                           # Ensure that the full browser element
                                           # appears in the screenshot
                                           "apz.allow_zooming": False,
                                           "android.widget_paints_background": False,
                                           # Ensure that scrollbars are always painted
-                                          "ui.scrollbarFadeBeginDelay": 100000})
+                                          "layout.testing.overlay-scrollbars.always-visible": True})
 
         if self.install_fonts:
             self.logger.debug("Copying Ahem font to profile")
             font_dir = os.path.join(self.profile.profile, "fonts")
             if not os.path.exists(font_dir):
                 os.makedirs(font_dir)
             with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
                 with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest: