Bug 1644395 - Marionette support for wpt print reftests, r=marionette-reviewers,hiro,maja_zf
authorJames Graham <james@hoppipolla.co.uk>
Tue, 23 Jun 2020 10:00:26 +0000
changeset 536767 ceff82dfdd5e5dc91d2fced74c92659443071b89
parent 536766 f35c2ac7810a4bad6e6fc80866ce49c750bc0138
child 536768 b1b59caf5d3f136ded1333b01511827ce03b72d7
push id37533
push userdluca@mozilla.com
push dateTue, 23 Jun 2020 21:38:40 +0000
treeherdermozilla-central@d48aa0f0aa0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarionette-reviewers, hiro, maja_zf
bugs1644395
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1644395 - Marionette support for wpt print reftests, r=marionette-reviewers,hiro,maja_zf Render print reftests to a PDF using the printing machinary, and use pdf.js from gecko itself to convert the PDF to an image for pixel comparisons. For the surrounding reftest machinery, the main change here is that we convert reftests to work in terms of lists of images rather than a single image. For normal reftests we only have a single image in the list; for print reftests we can have several (one per page). This is implemented in terms of iterators to avoid unnecessary renders when the test fails on an earlier page. Differential Revision: https://phabricator.services.mozilla.com/D79081
testing/marionette/driver.js
testing/marionette/reftest.js
testing/web-platform/tests/tools/manifest/sourcefile.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
--- a/testing/marionette/driver.js
+++ b/testing/marionette/driver.js
@@ -3661,30 +3661,42 @@ GeckoDriver.prototype.setupReftest = asy
   }
 
   if (this.context !== Context.Chrome) {
     throw new UnsupportedOperationError(
       "Must set chrome context before running reftests"
     );
   }
 
-  let { urlCount = {}, screenshot = "unexpected" } = cmd.parameters;
+  let {
+    urlCount = {},
+    screenshot = "unexpected",
+    isPrint = false,
+  } = cmd.parameters;
   if (!["always", "fail", "unexpected"].includes(screenshot)) {
     throw new InvalidArgumentError(
       "Value of `screenshot` should be 'always', 'fail' or 'unexpected'"
     );
   }
 
   this._reftest = new reftest.Runner(this);
-  this._reftest.setup(urlCount, screenshot);
+  this._reftest.setup(urlCount, screenshot, isPrint);
 };
 
 /** Run a reftest. */
 GeckoDriver.prototype.runReftest = async function(cmd) {
-  let { test, references, expected, timeout, width, height } = cmd.parameters;
+  let {
+    test,
+    references,
+    expected,
+    timeout,
+    width,
+    height,
+    pageRanges,
+  } = cmd.parameters;
 
   if (!this._reftest) {
     throw new UnsupportedOperationError(
       "Called reftest:run before reftest:start"
     );
   }
 
   assert.string(test);
@@ -3692,16 +3704,17 @@ GeckoDriver.prototype.runReftest = async
   assert.array(references);
 
   return {
     value: await this._reftest.run(
       test,
       references,
       expected,
       timeout,
+      pageRanges,
       width,
       height
     ),
   };
 };
 
 /**
  * End a reftest run.
--- a/testing/marionette/reftest.js
+++ b/testing/marionette/reftest.js
@@ -1,41 +1,44 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
+const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm");
 const { Preferences } = ChromeUtils.import(
   "resource://gre/modules/Preferences.jsm"
 );
 const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
 const { XPCOMUtils } = ChromeUtils.import(
   "resource://gre/modules/XPCOMUtils.jsm"
 );
 
 const { assert } = ChromeUtils.import("chrome://marionette/content/assert.js");
 const { capture } = ChromeUtils.import(
   "chrome://marionette/content/capture.js"
 );
 const { InvalidArgumentError } = ChromeUtils.import(
   "chrome://marionette/content/error.js"
 );
 const { Log } = ChromeUtils.import("chrome://marionette/content/log.js");
+const { print } = ChromeUtils.import("chrome://marionette/content/print.js");
 
 XPCOMUtils.defineLazyGetter(this, "logger", Log.get);
 
 ChromeUtils.defineModuleGetter(
   this,
   "E10SUtils",
   "resource://gre/modules/E10SUtils.jsm"
 );
 
 this.EXPORTED_SYMBOLS = ["reftest"];
 
+const XHTML_NS = "http://www.w3.org/1999/xhtml";
 const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
 const PREF_E10S = "browser.tabs.remote.autostart";
 const PREF_FISSION = "fission.autostart";
 
 const SCREENSHOT_MODE = {
   unexpected: 0,
   fail: 1,
   always: 2,
@@ -46,32 +49,42 @@ const STATUS = {
   FAIL: "FAIL",
   ERROR: "ERROR",
   TIMEOUT: "TIMEOUT",
 };
 
 const DEFAULT_REFTEST_WIDTH = 600;
 const DEFAULT_REFTEST_HEIGHT = 600;
 
+// reftest-print page dimensions in cm
+const CM_PER_INCH = 2.54;
+const DEFAULT_PAGE_WIDTH = 5 * CM_PER_INCH;
+const DEFAULT_PAGE_HEIGHT = 3 * CM_PER_INCH;
+const DEFAULT_PAGE_MARGIN = 0.5 * CM_PER_INCH;
+
+// CSS 96 pixels per inch, compared to pdf.js default 72 pixels per inch
+const DEFAULT_PDF_RESOLUTION = 96 / 72;
+
 /**
  * Implements an fast runner for web-platform-tests format reftests
  * c.f. http://web-platform-tests.org/writing-tests/reftests.html.
  *
  * @namespace
  */
 this.reftest = {};
 
 /**
  * @memberof reftest
  * @class Runner
  */
 reftest.Runner = class {
   constructor(driver) {
     this.driver = driver;
     this.canvasCache = new DefaultMap(undefined, () => new Map([[null, []]]));
+    this.isPrint = null;
     this.windowUtils = null;
     this.lastURL = null;
     this.useRemoteTabs = Preferences.get(PREF_E10S);
     this.useRemoteSubframes = Preferences.get(PREF_FISSION);
   }
 
   /**
    * Setup the required environment for running reftests.
@@ -81,26 +94,32 @@ reftest.Runner = class {
    *
    * @param {Object.<Number>} urlCount
    *     Object holding a map of URL: number of times the URL
    *     will be opened during the reftest run, where that's
    *     greater than 1.
    * @param {string} screenshotMode
    *     String enum representing when screenshots should be taken
    */
-  setup(urlCount, screenshotMode) {
+  setup(urlCount, screenshotMode, isPrint = false) {
+    this.isPrint = isPrint;
+
     this.parentWindow = assert.open(this.driver.getCurrentWindow());
 
     this.screenshotMode =
       SCREENSHOT_MODE[screenshotMode] || SCREENSHOT_MODE.unexpected;
 
     this.urlCount = Object.keys(urlCount || {}).reduce(
       (map, key) => map.set(key, urlCount[key]),
       new Map()
     );
+
+    if (isPrint) {
+      this.loadPdfJs();
+    }
   }
 
   async ensureWindow(timeout, width, height) {
     logger.debug(`ensuring we have a window ${width}x${height}`);
 
     if (this.reftestWin && !this.reftestWin.closed) {
       let browserRect = this.reftestWin.gBrowser.getBoundingClientRect();
       if (browserRect.width === width && browserRect.height === height) {
@@ -244,16 +263,17 @@ max-width: ${width}px; max-height: ${hei
    * @return {Object}
    *     Result object with fields status, message and extra.
    */
   async run(
     testUrl,
     references,
     expected,
     timeout,
+    pageRanges = {},
     width = DEFAULT_REFTEST_WIDTH,
     height = DEFAULT_REFTEST_HEIGHT
   ) {
     let timeoutHandle;
 
     let timeoutPromise = new Promise(resolve => {
       timeoutHandle = this.parentWindow.setTimeout(() => {
         resolve({ status: STATUS.TIMEOUT, message: null, extra: {} });
@@ -263,16 +283,17 @@ max-width: ${width}px; max-height: ${hei
     let testRunner = (async () => {
       let result;
       try {
         result = await this.runTest(
           testUrl,
           references,
           expected,
           timeout,
+          pageRanges,
           width,
           height
         );
       } catch (e) {
         result = {
           status: STATUS.ERROR,
           message: String(e),
           stack: e.stack,
@@ -286,17 +307,25 @@ max-width: ${width}px; max-height: ${hei
     this.parentWindow.clearTimeout(timeoutHandle);
     if (result.status === STATUS.TIMEOUT) {
       await this.abort();
     }
 
     return result;
   }
 
-  async runTest(testUrl, references, expected, timeout, width, height) {
+  async runTest(
+    testUrl,
+    references,
+    expected,
+    timeout,
+    pageRanges,
+    width,
+    height
+  ) {
     let win = await this.ensureWindow(timeout, width, height);
 
     function toBase64(screenshot) {
       let dataURL = screenshot.canvas.toDataURL();
       return dataURL.split(",")[1];
     }
 
     let result = {
@@ -323,16 +352,17 @@ max-width: ${width}px; max-height: ${hei
       let comparison;
       try {
         comparison = await this.compareUrls(
           win,
           lhsUrl,
           rhsUrl,
           relation,
           timeout,
+          pageRanges,
           extras
         );
       } catch (e) {
         comparison = {
           lhs: null,
           rhs: null,
           passed: false,
           error: e,
@@ -408,65 +438,125 @@ max-width: ${width}px; max-height: ${hei
       let lastScreenshot = screenshotData[screenshotData.length - 1];
       // eslint-disable-next-line camelcase
       result.extra.reftest_screenshots = lastScreenshot;
     }
 
     return result;
   }
 
-  async compareUrls(win, lhsUrl, rhsUrl, relation, timeout, extras) {
+  async compareUrls(
+    win,
+    lhsUrl,
+    rhsUrl,
+    relation,
+    timeout,
+    pageRanges,
+    extras
+  ) {
     logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`);
 
-    // Take the reference screenshot first so that if we pause
-    // we see the test rendering
-    let rhs = await this.screenshot(win, rhsUrl, timeout);
-    let lhs = await this.screenshot(win, lhsUrl, timeout);
+    if (relation !== "==" && relation != "!=") {
+      throw new InvalidArgumentError("Reftest operator should be '==' or '!='");
+    }
 
-    logger.debug(`lhs canvas size ${lhs.canvas.width}x${lhs.canvas.height}`);
-    logger.debug(`rhs canvas size ${rhs.canvas.width}x${rhs.canvas.height}`);
+    let lhsIter, lhsCount, rhsIter, rhsCount;
+    if (!this.isPrint) {
+      // Take the reference screenshot first so that if we pause
+      // we see the test rendering
+      rhsIter = [await this.screenshot(win, rhsUrl, timeout)].values();
+      lhsIter = [await this.screenshot(win, lhsUrl, timeout)].values();
+      lhsCount = rhsCount = 1;
+    } else {
+      [rhsIter, rhsCount] = await this.screenshotPaginated(
+        win,
+        rhsUrl,
+        timeout,
+        pageRanges
+      );
+      [lhsIter, lhsCount] = await this.screenshotPaginated(
+        win,
+        lhsUrl,
+        timeout,
+        pageRanges
+      );
+    }
 
-    let passed;
+    let passed = null;
     let error = null;
     let pixelsDifferent = null;
     let maxDifferences = {};
     let msg = null;
 
-    try {
-      pixelsDifferent = this.windowUtils.compareCanvases(
-        lhs.canvas,
-        rhs.canvas,
-        maxDifferences
-      );
-    } catch (e) {
+    if (lhsCount != rhsCount) {
       passed = false;
-      error = e;
+      msg = `Got different numbers of pages; test has ${lhsCount}, ref has ${rhsCount}`;
     }
 
-    if (error === null) {
-      passed = this.isAcceptableDifference(
-        maxDifferences.value,
-        pixelsDifferent,
-        extras.fuzzy
-      );
-      switch (relation) {
-        case "==":
-          if (!passed) {
+    let lhs = null;
+    let rhs = null;
+    logger.debug(`Comparing ${lhsCount} pages`);
+    if (passed === null) {
+      for (let i = 0; i < lhsCount; i++) {
+        lhs = (await lhsIter.next()).value;
+        rhs = (await rhsIter.next()).value;
+        logger.debug(
+          `lhs canvas size ${lhs.canvas.width}x${lhs.canvas.height}`
+        );
+        logger.debug(
+          `rhs canvas size ${rhs.canvas.width}x${rhs.canvas.height}`
+        );
+        try {
+          pixelsDifferent = this.windowUtils.compareCanvases(
+            lhs.canvas,
+            rhs.canvas,
+            maxDifferences
+          );
+        } catch (e) {
+          error = e;
+          passed = false;
+          break;
+        }
+
+        let areEqual = this.isAcceptableDifference(
+          maxDifferences.value,
+          pixelsDifferent,
+          extras.fuzzy
+        );
+        logger.debug(
+          `Page ${i + 1} maxDifferences: ${maxDifferences.value} ` +
+            `pixelsDifferent: ${pixelsDifferent}`
+        );
+        logger.debug(
+          `Page ${i + 1} ${areEqual ? "compare equal" : "compare unequal"}`
+        );
+        if (!areEqual) {
+          if (relation == "==") {
+            passed = false;
             msg =
               `Found ${pixelsDifferent} pixels different, ` +
               `maximum difference per channel ${maxDifferences.value}`;
+            if (this.isPrint) {
+              msg += ` on page ${i + 1}`;
+            }
+          } else {
+            passed = true;
           }
           break;
-        case "!=":
-          passed = !passed;
-          break;
-        default:
-          throw new InvalidArgumentError(
-            "Reftest operator should be '==' or '!='"
-          );
+        }
+      }
+    }
+
+    // If passed isn't set we got to the end without finding differences
+    if (passed === null) {
+      if (relation == "==") {
+        passed = true;
+      } else {
+        msg = `mismatch reftest has no differences`;
+        passed = false;
       }
     }
     return { lhs, rhs, passed, error, msg };
   }
 
   isAcceptableDifference(maxDifference, pixelsDifferent, allowed) {
     if (!allowed) {
       logger.info(`No differences allowed`);
@@ -522,16 +612,45 @@ max-width: ${width}px; max-height: ${hei
 
       // XXX: This appears to be working fine as is, should we be reinitializing
       // something here? If so, what? The listener.js framescript is registered
       // on the reftest.xhtml chrome window (which shouldn't be changing?), and
       // driver.js uses the global message manager to listen for messages.
     }
   }
 
+  async loadTestUrl(win, url, timeout) {
+    logger.debug(`Starting load of ${url}`);
+    let navigateOpts = {
+      commandId: this.driver.listener.activeMessageId,
+      pageTimeout: timeout,
+    };
+    if (this.lastURL === url) {
+      logger.debug(`Refreshing page`);
+      await this.driver.listener.refresh(navigateOpts);
+    } else {
+      // HACK: DocumentLoadListener currently doesn't know how to
+      // process-switch loads in a non-tabbed <browser>. We need to manually
+      // set the browser's remote type in order to ensure that the load
+      // happens in the correct process.
+      //
+      // See bug 1636169.
+      this.updateBrowserRemotenessByURL(win.gBrowser, url);
+
+      navigateOpts.url = url;
+      navigateOpts.loadEventExpected = false;
+      await this.driver.listener.get(navigateOpts);
+      this.lastURL = url;
+    }
+
+    this.ensureFocus(win);
+
+    await this.driver.listener.reftestWait(url, this.useRemoteTabs);
+  }
+
   async screenshot(win, url, timeout) {
     // On windows the above doesn't *actually* set the window to be the
     // reftest size; but *does* set the content area to be the right size;
     // the window is given some extra borders that aren't explicable from CSS
     let browserRect = win.gBrowser.getBoundingClientRect();
     let canvas = null;
     let remainingCount = this.urlCount.get(url) || 1;
     let cache = remainingCount > 1;
@@ -579,41 +698,18 @@ browserRect.top: ${browserRect.top}
 win.innerWidth: ${win.innerWidth}
 browserRect.width: ${browserRect.width}
 win.innerHeight: ${win.innerHeight}
 browserRect.height: ${browserRect.height}`);
         throw new Error("Window has incorrect dimensions");
       }
 
       url = new URL(url).href; // normalize the URL
-      logger.debug(`Starting load of ${url}`);
-      let navigateOpts = {
-        commandId: this.driver.listener.activeMessageId,
-        pageTimeout: timeout,
-      };
-      if (this.lastURL === url) {
-        logger.debug(`Refreshing page`);
-        await this.driver.listener.refresh(navigateOpts);
-      } else {
-        // HACK: DocumentLoadListener currently doesn't know how to
-        // process-switch loads in a non-tabbed <browser>. We need to manually
-        // set the browser's remote type in order to ensure that the load
-        // happens in the correct process.
-        //
-        // See bug 1636169.
-        this.updateBrowserRemotenessByURL(win.gBrowser, url);
 
-        navigateOpts.url = url;
-        navigateOpts.loadEventExpected = false;
-        await this.driver.listener.get(navigateOpts);
-        this.lastURL = url;
-      }
-
-      this.ensureFocus(win);
-      await this.driver.listener.reftestWait(url, this.useRemoteTabs);
+      await this.loadTestUrl(win, url, timeout);
 
       canvas = await capture.canvas(
         win,
         win.docShell.browsingContext,
         0, // left
         0, // top
         browserRect.width,
         browserRect.height,
@@ -631,16 +727,139 @@ browserRect.height: ${browserRect.height
       cache = false;
     }
     if (cache) {
       sizedCache.set(url, canvas);
     }
     this.urlCount.set(url, remainingCount - 1);
     return { canvas, reuseCanvas };
   }
+
+  async screenshotPaginated(win, url, timeout, pageRanges) {
+    url = new URL(url).href; // normalize the URL
+    await this.loadTestUrl(win, url, timeout);
+
+    const [width, height] = [DEFAULT_PAGE_WIDTH, DEFAULT_PAGE_HEIGHT];
+    const margin = DEFAULT_PAGE_MARGIN;
+    const settings = print.addDefaultSettings({
+      page: {
+        width,
+        height,
+      },
+      margin: {
+        left: margin,
+        right: margin,
+        top: margin,
+        bottom: margin,
+      },
+      shrinkToFit: false,
+      printBackground: true,
+    });
+
+    const filePath = await print.printToFile(
+      win.gBrowser.frameLoader,
+      win.gBrowser.outerWindowID,
+      settings
+    );
+
+    const fp = await OS.File.open(filePath, { read: true });
+    try {
+      const pdf = await this.loadPdf(url, fp);
+      let pages = this.getPages(pageRanges, url, pdf.numPages);
+      return [this.renderPages(pdf, pages), pages.size];
+    } finally {
+      fp.close();
+      await OS.File.remove(filePath);
+    }
+  }
+
+  async loadPdfJs() {
+    // Ensure pdf.js is loaded in the opener window
+    await new Promise((resolve, reject) => {
+      const doc = this.parentWindow.document;
+      const script = doc.createElement("script");
+      script.src = "resource://pdf.js/build/pdf.js";
+      script.onload = resolve;
+      script.onerror = () => reject(new Error("pdfjs load failed"));
+      doc.documentElement.appendChild(script);
+    });
+    this.parentWindow.pdfjsLib.GlobalWorkerOptions.workerSrc =
+      "resource://pdf.js/build/pdf.worker.js";
+  }
+
+  async loadPdf(url, fp) {
+    const data = await fp.read();
+    return this.parentWindow.pdfjsLib.getDocument({ data }).promise;
+  }
+
+  async *renderPages(pdf, pages) {
+    let canvas = null;
+    for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) {
+      if (!pages.has(pageNumber)) {
+        logger.info(`Skipping page ${pageNumber}/${pdf.numPages}`);
+        continue;
+      }
+      logger.info(`Rendering page ${pageNumber}/${pdf.numPages}`);
+      let page = await pdf.getPage(pageNumber);
+      let viewport = page.getViewport({ scale: DEFAULT_PDF_RESOLUTION });
+      // Prepare canvas using PDF page dimensions
+      if (canvas === null) {
+        canvas = this.parentWindow.document.createElementNS(XHTML_NS, "canvas");
+        canvas.height = viewport.height;
+        canvas.width = viewport.width;
+      }
+
+      // Render PDF page into canvas context
+      let context = canvas.getContext("2d");
+      let renderContext = {
+        canvasContext: context,
+        viewport,
+      };
+      await page.render(renderContext).promise;
+      yield { canvas, reuseCanvas: false };
+    }
+  }
+
+  getPages(pageRanges, url, totalPages) {
+    // Extract test id from URL without parsing
+    let afterHost = url.slice(url.indexOf(":") + 3);
+    afterHost = afterHost.slice(afterHost.indexOf("/"));
+    const ranges = pageRanges[afterHost];
+    let rv = new Set();
+
+    if (!ranges) {
+      for (let i = 1; i <= totalPages; i++) {
+        rv.add(i);
+      }
+      return rv;
+    }
+
+    for (let rangePart of ranges) {
+      if (rangePart.length === 1) {
+        rv.add(rangePart[0]);
+      } else {
+        if (rangePart.length !== 2) {
+          throw new Error(
+            `Page ranges must be <int> or <int> '-' <int>, got ${rangePart}`
+          );
+        }
+        let [lower, upper] = rangePart;
+        if (lower === null) {
+          lower = 1;
+        }
+        if (upper === null) {
+          upper = totalPages;
+        }
+        for (let i = lower; i <= upper; i++) {
+          rv.add(i);
+        }
+      }
+    }
+    return rv;
+  }
 };
 
 class DefaultMap extends Map {
   constructor(iterable, defaultFactory) {
     super(iterable);
     this.defaultFactory = defaultFactory;
   }
 
--- a/testing/web-platform/tests/tools/manifest/sourcefile.py
+++ b/testing/web-platform/tests/tools/manifest/sourcefile.py
@@ -580,16 +580,17 @@ class SourceFile(object):
     @cached_property
     def fuzzy_nodes(self):
         # type: () -> List[ElementTree.Element]
         """List of ElementTree Elements corresponding to nodes in a test that
         specify reftest fuzziness"""
         assert self.root is not None
         return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='fuzzy']")
 
+
     @cached_property
     def fuzzy(self):
         # type: () -> Dict[Optional[Tuple[Text, Text, Text]], List[List[int]]]
         rv = {}  # type: Dict[Optional[Tuple[Text, Text, Text]], List[List[int]]]
         if self.root is None:
             return rv
 
         if not self.fuzzy_nodes:
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -112,17 +112,17 @@ def executor_kwargs(test_type, server_co
     executor_kwargs["close_after_done"] = test_type != "reftest"
     executor_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type,
                                                                    run_info_data,
                                                                    **kwargs)
     executor_kwargs["e10s"] = run_info_data["e10s"]
     capabilities = {}
     if test_type == "testharness":
         capabilities["pageLoadStrategy"] = "eager"
-    if test_type == "reftest":
+    if test_type in ("reftest", "print-reftest"):
         executor_kwargs["reftest_internal"] = kwargs["reftest_internal"]
         executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"]
     if test_type == "wdspec":
         options = {}
         if kwargs["binary"]:
             options["binary"] = kwargs["binary"]
         if kwargs["binary_args"]:
             options["args"] = kwargs["binary_args"]
@@ -587,19 +587,22 @@ class ProfileCreator(object):
             "network.preload": True,
         })
         if self.e10s:
             profile.set_preferences({"browser.tabs.remote.autostart": True})
 
         if self.enable_fission:
             profile.set_preferences({"fission.autostart": True})
 
-        if self.test_type == "reftest":
+        if self.test_type in ("reftest", "print-reftest"):
             profile.set_preferences({"layout.interruptible-reflow.enabled": False})
 
+        if self.test_type == "print-reftest":
+            profile.set_preferences({"print.always_print_silent": True})
+
         # Bug 1262954: winxp + e10s, disable hwaccel
         if (self.e10s and platform.system() in ("Windows", "Microsoft") and
             "5.1" in platform.version()):
             self.profile.set_preferences({"layers.acceleration.disabled": True})
 
     def _setup_ssl(self, profile):
         """Create a certificate database to use in the test profile. This is configured
         to trust the CA Certificate that has signed the web-platform.test server
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
@@ -354,32 +354,34 @@ class TestExecutor(object):
 
 
 class TestharnessExecutor(TestExecutor):
     convert_result = testharness_result_converter
 
 
 class RefTestExecutor(TestExecutor):
     convert_result = reftest_result_converter
+    is_print = False
 
     def __init__(self, logger, browser, server_config, timeout_multiplier=1, screenshot_cache=None,
                  debug_info=None, **kwargs):
         TestExecutor.__init__(self, logger, browser, server_config,
                               timeout_multiplier=timeout_multiplier,
                               debug_info=debug_info)
 
         self.screenshot_cache = screenshot_cache
 
 
 class CrashtestExecutor(TestExecutor):
     convert_result = crashtest_result_converter
 
 
 class PrintRefTestExecutor(TestExecutor):
     convert_result = reftest_result_converter
+    is_print = True
 
 
 class RefTestImplementation(object):
     def __init__(self, executor):
         self.timeout_multiplier = executor.timeout_multiplier
         self.executor = executor
         # Cache of url:(screenshot hash, screenshot). Typically the
         # screenshot is None, but we set this value if a test fails
@@ -393,17 +395,17 @@ class RefTestImplementation(object):
 
     def teardown(self):
         pass
 
     @property
     def logger(self):
         return self.executor.logger
 
-    def get_hash(self, test, viewport_size, dpi):
+    def get_hash(self, test, viewport_size, dpi, page_ranges):
         key = (test.url, viewport_size, dpi)
 
         if key not in self.screenshot_cache:
             success, data = self.get_screenshot_list(test, viewport_size, dpi, page_ranges)
 
             if not success:
                 return False, data
 
@@ -493,33 +495,35 @@ class RefTestImplementation(object):
         extrema = image.getextrema()
         if all(min == max for min, max in extrema):
             color = ''.join('%02X' % value for value, _ in extrema)
             self.message.append("Screenshot is solid color 0x%s for %s\n" % (color, url))
 
     def run_test(self, test):
         viewport_size = test.viewport_size
         dpi = test.dpi
+        page_ranges = test.page_ranges
         self.message = []
 
+
         # Depth-first search of reference tree, with the goal
         # of reachings a leaf node with only pass results
 
         stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
         page_idx = None
         while stack:
             hashes = [None, None]
             screenshots = [None, None]
             urls = [None, None]
 
             nodes, relation = stack.pop()
             fuzzy = self.get_fuzzy(test, nodes, relation)
 
             for i, node in enumerate(nodes):
-                success, data = self.get_hash(node, viewport_size, dpi)
+                success, data = self.get_hash(node, viewport_size, dpi, page_ranges)
                 if success is False:
                     return {"status": data[0], "message": data[1]}
 
                 hashes[i], screenshots[i] = data
                 urls[i] = node.url
 
             is_pass, page_idx = self.check_pass(hashes, screenshots, urls, relation, fuzzy)
             if is_pass:
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
@@ -831,16 +831,18 @@ class MarionetteTestharnessExecutor(Test
 
         if self.protocol.coverage.is_enabled:
             self.protocol.coverage.dump()
 
         return rv
 
 
 class MarionetteRefTestExecutor(RefTestExecutor):
+    is_print = False
+
     def __init__(self, logger, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True,
                  debug_info=None, reftest_internal=False,
                  reftest_screenshot="unexpected", ccov=False,
                  group_metadata=None, capabilities=None, debug=False, **kwargs):
         """Marionette-based executor for reftests"""
         RefTestExecutor.__init__(self,
                                  logger,
@@ -961,17 +963,17 @@ class InternalRefTestImplementation(RefT
         self.timeout_multiplier = executor.timeout_multiplier
         self.executor = executor
 
     @property
     def logger(self):
         return self.executor.logger
 
     def setup(self, screenshot="unexpected"):
-        data = {"screenshot": screenshot}
+        data = {"screenshot": screenshot, "isPrint": self.executor.is_print}
         if self.executor.group_metadata is not None:
             data["urlCount"] = {urljoin(self.executor.server_url(key[0]), key[1]):value
                                 for key, value in iteritems(
                                     self.executor.group_metadata.get("url_count", {}))
                                 if value > 1}
         self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
         self.executor.protocol.marionette._send_message("reftest:setup", data)
 
@@ -1089,40 +1091,48 @@ class MarionetteCrashtestExecutor(Crasht
         if self.protocol.coverage.is_enabled:
             self.protocol.coverage.dump()
 
         return {"status": "PASS",
                 "message": None}
 
 
 class MarionettePrintRefTestExecutor(MarionetteRefTestExecutor):
+    is_print = True
+
     def __init__(self, logger, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True,
                  debug_info=None, reftest_screenshot="unexpected", ccov=False,
-                 group_metadata=None, capabilities=None, debug=False, **kwargs):
+                 group_metadata=None, capabilities=None, debug=False,
+                 reftest_internal=False, **kwargs):
         """Marionette-based executor for reftests"""
         MarionetteRefTestExecutor.__init__(self,
                                            logger,
                                            browser,
                                            server_config,
                                            timeout_multiplier=timeout_multiplier,
                                            screenshot_cache=screenshot_cache,
                                            close_after_done=close_after_done,
                                            debug_info=debug_info,
                                            reftest_screenshot=reftest_screenshot,
-                                           reftest_internal=False,
+                                           reftest_internal=reftest_internal,
                                            ccov=ccov,
                                            group_metadata=group_metadata,
                                            capabilities=capabilities,
                                            debug=debug,
                                            **kwargs)
 
     def setup(self, runner):
         super(MarionettePrintRefTestExecutor, self).setup(runner)
-        self.protocol.pdf_print.load_runner()
+        if not isinstance(self.implementation, InternalRefTestImplementation):
+            self.protocol.pdf_print.load_runner()
+
+    def get_implementation(self, reftest_internal):
+        return (InternalRefTestImplementation if reftest_internal
+                else RefTestImplementation)(self)
 
     def screenshot(self, test, viewport_size, dpi, page_ranges):
         # https://github.com/web-platform-tests/wpt/issues/7140
         assert dpi is None
 
         self.viewport_size = viewport_size
         timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None