Bug 1509983 - Enable wpt android reftests on try/m-c, r=gbrown,ato
authorJames Graham <james@hoppipolla.co.uk>
Fri, 30 Nov 2018 15:31:50 +0000
changeset 508206 a72c668be4352bd19d650019384ba44de24cc795
parent 508205 7d17aa7e056119b29b713caa1bd63d575ad2cc56
child 508207 44cedf1935796806b9120fbdeeb3999e18788c5d
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown, ato
bugs1509983
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509983 - Enable wpt android reftests on try/m-c, r=gbrown,ato Depends on D13089 Differential Revision: https://phabricator.services.mozilla.com/D13090
taskcluster/ci/test/test-sets.yml
taskcluster/ci/test/web-platform.yml
testing/marionette/reftest.js
testing/mozharness/scripts/web_platform_tests.py
testing/web-platform/mach_commands.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -345,17 +345,17 @@ android-x86-kvm-tests:
     # - mochitest-chrome
     - mochitest-clipboard
     - mochitest-gpu
     # - mochitest-media
     # - mochitest-webgl1-core
     # - reftest
     # - test-verify
     - web-platform-tests
-    # - web-platform-tests-reftests
+    - web-platform-tests-reftests
     # - web-platform-tests-wdspec
 
 android-ccov-tests:
     - geckoview-junit
     - robocop
 
 devtools-tests:
     - mochitest-devtools-chrome
--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@@ -39,17 +39,17 @@ web-platform-tests:
             windows10-64-ccov/debug: 10800
             default: 7200
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
-            android-em-7.0-x86/opt: ['mozilla-central']
+            android.*: ['mozilla-central', 'try']
             linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
     tier:
         by-test-platform:
             android.*: 2
             linux64-asan/opt: 2
             linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
             default: default
@@ -84,24 +84,27 @@ web-platform-tests-headless:
                 default:
                     - --test-type=testharness
                     - --headless
 
 web-platform-tests-reftests:
     description: "Web platform reftest run"
     suite: web-platform-tests-reftests
     treeherder-symbol: W(Wr)
-    chunks: 6
+    chunks:
+        by-test-platform:
+            android.*: 12
+            default: 6
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
-            android-em-7.0-x86/opt: ['mozilla-central']
+            android.*: ['mozilla-central', 'try']
             linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
     tier:
         by-test-platform:
             android.*: 3
             linux64-asan/opt: 2
             linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
             default: default
@@ -139,17 +142,17 @@ web-platform-tests-wdspec:
             linux64-ccov/debug: 4
             windows10-64-ccov/debug: 4
             default: 2
     mozharness:
         extra-options:
             - --test-type=wdspec
     run-on-projects:
         by-test-platform:
-            android-em-7.0-x86/opt: ['mozilla-central']
+            android.*: ['mozilla-central', 'try']
             linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
     tier:
         by-test-platform:
             android.*: 3
             linux64-asan/opt: 2
             linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
             default: default
--- a/testing/marionette/reftest.js
+++ b/testing/marionette/reftest.js
@@ -1,15 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
 ChromeUtils.import("resource://gre/modules/Preferences.jsm");
+ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 
 ChromeUtils.import("chrome://marionette/content/assert.js");
 ChromeUtils.import("chrome://marionette/content/capture.js");
 const {InvalidArgumentError} = ChromeUtils.import("chrome://marionette/content/error.js", {});
 const {Log} = ChromeUtils.import("chrome://marionette/content/log.js", {});
 
 XPCOMUtils.defineLazyGetter(this, "logger", Log.get);
@@ -76,66 +77,95 @@ reftest.Runner = class {
         SCREENSHOT_MODE.unexpected;
 
     this.urlCount = Object.keys(urlCount || {})
         .reduce((map, key) => map.set(key, urlCount[key]), new Map());
 
     await this.ensureWindow();
   }
 
-  async ensureWindow() {
+  async ensureWindow(timeout) {
     if (this.reftestWin && !this.reftestWin.closed) {
       return this.reftestWin;
     }
 
-    let reftestWin = await this.openWindow();
+    let reftestWin;
+    if (Services.appinfo.OS == "Android") {
+      logger.debug("Using current window");
+      reftestWin = this.parentWindow;
+      await this.driver.listener.get({
+        commandID: this.driver.listener.activeMessageId,
+        pageTimeout: timeout,
+        url: "about:blank",
+        loadEventExpected: false});
+
+    } else {
+      logger.debug("Using separate window");
+      reftestWin = await this.openWindow();
+    }
+
+    this.setupWindow(reftestWin);
+    this.windowUtils = reftestWin.windowUtils;
+    this.reftestWin = reftestWin;
 
     let found = this.driver.findWindow([reftestWin], () => true);
     await this.driver.setWindowHandle(found, true);
 
-    this.windowUtils = reftestWin.windowUtils;
-    this.reftestWin = reftestWin;
     return reftestWin;
   }
 
   async openWindow() {
     let reftestWin = this.parentWindow.open(
         "chrome://marionette/content/reftest.xul",
         "reftest",
         `chrome,height=${REFTEST_WIDTH},width=${REFTEST_HEIGHT}`);
 
     await new Promise(resolve => {
       reftestWin.addEventListener("load", resolve, {once: true});
     });
+    return reftestWin;
+  }
 
-    let browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
-    browser.permanentKey = {};
-    browser.setAttribute("id", "browser");
-    browser.setAttribute("anonid", "initialBrowser");
-    browser.setAttribute("type", "content");
-    browser.setAttribute("primary", "true");
-
-    if (this.remote) {
-      browser.setAttribute("remote", "true");
-      browser.setAttribute("remoteType", "web");
+  setupWindow(reftestWin) {
+    let browser;
+    if (Services.appinfo.OS === "Android") {
+      browser = reftestWin.document.getElementsByTagName("browser")[0];
+      browser.setAttribute("remote", "false");
+    } else {
+      browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
+      browser.permanentKey = {};
+      browser.setAttribute("id", "browser");
+      browser.setAttribute("anonid", "initialBrowser");
+      browser.setAttribute("type", "content");
+      browser.setAttribute("primary", "true");
+      if (this.remote) {
+        browser.setAttribute("remote", "true");
+        browser.setAttribute("remoteType", "web");
+      } else {
+        browser.setAttribute("remote", "false");
+      }
     }
     // Make sure the browser element is exactly 600x600, no matter
     // what size our window is
     const windowStyle = `padding: 0px; margin: 0px; border:none;
 min-width: ${REFTEST_WIDTH}px; min-height: ${REFTEST_HEIGHT}px;
 max-width: ${REFTEST_WIDTH}px; max-height: ${REFTEST_HEIGHT}px`;
     browser.setAttribute("style", windowStyle);
 
-    let doc = reftestWin.document.documentElement;
-    while (doc.firstChild) {
-      doc.firstChild.remove();
+    if (Services.appinfo.OS !== "Android") {
+      let doc = reftestWin.document.documentElement;
+      while (doc.firstChild) {
+        doc.firstChild.remove();
+      }
+      doc.appendChild(browser);
     }
-    doc.appendChild(browser);
+    if (reftestWin.BrowserApp) {
+      reftestWin.BrowserApp = browser;
+    }
     reftestWin.gBrowser = browser;
-
     return reftestWin;
   }
 
   abort() {
     if (this.reftestWin) {
       this.driver.closeChromeWindow();
     }
     this.reftestWin = null;
@@ -213,17 +243,17 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
     if (result.status === STATUS.TIMEOUT) {
       this.abort();
     }
 
     return result;
   }
 
   async runTest(testUrl, references, expected, timeout) {
-    let win = await this.ensureWindow();
+    let win = await this.ensureWindow(timeout);
 
     function toBase64(screenshot) {
       let dataURL = screenshot.canvas.toDataURL();
       return dataURL.split(",")[1];
     }
 
     let result = {
       status: STATUS.FAIL,
@@ -404,19 +434,28 @@ max-width: ${REFTEST_WIDTH}px; max-heigh
         canvas = canvases.pop();
       } else {
         canvas = null;
       }
       reuseCanvas = !cache;
 
       let ctxInterface = win.CanvasRenderingContext2D;
       let flags = ctxInterface.DRAWWINDOW_DRAW_CARET |
-          ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS |
           ctxInterface.DRAWWINDOW_DRAW_VIEW;
 
+      if (0 <= browserRect.left &&
+          0 <= browserRect.top &&
+          win.innerWidth >= browserRect.width &&
+          win.innerHeight >= browserRect.height) {
+        logger.debug("Using DRAWWINDOW_USE_WIDGET_LAYERS");
+        flags |= ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS;
+      } else {
+        logger.debug("Not using DRAWWINDOW_USE_WIDGET_LAYERS");
+      }
+
       logger.debug(`Starting load of ${url}`);
       let navigateOpts = {
         commandId: this.driver.listener.activeMessageId,
         pageTimeout: timeout,
       };
       if (this.lastURL === url) {
         logger.debug(`Refreshing page`);
         await this.driver.listener.refresh(navigateOpts);
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -223,18 +223,18 @@ class WebPlatformTest(TestingMixin, Merc
                 "--binary=%s" % self.binary_path,
                 "--symbols-path=%s" % self.query_symbols_url(),
                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),
                 "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin"),
                 "--run-by-dir=%i" % (3 if not mozinfo.info["asan"] else 0),
                 "--no-pause-after-test"]
 
         if self.is_android:
-            cmd += ["--device-serial=%s" % self.device_serial]
-            cmd += ["--package-name=%s" % self.query_package_name()]
+            cmd += ["--device-serial=%s" % self.device_serial,
+                    "--package-name=%s" % self.query_package_name()]
 
         if mozinfo.info["os"] == "win" and mozinfo.info["os_version"] == "6.1":
             # On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
             self._install_fonts()
         else:
             cmd += ["--install-fonts"]
 
         for test_type in test_types:
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -45,19 +45,16 @@ class WebPlatformTestsRunnerSetup(Mozbui
 
             grant_runtime_permissions(self, package_name, kwargs["device_serial"])
             if kwargs["certutil_binary"] is None:
                 kwargs["certutil_binary"] = os.path.join(os.environ.get('MOZ_HOST_BIN'), "certutil")
 
             if kwargs["install_fonts"] is None:
                 kwargs["install_fonts"] = True
 
-            if kwargs["reftest_internal"] is None:
-                kwargs["reftest_internal"] = False
-
         if kwargs["config"] is None:
             kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
 
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.join(self.topsrcdir, 'testing', 'profiles')
 
         if kwargs["stackfix_dir"] is None:
             kwargs["stackfix_dir"] = self.bindir
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
@@ -137,16 +137,27 @@ class FennecBrowser(FirefoxBrowser):
         preferences = self.load_prefs()
 
         self.profile = FirefoxProfile(preferences=preferences)
         self.profile.set_preferences({"marionette.port": self.marionette_port,
                                       "dom.disable_open_during_load": False,
                                       "places.history.enabled": False,
                                       "dom.send_after_paint_to_content": True,
                                       "network.preload": True})
+        if self.test_type == "reftest":
+            self.logger.info("Setting android reftest preferences")
+            self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
+                                          # Disable high DPI
+                                          "layout.css.devPixelsPerPx": "1.0",
+                                          # Ensure that the full browser element
+                                          # appears in the screenshot
+                                          "apz.allow_zooming": False,
+                                          "android.widget_paints_background": False,
+                                          # Ensure that scrollbars are always painted
+                                          "ui.scrollbarFadeBeginDelay": 100000})
 
         if self.install_fonts:
             self.logger.debug("Copying Ahem font to profile")
             font_dir = os.path.join(self.profile.profile, "fonts")
             if not os.path.exists(font_dir):
                 os.makedirs(font_dir)
             with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
                 with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
@@ -485,17 +485,17 @@ class MarionetteProtocol(Protocol):
                 if self.executor.debug_info is None:
                     raise
 
         self.logger.debug("Starting Marionette session")
         self.marionette.start_session(self.capabilities)
         self.logger.debug("Marionette session started")
 
     def after_connect(self):
-        self.testharness.load_runner(self.executor.last_environment["protocol"])
+        pass
 
     def teardown(self):
         try:
             self.marionette._request_in_app_shutdown()
             self.marionette.delete_session(send_request=False)
         except Exception:
             # This is typically because the session never started
             pass
@@ -628,16 +628,20 @@ class MarionetteTestharnessExecutor(Test
         self.window_id = str(uuid.uuid4())
         self.debug = debug
 
         self.original_pref_values = {}
 
         if marionette is None:
             do_delayed_imports()
 
+    def setup(self, runner):
+        super(MarionetteTestharnessExecutor, self).setup(runner)
+        self.protocol.testharness.load_runner(self.last_environment["protocol"])
+
     def is_alive(self):
         return self.protocol.is_alive
 
     def on_environment_change(self, new_environment):
         self.protocol.on_environment_change(self.last_environment, new_environment)
 
         if new_environment["protocol"] != self.last_environment["protocol"]:
             self.protocol.testharness.load_runner(new_environment["protocol"])
@@ -745,22 +749,24 @@ class MarionetteRefTestExecutor(RefTestE
 
     def setup(self, runner):
         super(self.__class__, self).setup(runner)
         self.implementation.setup(**self.implementation_kwargs)
 
     def teardown(self):
         try:
             self.implementation.teardown()
-            handle = self.protocol.marionette.window_handles[0]
-            self.protocol.marionette.switch_to_window(handle)
+            handles = self.protocol.marionette.window_handles
+            if handles:
+                self.protocol.marionette.switch_to_window(handles[0])
             super(self.__class__, self).teardown()
         except Exception as e:
             # Ignore errors during teardown
-            self.logger.warning(traceback.format_exc(e))
+            self.logger.warning("Exception during reftest teardown:\n%s" %
+                                traceback.format_exc(e))
 
     def is_alive(self):
         return self.protocol.is_alive
 
     def on_environment_change(self, new_environment):
         self.protocol.on_environment_change(self.last_environment, new_environment)
 
     def do_test(self, test):