Bug 1323620 - Add "fennec" product to wptrunner. r=jgraham
authorMaja Frydrychowicz <mjzffr@gmail.com>
Fri, 15 Jun 2018 16:30:58 +0000
changeset 476885 266c6cfb96a76daaa7bd2e3574f7d1d8ae60c4a6
parent 476884 7fbf8a9126e54addb86d01ff6ec1e62e7fda18e5
child 476886 eefee7582184562a823bc42cb6295e7fb9e3b0d5
push id9374
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:43:20 +0000
treeherdermozilla-beta@160e085dfb0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1323620
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1323620 - Add "fennec" product to wptrunner. r=jgraham This allows running web-platform-tests on Fennec given a running emulator. (Which is how we expect the tests to run in automation as well -- the android_emulator_unittest mozharness script takes care of emulator start-up.) It also hooks up ./mach wpt. wptrunner sets up a profile for Fennec, forwards the marionette port and starts up Fennec, etc. = Usage = Set your mozconfig to build fennec. Start an emulator: `./mach android-emulator --version x86` Install fennec: `./mach build && ./mach package && ./mach install` Run the tests: ``` ./mach wpt --product=fennec --testtype=testharness --certutil-binary path/to/host/os/certutil path/to/some/tests ``` Differential Revision: https://phabricator.services.mozilla.com/D1587
testing/web-platform/mach_commands.py
testing/web-platform/mach_commands_base.py
testing/web-platform/tests/tools/wpt/browser.py
testing/web-platform/tests/tools/wpt/run.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
testing/web-platform/wptrunner.ini
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -18,38 +18,35 @@ from mozbuild.base import (
 from mach.decorators import (
     CommandProvider,
     Command,
 )
 
 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
 
 
+def is_firefox_or_android(cls):
+    """Must have Firefox build or Android build."""
+    return conditions.is_firefox(cls) or conditions.is_android(cls)
+
+
 class WebPlatformTestsRunnerSetup(MozbuildObject):
     default_log_type = "mach"
 
-    def kwargs_firefox(self, kwargs):
-        from wptrunner import wptcommandline
-
+    def kwargs_common(self, kwargs):
         build_path = os.path.join(self.topobjdir, 'build')
         if build_path not in sys.path:
             sys.path.append(build_path)
 
         if kwargs["config"] is None:
             kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
 
-        if kwargs["binary"] is None:
-            kwargs["binary"] = self.get_binary_path()
-
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.join(self.topobjdir, '_tests', 'web-platform', "prefs")
 
-        if kwargs["certutil_binary"] is None:
-            kwargs["certutil_binary"] = self.get_binary_path('certutil')
-
         if kwargs["stackfix_dir"] is None:
             kwargs["stackfix_dir"] = self.bindir
 
         here = os.path.split(__file__)[0]
 
         if kwargs["exclude"] is None and kwargs["include"] is None and not sys.platform.startswith("linux"):
             kwargs["exclude"] = ["css"]
 
@@ -60,16 +57,28 @@ class WebPlatformTestsRunnerSetup(Mozbui
             if kwargs["host_key_path"] is None:
                 kwargs["host_key_path"] = os.path.join(here, "certs", "web-platform.test.key")
 
             if kwargs["host_cert_path"] is None:
                 kwargs["host_cert_path"] = os.path.join(here, "certs", "web-platform.test.pem")
 
         kwargs["capture_stdio"] = True
 
+        return kwargs
+
+    def kwargs_firefox(self, kwargs):
+        from wptrunner import wptcommandline
+        kwargs = self.kwargs_common(kwargs)
+
+        if kwargs["binary"] is None:
+            kwargs["binary"] = self.get_binary_path()
+
+        if kwargs["certutil_binary"] is None:
+            kwargs["certutil_binary"] = self.get_binary_path('certutil')
+
         if kwargs["webdriver_binary"] is None:
             kwargs["webdriver_binary"] = self.get_binary_path("geckodriver", validate_exists=False)
 
         self.setup_fonts_firefox()
 
         kwargs = wptcommandline.check_args(kwargs)
 
         return kwargs
@@ -114,41 +123,39 @@ class WebPlatformTestsRunnerSetup(Mozbui
             font_path = os.path.join(os.path.dirname(self.get_binary_path()), os.pardir, "Resources", "res", "fonts")
         ahem_src = os.path.join(self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf")
         ahem_dest = os.path.join(font_path, "Ahem.ttf")
         if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
             with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
                 dest.write(src.read())
 
 
-
 class WebPlatformTestsUpdater(MozbuildObject):
     """Update web platform tests."""
     def run_update(self, **kwargs):
         import update
         from update import updatecommandline
 
         if kwargs["config"] is None:
             kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
         if kwargs["product"] is None:
             kwargs["product"] = "firefox"
 
-
-
         kwargs = updatecommandline.check_args(kwargs)
         logger = update.setup_logging(kwargs, {"mach": sys.stdout})
 
         try:
             update.run_update(logger, **kwargs)
         except Exception:
             import pdb
             import traceback
             traceback.print_exc()
 #            pdb.post_mortem()
 
+
 class WebPlatformTestsReduce(WebPlatformTestsRunner):
 
     def run_reduce(self, **kwargs):
         from wptrunner import reduce
 
         self.setup_kwargs(kwargs)
 
         kwargs["capture_stdio"] = True
@@ -156,16 +163,17 @@ class WebPlatformTestsReduce(WebPlatform
         tests = reduce.do_reduce(**kwargs)
 
         if not tests:
             logger.warning("Test was not unstable")
 
         for item in tests:
             logger.info(item.id)
 
+
 class WebPlatformTestsCreator(MozbuildObject):
     template_prefix = """<!doctype html>
 %(documentElement)s<meta charset=utf-8>
 """
     template_long_timeout = "<meta name=timeout content=long>\n"
 
     template_body_th = """<title></title>
 <script src=/resources/testharness.js></script>
@@ -216,17 +224,16 @@ testing/web-platform/mozilla/tests for G
             return 1
 
         if ref_path and self.rel_url(ref_path) is None:
             print("""Reference path %s is not in wpt directories:
 testing/web-platform/tests for tests that may be shared
             testing/web-platform/mozilla/tests for Gecko-only tests""" % ref_path)
             return 1
 
-
         if os.path.exists(path) and not kwargs["overwrite"]:
             print("Test path already exists, pass --overwrite to replace")
             return 1
 
         if kwargs["mismatch"] and not kwargs["reftest"]:
             print("--mismatch only makes sense for a reftest")
             return 1
 
@@ -299,20 +306,22 @@ class WPTManifestDownloader(MozbuildObje
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
         manifestdownload.run(logger, wpt_dir, self.topsrcdir, force)
 
 
 def create_parser_update():
     from update import updatecommandline
     return updatecommandline.create_parser()
 
+
 def create_parser_reduce():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser_reduce()
 
+
 def create_parser_create():
     import argparse
     p = argparse.ArgumentParser()
     p.add_argument("--no-editor", action="store_true",
                    help="Don't try to open the test in an editor")
     p.add_argument("-e", "--editor", action="store", help="Editor to use")
     p.add_argument("--long-timeout", action="store_true",
                    help="Test should be given a long timeout (typically 60s rather than 10s, but varies depending on environment)")
@@ -340,33 +349,37 @@ def create_parser_manifest_download():
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     def setup(self):
         self._activate_virtualenv()
 
     @Command("web-platform-tests",
              category="testing",
-             conditions=[conditions.is_firefox],
+             conditions=[is_firefox_or_android],
              parser=create_parser_wpt)
     def run_web_platform_tests(self, **params):
         self.setup()
-
+        if conditions.is_android(self) and params["product"] != "fennec":
+            if params["product"] is None:
+                params["product"] = "fennec"
+            else:
+                raise ValueError("Must specify --product=fennec in Android environment.")
         if "test_objects" in params:
             for item in params["test_objects"]:
                 params["include"].append(item["name"])
             del params["test_objects"]
 
         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
         wpt_runner = WebPlatformTestsRunner(wpt_setup)
         return wpt_runner.run(**params)
 
     @Command("wpt",
              category="testing",
-             conditions=[conditions.is_firefox],
+             conditions=[is_firefox_or_android],
              parser=create_parser_wpt)
     def run_wpt(self, **params):
         return self.run_web_platform_tests(**params)
 
     @Command("web-platform-tests-update",
              category="testing",
              parser=create_parser_update)
     def update_web_platform_tests(self, **params):
--- a/testing/web-platform/mach_commands_base.py
+++ b/testing/web-platform/mach_commands_base.py
@@ -2,28 +2,31 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 
 
 def create_parser_wpt():
     from wptrunner import wptcommandline
-    return wptcommandline.create_parser(["firefox", "chrome", "edge", "servo"])
+    return wptcommandline.create_parser(["fennec", "firefox", "chrome", "edge", "servo"])
 
 
 class WebPlatformTestsRunner(object):
     """Run web platform tests."""
 
     def __init__(self, setup):
         self.setup = setup
 
     def run(self, **kwargs):
         from wptrunner import wptrunner
         if kwargs["product"] in ["firefox", None]:
             kwargs = self.setup.kwargs_firefox(kwargs)
+        elif kwargs["product"] == "fennec":
+            from wptrunner import wptcommandline
+            kwargs = wptcommandline.check_args(self.setup.kwargs_common(kwargs))
         elif kwargs["product"] in ("chrome", "edge", "servo"):
             kwargs = self.setup.kwargs_wptrun(kwargs)
         else:
             raise ValueError("Unknown product %s" % kwargs["product"])
         logger = wptrunner.setup_logging(kwargs, {self.setup.default_log_type: sys.stdout})
         result = wptrunner.start(**kwargs)
         return int(not result)
--- a/testing/web-platform/tests/tools/wpt/browser.py
+++ b/testing/web-platform/tests/tools/wpt/browser.py
@@ -281,16 +281,38 @@ class Firefox(Browser):
         binary = binary or self.find_binary()
         version_string = call(binary, "--version").strip()
         m = re.match(r"Mozilla Firefox (.*)", version_string)
         if not m:
             return None
         return m.group(1)
 
 
+class Fennec(Browser):
+    """Fennec-specific interface."""
+
+    product = "fennec"
+    requirements = "requirements_firefox.txt"
+
+    def install(self, dest=None):
+        raise NotImplementedError
+
+    def find_binary(self, venv_path=None):
+        raise NotImplementedError
+
+    def find_webdriver(self):
+        raise NotImplementedError
+
+    def install_webdriver(self, dest=None):
+        raise NotImplementedError
+
+    def version(self, binary=None):
+        return None
+
+
 class Chrome(Browser):
     """Chrome-specific interface.
 
     Includes webdriver installation, and wptrunner setup methods.
     """
 
     product = "chrome"
     requirements = "requirements_chrome.txt"
--- a/testing/web-platform/tests/tools/wpt/run.py
+++ b/testing/web-platform/tests/tools/wpt/run.py
@@ -204,16 +204,24 @@ Consider installing certutil via your OS
                 print("Unable to find or install geckodriver, skipping wdspec tests")
                 kwargs["test_types"].remove("wdspec")
 
         if kwargs["prefs_root"] is None:
             prefs_root = self.browser.install_prefs(kwargs["binary"], self.venv.path)
             kwargs["prefs_root"] = prefs_root
 
 
+class Fennec(BrowserSetup):
+    name = "fennec"
+    browser_cls = browser.Fennec
+
+    def setup_kwargs(self, kwargs):
+        pass
+
+
 class Chrome(BrowserSetup):
     name = "chrome"
     browser_cls = browser.Chrome
 
     def setup_kwargs(self, kwargs):
         if kwargs["webdriver_binary"] is None:
             webdriver_binary = self.browser.find_webdriver()
 
@@ -374,16 +382,17 @@ class WebKit(BrowserSetup):
     def install(self, venv):
         raise NotImplementedError
 
     def setup_kwargs(self, kwargs):
         pass
 
 
 product_setup = {
+    "fennec": Fennec,
     "firefox": Firefox,
     "chrome": Chrome,
     "chrome_android": ChromeAndroid,
     "edge": Edge,
     "ie": InternetExplorer,
     "safari": Safari,
     "servo": Servo,
     "sauce": Sauce,
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/__init__.py
@@ -1,17 +1,17 @@
 """Subpackage where each product is defined. Each product is created by adding a
 a .py file containing a __wptrunner__ variable in the global scope. This must be
 a dictionary with the fields
 
 "product": Name of the product, assumed to be unique.
 "browser": String indicating the Browser implementation used to launch that
            product.
 "executor": Dictionary with keys as supported test types and values as the name
-            of the Executor implemantation that will be used to run that test
+            of the Executor implementation that will be used to run that test
             type.
 "browser_kwargs": String naming function that takes product, binary,
                   prefs_root and the wptrunner.run_tests kwargs dict as arguments
                   and returns a dictionary of kwargs to use when creating the
                   Browser class.
 "executor_kwargs": String naming a function that takes http server url and
                    timeout multiplier and returns kwargs to use when creating
                    the executor class.
@@ -20,16 +20,17 @@ a dictionary with the fields
 
 All classes and functions named in the above dict must be imported into the
 module global scope.
 """
 
 product_list = ["chrome",
                 "chrome_android",
                 "edge",
+                "fennec",
                 "firefox",
                 "ie",
                 "safari",
                 "sauce",
                 "servo",
                 "servodriver",
                 "opera",
                 "webkit"]
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/fennec.py
@@ -0,0 +1,268 @@
+import json
+import os
+import platform
+import signal
+import subprocess
+import sys
+import tempfile
+import traceback
+
+import mozinfo
+import moznetwork
+import mozleak
+from mozprocess import ProcessHandler
+from mozprofile import FirefoxProfile, Preferences
+from mozrunner import FennecEmulatorRunner
+from mozrunner.utils import get_stack_fixer_function
+from mozcrash import mozcrash
+
+from serve.serve import make_hosts_file
+
+from .base import (get_free_port,
+                   Browser,
+                   ExecutorBrowser,
+                   require_arg,
+                   cmd_arg,
+                   browser_command)
+from ..executors import executor_kwargs as base_executor_kwargs
+from ..executors.executormarionette import MarionetteTestharnessExecutor
+from .firefox import (get_timeout_multiplier,
+                      update_properties,
+                      executor_kwargs,
+                      FirefoxBrowser)
+
+
+__wptrunner__ = {"product": "fennec",
+                 "check_args": "check_args",
+                 "browser": "FennecBrowser",
+                 "executor": {"testharness": "MarionetteTestharnessExecutor"},
+                 "browser_kwargs": "browser_kwargs",
+                 "executor_kwargs": "executor_kwargs",
+                 "env_extras": "env_extras",
+                 "env_options": "env_options",
+                 "run_info_extras": "run_info_extras",
+                 "update_properties": "update_properties"}
+
+class FennecProfile(FirefoxProfile):
+    # WPT-specific prefs are set in FennecBrowser.start()
+    FirefoxProfile.preferences.update({
+        # Make sure Shield doesn't hit the network.
+        "app.normandy.api_url": "",
+        # Increase the APZ content response timeout in tests to 1 minute.
+        "apz.content_response_timeout": 60000,
+        # Enable output of dump()
+        "browser.dom.window.dump.enabled": True,
+        # Disable safebrowsing components
+        "browser.safebrowsing.blockedURIs.enabled": False,
+        "browser.safebrowsing.downloads.enabled": False,
+        "browser.safebrowsing.passwords.enabled": False,
+        "browser.safebrowsing.malware.enabled": False,
+        "browser.safebrowsing.phishing.enabled": False,
+        # Do not restore the last open set of tabs if the browser has crashed
+        "browser.sessionstore.resume_from_crash": False,
+        # Disable Android snippets
+        "browser.snippets.enabled": False,
+        "browser.snippets.syncPromo.enabled": False,
+        "browser.snippets.firstrunHomepage.enabled": False,
+        # Do not allow background tabs to be zombified, otherwise for tests that
+        # open additional tabs, the test harness tab itself might get unloaded
+        "browser.tabs.disableBackgroundZombification": True,
+        # Disable e10s by default
+        "browser.tabs.remote.autostart": False,
+        # Don't warn when exiting the browser
+        "browser.warnOnQuit": False,
+        # Don't send Firefox health reports to the production server
+        "datareporting.healthreport.about.reportUrl": "http://%(server)s/dummy/abouthealthreport/",
+        # Automatically unload beforeunload alerts
+        "dom.disable_beforeunload": True,
+        # Disable the ProcessHangMonitor
+        "dom.ipc.reportProcessHangs": False,
+        # No slow script dialogs
+        "dom.max_chrome_script_run_time": 0,
+        "dom.max_script_run_time": 0,
+        # Make sure opening about:addons won"t hit the network
+        "extensions.webservice.discoverURL": "http://%(server)s/dummy/discoveryURL",
+        # No hang monitor
+        "hangmonitor.timeout": 0,
+
+        "javascript.options.showInConsole": True,
+        # Ensure blocklist updates don't hit the network
+        "services.settings.server": "http://%(server)s/dummy/blocklist/",
+        # Disable password capture, so that tests that include forms aren"t
+        # influenced by the presence of the persistent doorhanger notification
+        "signon.rememberSignons": False,
+    })
+
+
+def check_args(**kwargs):
+    pass
+
+def browser_kwargs(test_type, run_info_data, **kwargs):
+    return {"package_name": kwargs["package_name"],
+            "device_serial": kwargs["device_serial"],
+            "prefs_root": kwargs["prefs_root"],
+            "extra_prefs": kwargs["extra_prefs"],
+            "test_type": test_type,
+            "debug_info": kwargs["debug_info"],
+            "symbols_path": kwargs["symbols_path"],
+            "stackwalk_binary": kwargs["stackwalk_binary"],
+            "certutil_binary": kwargs["certutil_binary"],
+            "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
+            "stackfix_dir": kwargs["stackfix_dir"],
+            "binary_args": kwargs["binary_args"],
+            "timeout_multiplier": get_timeout_multiplier(test_type,
+                                                         run_info_data,
+                                                         **kwargs),
+            "leak_check": kwargs["leak_check"],
+            "stylo_threads": kwargs["stylo_threads"],
+            "chaos_mode_flags": kwargs["chaos_mode_flags"],
+            "config": kwargs["config"]}
+
+
+def env_extras(**kwargs):
+    return []
+
+
+def run_info_extras(**kwargs):
+    return {"e10s": False,
+            "headless": False}
+
+
+def env_options():
+    # The server host is set to public localhost IP so that resources can be accessed
+    # from Android emulator
+    return {"server_host": moznetwork.get_ip(),
+            "bind_address": False,
+            "supports_debugger": True}
+
+
+def write_hosts_file(config, device):
+    new_hosts = make_hosts_file(config, moznetwork.get_ip())
+    current_hosts = device.get_file("/etc/hosts")
+    if new_hosts == current_hosts:
+        return
+    hosts_fd, hosts_path = tempfile.mkstemp()
+    try:
+        with os.fdopen(hosts_fd, "w") as f:
+            f.write(new_hosts)
+        device.remount()
+        device.push(hosts_path, "/etc/hosts")
+    finally:
+        os.remove(hosts_path)
+
+
+class FennecBrowser(FirefoxBrowser):
+    used_ports = set()
+    init_timeout = 300
+    shutdown_timeout = 60
+
+    def __init__(self, logger, prefs_root, test_type, package_name=None,
+                 device_serial="emulator-5444", **kwargs):
+        FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
+        self._package_name = package_name
+        self.device_serial = device_serial
+
+    @property
+    def package_name(self):
+        """
+        Name of app to run on emulator.
+        """
+        if self._package_name is None:
+            self._package_name = "org.mozilla.fennec"
+            user = os.getenv("USER")
+            if user:
+                self._package_name += "_" + user
+        return self._package_name
+
+    def start(self, **kwargs):
+        if self.marionette_port is None:
+            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
+            self.used_ports.add(self.marionette_port)
+
+        env = {}
+        env["MOZ_CRASHREPORTER"] = "1"
+        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
+        env["STYLO_THREADS"] = str(self.stylo_threads)
+        if self.chaos_mode_flags is not None:
+            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
+
+        preferences = self.load_prefs()
+
+        self.profile = FennecProfile(preferences=preferences)
+        self.profile.set_preferences({"marionette.port": self.marionette_port,
+                                      "dom.disable_open_during_load": False,
+                                      "places.history.enabled": False,
+                                      "dom.send_after_paint_to_content": True,
+                                      "network.preload": True})
+
+        if self.leak_check and kwargs.get("check_leaks", True):
+            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
+            if os.path.exists(self.leak_report_file):
+                os.remove(self.leak_report_file)
+            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
+        else:
+            self.leak_report_file = None
+
+        if self.ca_certificate_path is not None:
+            self.setup_ssl()
+
+        debug_args, cmd = browser_command(self.package_name,
+                                          self.binary_args if self.binary_args else [] +
+                                          [cmd_arg("marionette"), "about:blank"],
+                                          self.debug_info)
+
+        self.runner = FennecEmulatorRunner(app=self.package_name,
+                                           profile=self.profile,
+                                           cmdargs=cmd[1:],
+                                           env=env,
+                                           symbols_path=self.symbols_path,
+                                           serial=self.device_serial,
+                                           # TODO - choose appropriate log dir
+                                           logdir=os.getcwd(),
+                                           process_class=ProcessHandler,
+                                           process_args={"processOutputLine": [self.on_output]})
+
+        self.logger.debug("Starting Fennec")
+        # connect to a running emulator
+        self.runner.device.connect()
+
+        write_hosts_file(self.config, self.runner.device.device)
+
+        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
+
+        # gecko_log comes from logcat when running with device/emulator
+        logcat_args = {
+            "filterspec": "Gecko",
+            "serial": self.runner.device.app_ctx.device_serial
+        }
+        # TODO setting logcat_args["logfile"] yields an almost empty file
+        # even without filterspec
+        logcat_args["stream"] = sys.stdout
+        self.runner.device.start_logcat(**logcat_args)
+
+        self.runner.device.device.forward(
+            local="tcp:{}".format(self.marionette_port),
+            remote="tcp:{}".format(self.marionette_port))
+
+        self.logger.debug("Fennec Started")
+
+    def stop(self, force=False):
+        if self.runner is not None:
+            try:
+                if self.runner.device.connected:
+                    self.runner.device.device.remove_forwards(
+                        "tcp:{}".format(self.marionette_port))
+            except Exception:
+                traceback.print_exception(*sys.exc_info())
+            # We assume that stopping the runner prompts the
+            # browser to shut down. This allows the leak log to be written
+            for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)),
+                                  (False, lambda: self.runner.stop(signal.SIGTERM)),
+                                  (False, lambda: self.runner.stop(signal.SIGKILL))]:
+                if not force or not clean:
+                    retcode = stop_f()
+                    if retcode is not None:
+                        self.logger.info("Browser exited with return code %s" % retcode)
+                        break
+        self.logger.debug("stopped")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -50,16 +50,18 @@ def get_timeout_multiplier(test_type, ru
             return 4
         else:
             return 2
     elif run_info_data["debug"] or run_info_data.get("asan"):
         if run_info_data.get("ccov"):
             return 4
         else:
             return 3
+    elif run_info_data["os"] == "android":
+        return 4
     return 1
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
 
 
 def browser_kwargs(test_type, run_info_data, **kwargs):
@@ -368,17 +370,17 @@ class FirefoxBrowser(Browser):
             return
 
         self.logger.info("Setting up ssl")
 
         # Make sure the certutil libraries from the source tree are loaded when using a
         # local copy of certutil
         # TODO: Maybe only set this if certutil won't launch?
         env = os.environ.copy()
-        certutil_dir = os.path.dirname(self.binary)
+        certutil_dir = os.path.dirname(self.binary or self.certutil_binary)
         if mozinfo.isMac:
             env_var = "DYLD_LIBRARY_PATH"
         elif mozinfo.isUnix:
             env_var = "LD_LIBRARY_PATH"
         else:
             env_var = "PATH"
 
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -161,26 +161,29 @@ scheme host and port.""")
     debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
                                  help="Path to stackwalker program used to analyse minidumps.")
 
     debugging_group.add_argument("--pdb", action="store_true",
                                  help="Drop into pdb on python exception")
 
     config_group = parser.add_argument_group("Configuration")
     config_group.add_argument("--binary", action="store",
-                              type=abs_path, help="Binary to run tests against")
+                              type=abs_path, help="Desktop binary to run tests against")
     config_group.add_argument('--binary-arg',
                               default=[], action="append", dest="binary_args",
                               help="Extra argument for the binary")
     config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY",
                               type=abs_path, help="WebDriver server binary to use")
     config_group.add_argument('--webdriver-arg',
                               default=[], action="append", dest="webdriver_args",
                               help="Extra argument for the WebDriver binary")
-
+    config_group.add_argument("--package-name", action="store",
+                              help="Android package name to run tests against")
+    config_group.add_argument("--device-serial", action="store",
+                              help="Running Android instance to connect to, if not emulator-5554")
     config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
                               help="Path to root directory containing test metadata"),
     config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
                               help="Path to root directory containing test files"),
     config_group.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path",
                               help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)")
     config_group.add_argument("--run-info", action="store", type=abs_path,
                               help="Path to directory containing extra json files to add to run info")
--- a/testing/web-platform/wptrunner.ini
+++ b/testing/web-platform/wptrunner.ini
@@ -1,10 +1,11 @@
 [products]
 firefox =
+fennec =
 chrome =
 edge =
 servo =
 
 [web-platform-tests]
 remote_url = https://github.com/w3c/web-platform-tests.git
 branch = master
 sync_path = sync
@@ -16,9 +17,9 @@ run_info = .
 [manifest:upstream]
 tests = tests
 metadata = meta
 url_base = /
 
 [manifest:mozilla]
 tests = mozilla/tests
 metadata = mozilla/meta
-url_base = /_mozilla/
\ No newline at end of file
+url_base = /_mozilla/