Bug 1318666 - Update to latest wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Fri, 04 Nov 2016 16:28:59 +0000
changeset 323432 1ee751ec56eba735291b9ec0e3d451db2cca41e1
parent 323431 7284d58972a75c7281dc83a684f6ba56125a0b1e
child 323433 95bdac0edd59104718a8f1ea8464d39c4069c7ac
push id30978
push usercbook@mozilla.com
push dateMon, 21 Nov 2016 14:44:46 +0000
treeherdermozilla-central@0534254e9a40 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1318666
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1318666 - Update to latest wptrunner, a=testonly MozReview-Commit-ID: 9HnoUTtAf4l
testing/web-platform/harness/.travis.yml
testing/web-platform/harness/MANIFEST.in
testing/web-platform/harness/requirements.txt
testing/web-platform/harness/requirements_b2g.txt
testing/web-platform/harness/tox.ini
testing/web-platform/harness/wptrunner/browsers/__init__.py
testing/web-platform/harness/wptrunner/browsers/b2g.py
testing/web-platform/harness/wptrunner/browsers/edge.py
testing/web-platform/harness/wptrunner/browsers/servo.py
testing/web-platform/harness/wptrunner/browsers/servodriver.py
testing/web-platform/harness/wptrunner/executors/executormarionette.py
testing/web-platform/harness/wptrunner/executors/executorselenium.py
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/executors/executorservodriver.py
testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py
testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
testing/web-platform/harness/wptrunner/testharnessreport-servo.js
testing/web-platform/harness/wptrunner/testloader.py
testing/web-platform/harness/wptrunner/testrunner.py
testing/web-platform/harness/wptrunner/tests/test_chunker.py
testing/web-platform/harness/wptrunner/tests/test_hosts.py
testing/web-platform/harness/wptrunner/tests/test_testloader.py
testing/web-platform/harness/wptrunner/tests/test_update.py
testing/web-platform/harness/wptrunner/webdriver_server.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptmanifest/parser.py
testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py
testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py
testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py
testing/web-platform/harness/wptrunner/wptrunner.py
testing/web-platform/harness/wptrunner/wpttest.py
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/harness/.travis.yml
@@ -0,0 +1,20 @@
+language: python
+python: 2.7
+
+sudo: false
+
+cache:
+  directories:
+    - $HOME/.cache/pip
+
+env:
+  - TOXENV="{py27,pypy}-base"
+  - TOXENV="{py27,pypy}-chrome"
+  - TOXENV="{py27,pypy}-firefox"
+  - TOXENV="{py27,pypy}-servo"
+
+install:
+  - pip install -U tox
+
+script:
+  - tox
--- a/testing/web-platform/harness/MANIFEST.in
+++ b/testing/web-platform/harness/MANIFEST.in
@@ -1,13 +1,17 @@
 exclude MANIFEST.in
 include requirements.txt
 include wptrunner/browsers/b2g_setup/*
 include wptrunner.default.ini
 include wptrunner/testharness_runner.html
 include wptrunner/testharnessreport.js
 include wptrunner/testharnessreport-servo.js
+include wptrunner/testharnessreport-servodriver.js
 include wptrunner/executors/testharness_marionette.js
+include wptrunner/executors/testharness_servodriver.js
 include wptrunner/executors/testharness_webdriver.js
 include wptrunner/executors/reftest.js
 include wptrunner/executors/reftest-wait.js
+include wptrunner/executors/reftest-wait_servodriver.js
+include wptrunner/executors/reftest-wait_webdriver.js
 include wptrunner/config.json
 include wptrunner/browsers/server-locations.txt
\ No newline at end of file
--- a/testing/web-platform/harness/requirements.txt
+++ b/testing/web-platform/harness/requirements.txt
@@ -1,4 +1,4 @@
 html5lib >= 0.99
 mozinfo >= 0.7
-mozlog >= 3.0
+mozlog >= 3.3
 mozdebug >= 0.1
deleted file mode 100644
--- a/testing/web-platform/harness/requirements_b2g.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-fxos_appgen >= 0.5
-mozdevice >= 0.41
-gaiatest >= 0.26
-marionette_client >= 0.7.10
-moznetwork >= 0.24
-mozprofile >= 0.21
-mozrunner >= 6.1
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/harness/tox.ini
@@ -0,0 +1,15 @@
+[pytest]
+xfail_strict=true
+
+[tox]
+envlist = {py27,pypy}-{base,b2g,chrome,firefox,servo}
+
+[testenv]
+deps =
+     pytest>=2.9
+     -r{toxinidir}/requirements.txt
+     chrome: -r{toxinidir}/requirements_chrome.txt
+     firefox: -r{toxinidir}/requirements_firefox.txt
+     servo: -r{toxinidir}/requirements_servo.txt
+
+commands = py.test []
--- a/testing/web-platform/harness/wptrunner/browsers/__init__.py
+++ b/testing/web-platform/harness/wptrunner/browsers/__init__.py
@@ -21,13 +21,13 @@ a dictionary with the fields
                    the executor class.
 "env_options": String naming a funtion of no arguments that returns the
                arguments passed to the TestEnvironment.
 
 All classes and functions named in the above dict must be imported into the
 module global scope.
 """
 
-product_list = ["b2g",
-                "chrome",
+product_list = ["chrome",
+                "edge",
                 "firefox",
                 "servo",
                 "servodriver"]
deleted file mode 100644
--- a/testing/web-platform/harness/wptrunner/browsers/b2g.py
+++ /dev/null
@@ -1,243 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import tempfile
-import shutil
-import subprocess
-
-import fxos_appgen
-import gaiatest
-import mozdevice
-import moznetwork
-import mozrunner
-from marionette import expected
-from marionette.by import By
-from marionette.wait import Wait
-from mozprofile import FirefoxProfile, Preferences
-
-from .base import get_free_port, BrowserError, Browser, ExecutorBrowser
-from ..executors.executormarionette import MarionetteTestharnessExecutor
-from ..hosts import HostsFile, HostsLine
-from ..environment import hostnames
-
-here = os.path.split(__file__)[0]
-
-__wptrunner__ = {"product": "b2g",
-                 "check_args": "check_args",
-                 "browser": "B2GBrowser",
-                 "executor": {"testharness": "B2GMarionetteTestharnessExecutor"},
-                 "browser_kwargs": "browser_kwargs",
-                 "executor_kwargs": "executor_kwargs",
-                 "env_options": "env_options"}
-
-
-def check_args(**kwargs):
-    pass
-
-
-def browser_kwargs(test_environment, **kwargs):
-    return {"prefs_root": kwargs["prefs_root"],
-            "no_backup": kwargs.get("b2g_no_backup", False)}
-
-
-def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
-                    **kwargs):
-    timeout_multiplier = kwargs["timeout_multiplier"]
-    if timeout_multiplier is None:
-        timeout_multiplier = 2
-
-    executor_kwargs = {"server_config": server_config,
-                       "timeout_multiplier": timeout_multiplier,
-                       "close_after_done": False}
-
-    if test_type == "reftest":
-        executor_kwargs["cache_manager"] = cache_manager
-
-    return executor_kwargs
-
-
-def env_options():
-    return {"host": "web-platform.test",
-            "bind_hostname": "false",
-            "test_server_port": False}
-
-
-class B2GBrowser(Browser):
-    used_ports = set()
-    init_timeout = 180
-
-    def __init__(self, logger, prefs_root, no_backup=False):
-        Browser.__init__(self, logger)
-        logger.info("Waiting for device")
-        subprocess.call(["adb", "wait-for-device"])
-        self.device = mozdevice.DeviceManagerADB()
-        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
-        self.used_ports.add(self.marionette_port)
-        self.cert_test_app = None
-        self.runner = None
-        self.prefs_root = prefs_root
-
-        self.no_backup = no_backup
-        self.backup_path = None
-        self.backup_paths = []
-        self.backup_dirs = []
-
-    def setup(self):
-        self.logger.info("Running B2G setup")
-        self.backup_path = tempfile.mkdtemp()
-
-        self.logger.debug("Backing up device to %s"  % (self.backup_path,))
-
-        if not self.no_backup:
-            self.backup_dirs = [("/data/local", os.path.join(self.backup_path, "local")),
-                                ("/data/b2g/mozilla", os.path.join(self.backup_path, "profile"))]
-
-            self.backup_paths = [("/system/etc/hosts", os.path.join(self.backup_path, "hosts"))]
-
-            for remote, local in self.backup_dirs:
-                self.device.getDirectory(remote, local)
-
-            for remote, local in self.backup_paths:
-                self.device.getFile(remote, local)
-
-        self.setup_hosts()
-
-    def start(self):
-        profile = FirefoxProfile()
-
-        profile.set_preferences({"dom.disable_open_during_load": False,
-                                 "marionette.defaultPrefs.enabled": True})
-
-        self.logger.debug("Creating device runner")
-        self.runner = mozrunner.B2GDeviceRunner(profile=profile)
-        self.logger.debug("Starting device runner")
-        self.runner.start()
-        self.logger.debug("Device runner started")
-
-    def setup_hosts(self):
-        host_ip = moznetwork.get_ip()
-
-        temp_dir = tempfile.mkdtemp()
-        hosts_path = os.path.join(temp_dir, "hosts")
-        remote_path = "/system/etc/hosts"
-        try:
-            self.device.getFile("/system/etc/hosts", hosts_path)
-
-            with open(hosts_path) as f:
-                hosts_file = HostsFile.from_file(f)
-
-            for canonical_hostname in hostnames:
-                hosts_file.set_host(HostsLine(host_ip, canonical_hostname))
-
-            with open(hosts_path, "w") as f:
-                hosts_file.to_file(f)
-
-            self.logger.info("Installing hosts file")
-
-            self.device.remount()
-            self.device.removeFile(remote_path)
-            self.device.pushFile(hosts_path, remote_path)
-        finally:
-            os.unlink(hosts_path)
-            os.rmdir(temp_dir)
-
-    def load_prefs(self):
-        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
-        if os.path.exists(prefs_path):
-            preferences = Preferences.read_prefs(prefs_path)
-        else:
-            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
-            preferences = []
-
-        return preferences
-
-    def stop(self):
-        pass
-
-    def on_output(self):
-        raise NotImplementedError
-
-    def cleanup(self):
-        self.logger.debug("Running browser cleanup steps")
-
-        self.device.remount()
-
-        for remote, local in self.backup_dirs:
-            self.device.removeDir(remote)
-            self.device.pushDir(local, remote)
-
-        for remote, local in self.backup_paths:
-            self.device.removeFile(remote)
-            self.device.pushFile(local, remote)
-
-        shutil.rmtree(self.backup_path)
-        self.device.reboot(wait=True)
-
-    def pid(self):
-        return None
-
-    def is_alive(self):
-        return True
-
-    def executor_browser(self):
-        return B2GExecutorBrowser, {"marionette_port": self.marionette_port}
-
-
-class B2GExecutorBrowser(ExecutorBrowser):
-    # The following methods are called from a different process
-    def __init__(self, *args, **kwargs):
-        ExecutorBrowser.__init__(self, *args, **kwargs)
-
-        import sys, subprocess
-
-        self.device = mozdevice.ADBB2G()
-        self.device.forward("tcp:%s" % self.marionette_port,
-                            "tcp:2828")
-        self.executor = None
-        self.marionette = None
-        self.gaia_device = None
-        self.gaia_apps = None
-
-    def after_connect(self, executor):
-        self.executor = executor
-        self.marionette = executor.marionette
-        self.executor.logger.debug("Running browser.after_connect steps")
-
-        self.gaia_apps = gaiatest.GaiaApps(marionette=executor.marionette)
-
-        self.executor.logger.debug("Waiting for homescreen to load")
-
-        # Moved out of gaia_test temporarily
-        self.executor.logger.info("Waiting for B2G to be ready")
-        self.wait_for_homescreen(timeout=60)
-
-        self.install_cert_app()
-        self.use_cert_app()
-
-    def install_cert_app(self):
-        """Install the container app used to run the tests"""
-        if fxos_appgen.is_installed("CertTest App"):
-            self.executor.logger.info("CertTest App is already installed")
-            return
-        self.executor.logger.info("Installing CertTest App")
-        app_path = os.path.join(here, "b2g_setup", "certtest_app.zip")
-        fxos_appgen.install_app("CertTest App", app_path, marionette=self.marionette)
-        self.executor.logger.debug("Install complete")
-
-    def use_cert_app(self):
-        """Start the app used to run the tests"""
-        self.executor.logger.info("Homescreen loaded")
-        self.gaia_apps.launch("CertTest App")
-
-    def wait_for_homescreen(self, timeout):
-        self.executor.logger.info("Waiting for home screen to load")
-        Wait(self.marionette, timeout).until(expected.element_present(
-            By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
-
-
-class B2GMarionetteTestharnessExecutor(MarionetteTestharnessExecutor):
-    def after_connect(self):
-        self.browser.after_connect(self)
-        MarionetteTestharnessExecutor.after_connect(self)
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/browsers/edge.py
@@ -0,0 +1,71 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from .base import Browser, ExecutorBrowser, require_arg
+from ..webdriver_server import EdgeDriverServer
+from ..executors import executor_kwargs as base_executor_kwargs
+from ..executors.executorselenium import (SeleniumTestharnessExecutor,
+                                          SeleniumRefTestExecutor)
+
+__wptrunner__ = {"product": "edge",
+                 "check_args": "check_args",
+                 "browser": "EdgeBrowser",
+                 "executor": {"testharness": "SeleniumTestharnessExecutor",
+                              "reftest": "SeleniumRefTestExecutor"},
+                 "browser_kwargs": "browser_kwargs",
+                 "executor_kwargs": "executor_kwargs",
+                 "env_options": "env_options"}
+
+
+def check_args(**kwargs):
+    require_arg(kwargs, "webdriver_binary")
+
+def browser_kwargs(**kwargs):
+    return {"webdriver_binary": kwargs["webdriver_binary"]}
+
+def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
+                    **kwargs):
+    from selenium.webdriver import DesiredCapabilities
+
+    executor_kwargs = base_executor_kwargs(test_type, server_config,
+                                           cache_manager, **kwargs)
+    executor_kwargs["close_after_done"] = True
+    executor_kwargs["capabilities"] = dict(DesiredCapabilities.EDGE.items())
+    return executor_kwargs
+
+def env_options():
+    return {"host": "web-platform.test",
+            "bind_hostname": "true",
+            "supports_debugger": False}
+
+class EdgeBrowser(Browser):
+    used_ports = set()
+
+    def __init__(self, logger, webdriver_binary):
+        Browser.__init__(self, logger)
+        self.server = EdgeDriverServer(self.logger, binary=webdriver_binary)
+        self.webdriver_host = "localhost"
+        self.webdriver_port = self.server.port
+
+    def start(self):
+        print self.server.url
+        self.server.start()
+
+    def stop(self):
+        self.server.stop()
+
+    def pid(self):
+        return self.server.pid
+
+    def is_alive(self):
+        # TODO(ato): This only indicates the server is alive,
+        # and doesn't say anything about whether a browser session
+        # is active.
+        return self.server.is_alive()
+
+    def cleanup(self):
+        self.stop()
+
+    def executor_browser(self):
+        return ExecutorBrowser, {"webdriver_url": self.server.url}
--- a/testing/web-platform/harness/wptrunner/browsers/servo.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servo.py
@@ -1,25 +1,26 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 
 from .base import NullBrowser, ExecutorBrowser, require_arg
 from ..executors import executor_kwargs as base_executor_kwargs
-from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor
+from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor
 
 here = os.path.join(os.path.split(__file__)[0])
 
 __wptrunner__ = {"product": "servo",
                  "check_args": "check_args",
                  "browser": "ServoBrowser",
                  "executor": {"testharness": "ServoTestharnessExecutor",
-                              "reftest": "ServoRefTestExecutor"},
+                              "reftest": "ServoRefTestExecutor",
+                              "wdspec": "ServoWdspecExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options",
                  "run_info_extras": "run_info_extras",
                  "update_properties": "update_properties"}
 
 
 def check_args(**kwargs):
@@ -59,17 +60,17 @@ def update_properties():
 
 
 def render_arg(render_backend):
     return {"cpu": "--cpu", "webrender": "-w"}[render_backend]
 
 
 class ServoBrowser(NullBrowser):
     def __init__(self, logger, binary, debug_info=None, binary_args=None,
-                 user_stylesheets=None, render_backend="cpu"):
+                 user_stylesheets=None, render_backend="webrender"):
         NullBrowser.__init__(self, logger)
         self.binary = binary
         self.debug_info = debug_info
         self.binary_args = binary_args or []
         self.user_stylesheets = user_stylesheets or []
         self.render_backend = render_backend
 
     def executor_browser(self):
--- a/testing/web-platform/harness/wptrunner/browsers/servodriver.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servodriver.py
@@ -75,17 +75,17 @@ def make_hosts_file():
         f.write(hosts_text)
     return hosts_path
 
 
 class ServoWebDriverBrowser(Browser):
     used_ports = set()
 
     def __init__(self, logger, binary, debug_info=None, webdriver_host="127.0.0.1",
-                 user_stylesheets=None, render_backend="cpu"):
+                 user_stylesheets=None, render_backend="webrender"):
         Browser.__init__(self, logger)
         self.binary = binary
         self.webdriver_host = webdriver_host
         self.webdriver_port = None
         self.proc = None
         self.debug_info = debug_info
         self.hosts_path = make_hosts_file()
         self.command = None
--- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -12,21 +12,21 @@ import traceback
 import urlparse
 import uuid
 from collections import defaultdict
 
 from ..wpttest import WdspecResult, WdspecSubtestResult
 
 errors = None
 marionette = None
+pytestrunner = None
 webdriver = None
 
 here = os.path.join(os.path.split(__file__)[0])
 
-from . import pytestrunner
 from .base import (ExecutorException,
                    Protocol,
                    RefTestExecutor,
                    RefTestImplementation,
                    TestExecutor,
                    TestharnessExecutor,
                    testharness_result_converter,
                    reftest_result_converter,
@@ -36,28 +36,26 @@ from ..testrunner import Stop
 from ..webdriver_server import GeckoDriverServer
 
 # Extra timeout to use after internal test timeout at which the harness
 # should force a timeout
 extra_timeout = 5 # seconds
 
 
 def do_delayed_imports():
-    global errors, marionette, webdriver
+    global errors, marionette
 
     # Marionette client used to be called marionette, recently it changed
     # to marionette_driver for unfathomable reasons
     try:
         import marionette
         from marionette import errors
     except ImportError:
         from marionette_driver import marionette, errors
 
-    import webdriver
-
 
 class MarionetteProtocol(Protocol):
     def __init__(self, executor, browser):
         do_delayed_imports()
 
         Protocol.__init__(self, executor, browser)
         self.marionette = None
         self.marionette_port = browser.marionette_port
@@ -556,16 +554,17 @@ class WdspecRun(object):
             self.result = False, ("ERROR", message)
         finally:
             self.result_flag.set()
 
 
 class MarionetteWdspecExecutor(WdspecExecutor):
     def __init__(self, browser, server_config, webdriver_binary,
                  timeout_multiplier=1, close_after_done=True, debug_info=None):
+        self.do_delayed_imports()
         WdspecExecutor.__init__(self, browser, server_config,
                                 timeout_multiplier=timeout_multiplier,
                                 debug_info=debug_info)
         self.webdriver_binary = webdriver_binary
         self.protocol = RemoteMarionetteProtocol(self, browser)
 
     def is_alive(self):
         return self.protocol.is_alive
@@ -585,8 +584,13 @@ class MarionetteWdspecExecutor(WdspecExe
             return self.convert_result(test, data)
 
         return (test.result_cls(*data), [])
 
     def do_wdspec(self, session, path, timeout):
         harness_result = ("OK", None)
         subtest_results = pytestrunner.run(path, session, timeout=timeout)
         return (harness_result, subtest_results)
+
+    def do_delayed_imports(self):
+        global pytestrunner, webdriver
+        from . import pytestrunner
+        import webdriver
--- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
@@ -17,30 +17,31 @@ from .base import (ExecutorException,
                    RefTestImplementation,
                    TestExecutor,
                    TestharnessExecutor,
                    testharness_result_converter,
                    reftest_result_converter,
                    strip_server)
 from ..testrunner import Stop
 
-
 here = os.path.join(os.path.split(__file__)[0])
 
 webdriver = None
 exceptions = None
+RemoteConnection = None
 
 extra_timeout = 5
 
 def do_delayed_imports():
     global webdriver
     global exceptions
+    global RemoteConnection
     from selenium import webdriver
     from selenium.common import exceptions
-
+    from selenium.webdriver.remote.remote_connection import RemoteConnection
 
 class SeleniumProtocol(Protocol):
     def __init__(self, executor, browser, capabilities, **kwargs):
         do_delayed_imports()
 
         Protocol.__init__(self, executor, browser)
         self.capabilities = capabilities
         self.url = browser.webdriver_url
@@ -48,18 +49,19 @@ class SeleniumProtocol(Protocol):
 
     def setup(self, runner):
         """Connect to browser via Selenium's WebDriver implementation."""
         self.runner = runner
         self.logger.debug("Connecting to Selenium on URL: %s" % self.url)
 
         session_started = False
         try:
-            self.webdriver = webdriver.Remote(
-                self.url, desired_capabilities=self.capabilities)
+            self.webdriver = webdriver.Remote(command_executor=RemoteConnection(self.url.strip("/"),
+                                                                                resolve_ip=False),
+                                              desired_capabilities=self.capabilities)
         except:
             self.logger.warning(
                 "Connecting to Selenium failed:\n%s" % traceback.format_exc())
         else:
             self.logger.debug("Selenium session started")
             session_started = True
 
         if not session_started:
@@ -226,27 +228,17 @@ class SeleniumRefTestExecutor(RefTestExe
             self.wait_script = f.read()
 
     def is_alive(self):
         return self.protocol.is_alive()
 
     def do_test(self, test):
         self.logger.info("Test requires OS-level window focus")
 
-        if self.close_after_done and self.has_window:
-            self.protocol.webdriver.close()
-            self.protocol.webdriver.switch_to_window(
-                self.protocol.webdriver.window_handles[-1])
-            self.has_window = False
-
-        if not self.has_window:
-            self.protocol.webdriver.execute_script(self.script)
-            self.protocol.webdriver.switch_to_window(
-                self.protocol.webdriver.window_handles[-1])
-            self.has_window = True
+        self.protocol.webdriver.set_window_size(600, 600)
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
     def screenshot(self, test, viewport_size, dpi):
         # https://github.com/w3c/wptrunner/issues/166
         assert viewport_size is None
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -1,34 +1,44 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import base64
 import hashlib
+import httplib
 import json
 import os
 import subprocess
 import tempfile
 import threading
+import traceback
 import urlparse
 import uuid
 from collections import defaultdict
 
 from mozprocess import ProcessHandler
 
 from .base import (ExecutorException,
                    Protocol,
                    RefTestImplementation,
                    testharness_result_converter,
-                   reftest_result_converter)
+                   reftest_result_converter,
+                   WdspecExecutor)
 from .process import ProcessTestExecutor
 from ..browsers.base import browser_command
+from ..wpttest import WdspecResult, WdspecSubtestResult
+from ..webdriver_server import ServoDriverServer
+from .executormarionette import WdspecRun
+
+pytestrunner = None
 render_arg = None
+webdriver = None
 
+extra_timeout = 5 # seconds
 
 def do_delayed_imports():
     global render_arg
     from ..browsers.servo import render_arg
 
 hosts_text = """127.0.0.1 web-platform.test
 127.0.0.1 www.web-platform.test
 127.0.0.1 www1.web-platform.test
@@ -200,31 +210,33 @@ class ServoRefTestExecutor(ProcessTestEx
     def screenshot(self, test, viewport_size, dpi):
         full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
             debug_args, command = browser_command(
                 self.binary,
                 [render_arg(self.browser.render_backend), "--hard-fail", "--exit",
                  "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
-                 "--output=%s" % output_path, full_url],
+                 "--output=%s" % output_path, full_url] + self.browser.binary_args,
                 self.debug_info)
 
             for stylesheet in self.browser.user_stylesheets:
                 command += ["--user-stylesheet", stylesheet]
 
-            for pref in test.environment.get('prefs', {}):
-                command += ["--pref", pref]
+            for pref, value in test.environment.get('prefs', {}).iteritems():
+                command += ["--pref", "%s=%s" % (pref, value)]
 
-            if viewport_size:
-                command += ["--resolution", viewport_size]
+            command += ["--resolution", viewport_size or "800x600"]
 
             if dpi:
                 command += ["--device-pixel-ratio", dpi]
 
+            # Run ref tests in headless mode
+            command += ["-z"]
+
             self.command = debug_args + command
 
             env = os.environ.copy()
             env["HOST_FILE"] = self.hosts_path
             env["RUST_BACKTRACE"] = "1"
 
             if not self.interactive:
                 self.proc = ProcessHandler(self.command,
@@ -268,8 +280,88 @@ class ServoRefTestExecutor(ProcessTestEx
     def on_output(self, line):
         line = line.decode("utf8", "replace")
         if self.interactive:
             print line
         else:
             self.logger.process_output(self.proc.pid,
                                        line,
                                        " ".join(self.command))
+
+class ServoWdspecProtocol(Protocol):
+    def __init__(self, executor, browser):
+        self.do_delayed_imports()
+        Protocol.__init__(self, executor, browser)
+        self.session = None
+        self.server = None
+
+    def setup(self, runner):
+        try:
+            self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend)
+            self.server.start(block=False)
+            self.logger.info(
+                "WebDriver HTTP server listening at %s" % self.server.url)
+
+            self.logger.info(
+                "Establishing new WebDriver session with %s" % self.server.url)
+            self.session = webdriver.Session(
+                self.server.host, self.server.port, self.server.base_path)
+        except Exception:
+            self.logger.error(traceback.format_exc())
+            self.executor.runner.send_message("init_failed")
+        else:
+            self.executor.runner.send_message("init_succeeded")
+
+    def teardown(self):
+        if self.server is not None:
+            try:
+                if self.session.session_id is not None:
+                    self.session.end()
+            except Exception:
+                pass
+            if self.server.is_alive:
+                self.server.stop()
+
+    @property
+    def is_alive(self):
+        conn = httplib.HTTPConnection(self.server.host, self.server.port)
+        conn.request("HEAD", self.server.base_path + "invalid")
+        res = conn.getresponse()
+        return res.status == 404
+
+    def do_delayed_imports(self):
+        global pytestrunner, webdriver
+        from . import pytestrunner
+        import webdriver
+
+
+class ServoWdspecExecutor(WdspecExecutor):
+    def __init__(self, browser, server_config,
+                 timeout_multiplier=1, close_after_done=True, debug_info=None,
+                 **kwargs):
+        WdspecExecutor.__init__(self, browser, server_config,
+                                timeout_multiplier=timeout_multiplier,
+                                debug_info=debug_info)
+        self.protocol = ServoWdspecProtocol(self, browser)
+
+    def is_alive(self):
+        return self.protocol.is_alive
+
+    def on_environment_change(self, new_environment):
+        pass
+
+    def do_test(self, test):
+        timeout = test.timeout * self.timeout_multiplier + extra_timeout
+
+        success, data = WdspecRun(self.do_wdspec,
+                                  self.protocol.session,
+                                  test.path,
+                                  timeout).run()
+
+        if success:
+            return self.convert_result(test, data)
+
+        return (test.result_cls(*data), [])
+
+    def do_wdspec(self, session, path, timeout):
+        harness_result = ("OK", None)
+        subtest_results = pytestrunner.run(path, session, timeout=timeout)
+        return (harness_result, subtest_results)
--- a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
@@ -9,29 +9,28 @@ import threading
 import time
 import traceback
 
 from .base import (Protocol,
                    RefTestExecutor,
                    RefTestImplementation,
                    TestharnessExecutor,
                    strip_server)
-from .. import webdriver
 from ..testrunner import Stop
 
 webdriver = None
 
 here = os.path.join(os.path.split(__file__)[0])
 
 extra_timeout = 5
 
 
 def do_delayed_imports():
     global webdriver
-    import webdriver
+    from tools import webdriver
 
 
 class ServoWebDriverProtocol(Protocol):
     def __init__(self, executor, browser, capabilities, **kwargs):
         do_delayed_imports()
         Protocol.__init__(self, executor, browser)
         self.capabilities = capabilities
         self.host = browser.webdriver_host
--- a/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py
+++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/fixtures.py
@@ -1,28 +1,32 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import pytest
+import webdriver
+
+import contextlib
+import httplib
 
 
 """pytest fixtures for use in Python-based WPT tests.
 
 The purpose of test fixtures is to provide a fixed baseline upon which
 tests can reliably and repeatedly execute.
 """
 
 
 class Session(object):
     """Fixture to allow access to wptrunner's existing WebDriver session
     in tests.
 
     The session is not created by default to enable testing of session
-    creation.  However, a module-scoped session will be implicitly created
+    creation.  However, a function-scoped session will be implicitly created
     at the first call to a WebDriver command.  This means methods such as
     `session.send_command` and `session.session_id` are possible to use
     without having a session.
 
     To illustrate implicit session creation::
 
         def test_session_scope(session):
             # at this point there is no session
@@ -40,19 +44,93 @@ class Session(object):
 
         @pytest.fixture(scope="function")
         def setup(request, session):
             session.url = "https://example.org"
 
         def test_something(setup, session):
             assert session.url == "https://example.org"
 
-    The session is closed when the test module goes out of scope by an
-    implicit call to `session.end`.
+    When the test function goes out of scope, any remaining user prompts
+    and opened windows are closed, and the current browsing context is
+    switched back to the top-level browsing context.
     """
 
     def __init__(self, client):
         self.client = client
 
-    @pytest.fixture(scope="module")
+    @pytest.fixture(scope="function")
     def session(self, request):
-        request.addfinalizer(self.client.end)
+        # finalisers are popped off a stack,
+        # making their ordering reverse
+        request.addfinalizer(self.switch_to_top_level_browsing_context)
+        request.addfinalizer(self.restore_windows)
+        request.addfinalizer(self.dismiss_user_prompts)
+
         return self.client
+
+    def dismiss_user_prompts(self):
+        """Dismisses any open user prompts in windows."""
+        current_window = self.client.window_handle
+
+        for window in self.windows():
+            self.client.window_handle = window
+            try:
+                self.client.alert.dismiss()
+            except webdriver.NoSuchAlertException:
+                pass
+
+        self.client.window_handle = current_window
+
+    def restore_windows(self):
+        """Closes superfluous windows opened by the test without ending
+        the session implicitly by closing the last window.
+        """
+        current_window = self.client.window_handle
+
+        for window in self.windows(exclude=[current_window]):
+            self.client.window_handle = window
+            if len(self.client.window_handles) > 1:
+                self.client.close()
+
+        self.client.window_handle = current_window
+
+    def switch_to_top_level_browsing_context(self):
+        """If the current browsing context selected by WebDriver is a
+        `<frame>` or an `<iframe>`, switch it back to the top-level
+        browsing context.
+        """
+        self.client.switch_frame(None)
+
+    def windows(self, exclude=None):
+        """Set of window handles, filtered by an `exclude` list if
+        provided.
+        """
+        if exclude is None:
+            exclude = []
+        wins = [w for w in self.client.handles if w not in exclude]
+        return set(wins)
+
+
+class HTTPRequest(object):
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+
+    def head(self, path):
+        return self._request("HEAD", path)
+
+    def get(self, path):
+        return self._request("GET", path)
+
+    @contextlib.contextmanager
+    def _request(self, method, path):
+        conn = httplib.HTTPConnection(self.host, self.port)
+        try:
+            conn.request(method, path)
+            yield conn.getresponse()
+        finally:
+            conn.close()
+
+
+@pytest.fixture(scope="module")
+def http(session):
+    return HTTPRequest(session.transport.host, session.transport.port)
--- a/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
+++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
@@ -40,16 +40,17 @@ def run(path, session, timeout=0):
         status, message, stacktrace).
     """
 
     if pytest is None:
         do_delayed_imports()
 
     recorder = SubtestResultRecorder()
     plugins = [recorder,
+               fixtures,
                fixtures.Session(session)]
 
     # TODO(ato): Deal with timeouts
 
     with TemporaryDirectory() as cache:
         pytest.main(["--strict",  # turn warnings into errors
                      "--verbose",  # show each individual subtest
                      "--capture", "no",  # enable stdout/stderr from tests
--- a/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
+++ b/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
@@ -1,23 +1,27 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 var callback = arguments[arguments.length - 1];
 window.timeout_multiplier = %(timeout_multiplier)d;
 
-window.addEventListener("message", function(event) {
-  var tests = event.data[0];
-  var status = event.data[1];
+window.addEventListener("message", function f(event) {
+  if (event.data.type != "complete") {
+    return;
+  }
+  window.removeEventListener("message", f);
+
+  var tests = event.data.tests;
+  var status = event.data.status;
 
   var subtest_results = tests.map(function(x) {
-      return [x.name, x.status, x.message, x.stack]
+    return [x.name, x.status, x.message, x.stack]
   });
-
   clearTimeout(timer);
   callback(["%(url)s",
             status.status,
             status.message,
             status.stack,
             subtest_results]);
 }, false);
 
--- a/testing/web-platform/harness/wptrunner/testharnessreport-servo.js
+++ b/testing/web-platform/harness/wptrunner/testharnessreport-servo.js
@@ -1,19 +1,20 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 var props = {output:%(output)d};
-
+var start_loc = document.createElement('a');
+start_loc.href = location.href;
 setup(props);
 
 add_completion_callback(function (tests, harness_status) {
-    var id = location.pathname + location.search + location.hash;
-    alert("RESULT: " + JSON.stringify([
+    var id = start_loc.pathname + start_loc.search + start_loc.hash;
+    console.log("ALERT: RESULT: " + JSON.stringify([
         id,
         harness_status.status,
         harness_status.message,
         harness_status.stack,
         tests.map(function(t) {
             return [t.name, t.status, t.message, t.stack]
         }),
     ]));
--- a/testing/web-platform/harness/wptrunner/testloader.py
+++ b/testing/web-platform/harness/wptrunner/testloader.py
@@ -22,16 +22,17 @@ def do_delayed_imports():
     from manifest import update as manifest_update
 
 class TestChunker(object):
     def __init__(self, total_chunks, chunk_number):
         self.total_chunks = total_chunks
         self.chunk_number = chunk_number
         assert self.chunk_number <= self.total_chunks
         self.logger = structured.get_default_logger()
+        assert self.logger
 
     def __call__(self, manifest):
         raise NotImplementedError
 
 
 class Unchunked(TestChunker):
     def __init__(self, *args, **kwargs):
         TestChunker.__init__(self, *args, **kwargs)
--- a/testing/web-platform/harness/wptrunner/testrunner.py
+++ b/testing/web-platform/harness/wptrunner/testrunner.py
@@ -163,17 +163,17 @@ def next_manager_number():
     return local
 
 
 class TestRunnerManager(threading.Thread):
     init_lock = threading.Lock()
 
     def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
                  executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
-                 pause_on_unexpected=False, debug_info=None):
+                 pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None):
         """Thread that owns a single TestRunner process and any processes required
         by the TestRunner (e.g. the Firefox binary).
 
         TestRunnerManagers are responsible for launching the browser process and the
         runner process, and for logging the test progress. The actual test running
         is done by the TestRunner. In particular they:
 
         * Start the binary of the program under test
@@ -202,16 +202,17 @@ class TestRunnerManager(threading.Thread
         self.browser_started = False
 
         # Flags used to shut down this thread if we get a sigint
         self.parent_stop_flag = stop_flag
         self.child_stop_flag = multiprocessing.Event()
 
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
+        self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
 
         self.manager_number = next_manager_number()
 
         self.command_queue = Queue()
         self.remote_queue = Queue()
 
         self.test_runner_proc = None
@@ -521,17 +522,18 @@ class TestRunnerManager(threading.Thread
                              message=file_result.message,
                              expected=expected,
                              extra=file_result.extra)
 
         self.test = None
 
         restart_before_next = (test.restart_after or
                                file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
-                               subtest_unexpected or is_unexpected)
+                               ((subtest_unexpected or is_unexpected)
+                                and self.restart_on_unexpected))
 
         if (self.pause_after_test or
             (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
             self.logger.info("Pausing until the browser exits")
             self.send_message("wait")
         else:
             self.after_test_ended(restart_before_next)
 
@@ -588,28 +590,30 @@ class TestQueue(object):
 
 
 class ManagerGroup(object):
     def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
                  browser_cls, browser_kwargs,
                  executor_cls, executor_kwargs,
                  pause_after_test=False,
                  pause_on_unexpected=False,
+                 restart_on_unexpected=True,
                  debug_info=None):
         """Main thread object that owns all the TestManager threads."""
         self.suite_name = suite_name
         self.size = size
         self.test_source_cls = test_source_cls
         self.test_source_kwargs = test_source_kwargs
         self.browser_cls = browser_cls
         self.browser_kwargs = browser_kwargs
         self.executor_cls = executor_cls
         self.executor_kwargs = executor_kwargs
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
+        self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
 
         self.pool = set()
         # Event that is polled by threads so that they can gracefully exit in the face
         # of sigint
         self.stop_flag = threading.Event()
         self.logger = structuredlog.StructuredLogger(suite_name)
         self.test_queue = None
@@ -638,16 +642,17 @@ class ManagerGroup(object):
                                             self.test_source_cls,
                                             self.browser_cls,
                                             self.browser_kwargs,
                                             self.executor_cls,
                                             self.executor_kwargs,
                                             self.stop_flag,
                                             self.pause_after_test,
                                             self.pause_on_unexpected,
+                                            self.restart_on_unexpected,
                                             self.debug_info)
                 manager.start()
                 self.pool.add(manager)
             self.wait()
 
     def is_alive(self):
         """Boolean indicating whether any manager in the group is still alive"""
         return any(manager.is_alive() for manager in self.pool)
--- a/testing/web-platform/harness/wptrunner/tests/test_chunker.py
+++ b/testing/web-platform/harness/wptrunner/tests/test_chunker.py
@@ -1,17 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 import sys
-sys.path.insert(0, "..")
+from os.path import join, dirname
+from mozlog import structured
+
+import pytest
 
-from wptrunner import wptrunner
+sys.path.insert(0, join(dirname(__file__), "..", ".."))
+
+from wptrunner.testloader import EqualTimeChunker
+
+structured.set_default_logger(structured.structuredlog.StructuredLogger("TestChunker"))
 
 class MockTest(object):
     def __init__(self, id, timeout=10):
         self.id = id
         self.item_type = "testharness"
         self.timeout = timeout
 
 
@@ -23,57 +30,57 @@ def make_mock_manifest(*items):
     return rv
 
 
 class TestEqualTimeChunker(unittest.TestCase):
 
     def test_include_all(self):
         tests = make_mock_manifest(("a", 10), ("a/b", 10), ("c", 10))
 
-        chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests))
-        chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests))
-        chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests))
+        chunk_1 = list(EqualTimeChunker(3, 1)(tests))
+        chunk_2 = list(EqualTimeChunker(3, 2)(tests))
+        chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:10], chunk_1)
         self.assertEquals(tests[10:20], chunk_2)
         self.assertEquals(tests[20:], chunk_3)
 
     def test_include_all_1(self):
         tests = make_mock_manifest(("a", 5), ("a/b", 5), ("c", 10), ("d", 10))
 
-        chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests))
-        chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests))
-        chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests))
+        chunk_1 = list(EqualTimeChunker(3, 1)(tests))
+        chunk_2 = list(EqualTimeChunker(3, 2)(tests))
+        chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:10], chunk_1)
         self.assertEquals(tests[10:20], chunk_2)
         self.assertEquals(tests[20:], chunk_3)
 
     def test_long(self):
         tests = make_mock_manifest(("a", 100), ("a/b", 1), ("c", 1))
 
-        chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests))
-        chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests))
-        chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests))
+        chunk_1 = list(EqualTimeChunker(3, 1)(tests))
+        chunk_2 = list(EqualTimeChunker(3, 2)(tests))
+        chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:100], chunk_1)
         self.assertEquals(tests[100:101], chunk_2)
         self.assertEquals(tests[101:102], chunk_3)
 
     def test_long_1(self):
         tests = make_mock_manifest(("a", 1), ("a/b", 100), ("c", 1))
 
-        chunk_1 = list(wptrunner.EqualTimeChunker(3, 1)(tests))
-        chunk_2 = list(wptrunner.EqualTimeChunker(3, 2)(tests))
-        chunk_3 = list(wptrunner.EqualTimeChunker(3, 3)(tests))
+        chunk_1 = list(EqualTimeChunker(3, 1)(tests))
+        chunk_2 = list(EqualTimeChunker(3, 2)(tests))
+        chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:1], chunk_1)
         self.assertEquals(tests[1:101], chunk_2)
         self.assertEquals(tests[101:102], chunk_3)
 
     def test_too_few_dirs(self):
         with self.assertRaises(ValueError):
             tests = make_mock_manifest(("a", 1), ("a/b", 100), ("c", 1))
-            list(wptrunner.EqualTimeChunker(4, 1)(tests))
+            list(EqualTimeChunker(4, 1)(tests))
 
 
 if __name__ == "__main__":
     unittest.main()
--- a/testing/web-platform/harness/wptrunner/tests/test_hosts.py
+++ b/testing/web-platform/harness/wptrunner/tests/test_hosts.py
@@ -1,19 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 import sys
+from os.path import join, dirname
 from cStringIO import StringIO
 
-sys.path.insert(0, "..")
+sys.path.insert(0, join(dirname(__file__), "..", ".."))
 
-import hosts
+from wptrunner import hosts
 
 
 class HostsTest(unittest.TestCase):
     def do_test(self, input, expected):
         host_file = hosts.HostsFile.from_file(StringIO(input))
         self.assertEquals(host_file.to_string(), expected)
 
     def test_simple(self):
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/harness/wptrunner/tests/test_testloader.py
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
+
+from mozlog import structured
+from wptrunner.testloader import TestFilter as Filter
+from .test_chunker import make_mock_manifest
+
+structured.set_default_logger(structured.structuredlog.StructuredLogger("TestLoader"))
+
+include_ini = """\
+skip: true
+[test_\u53F0]
+  skip: false
+"""
+
+def test_filter_unicode():
+    tests = make_mock_manifest(("a", 10), ("a/b", 10), ("c", 10))
+
+    with tempfile.NamedTemporaryFile("wb", suffix=".ini") as f:
+        f.write(include_ini.encode('utf-8'))
+        f.flush()
+
+        Filter(manifest_path=f.name, test_manifests=tests)
--- a/testing/web-platform/harness/wptrunner/tests/test_update.py
+++ b/testing/web-platform/harness/wptrunner/tests/test_update.py
@@ -1,15 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import unittest
 import StringIO
 
+import pytest
+
 from .. import metadata, manifestupdate
 from mozlog import structuredlog, handlers, formatters
 
 
 class TestExpectedUpdater(unittest.TestCase):
     def create_manifest(self, data, test_path="path/to/test.ini"):
         f = StringIO.StringIO(data)
         return manifestupdate.compile(f, test_path)
@@ -46,16 +48,17 @@ class TestExpectedUpdater(unittest.TestC
 
     def coalesce_results(self, trees):
         for tree in trees:
             for test in tree.iterchildren():
                 for subtest in test.iterchildren():
                     subtest.coalesce_expected()
                 test.coalesce_expected()
 
+    @pytest.mark.xfail
     def test_update_0(self):
         prev_data = [("path/to/test.htm.ini", ["/path/to/test.htm"], """[test.htm]
   type: testharness
   [test1]
     expected: FAIL""")]
 
         new_data = self.create_log(("test_start", {"test": "/path/to/test.htm"}),
                                    ("test_status", {"test": "/path/to/test.htm",
@@ -66,16 +69,17 @@ class TestExpectedUpdater(unittest.TestC
                                                  "status": "OK"}))
         updater = self.create_updater(prev_data)
         updater.update_from_log(new_data)
 
         new_manifest = updater.expected_tree["path/to/test.htm.ini"]
         self.coalesce_results([new_manifest])
         self.assertTrue(new_manifest.is_empty)
 
+    @pytest.mark.xfail
     def test_update_1(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected: ERROR""")]
 
         new_data = self.create_log(("test_start", {"test": test_id}),
@@ -88,16 +92,17 @@ class TestExpectedUpdater(unittest.TestC
         updater = self.create_updater(prev_data)
         updater.update_from_log(new_data)
 
         new_manifest = updater.expected_tree["path/to/test.htm.ini"]
         self.coalesce_results([new_manifest])
         self.assertFalse(new_manifest.is_empty)
         self.assertEquals(new_manifest.get_test(test_id).children[0].get("expected"), "FAIL")
 
+    @pytest.mark.xfail
     def test_new_subtest(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected: FAIL""")]
 
         new_data = self.create_log(("test_start", {"test": test_id}),
@@ -115,16 +120,17 @@ class TestExpectedUpdater(unittest.TestC
         updater.update_from_log(new_data)
 
         new_manifest = updater.expected_tree["path/to/test.htm.ini"]
         self.coalesce_results([new_manifest])
         self.assertFalse(new_manifest.is_empty)
         self.assertEquals(new_manifest.get_test(test_id).children[0].get("expected"), "FAIL")
         self.assertEquals(new_manifest.get_test(test_id).children[1].get("expected"), "FAIL")
 
+    @pytest.mark.xfail
     def test_update_multiple_0(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected: FAIL""")]
 
         new_data_0 = self.create_log(("test_start", {"test": test_id}),
@@ -154,16 +160,17 @@ class TestExpectedUpdater(unittest.TestC
         self.coalesce_results([new_manifest])
 
         self.assertFalse(new_manifest.is_empty)
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "osx"}), "FAIL")
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "linux"}), "TIMEOUT")
 
+    @pytest.mark.xfail
     def test_update_multiple_1(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected: FAIL""")]
 
         new_data_0 = self.create_log(("test_start", {"test": test_id}),
@@ -195,16 +202,17 @@ class TestExpectedUpdater(unittest.TestC
         self.assertFalse(new_manifest.is_empty)
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "osx"}), "FAIL")
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "linux"}), "TIMEOUT")
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "windows"}), "FAIL")
 
+    @pytest.mark.xfail
     def test_update_multiple_2(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected: FAIL""")]
 
         new_data_0 = self.create_log(("test_start", {"test": test_id}),
@@ -234,16 +242,17 @@ class TestExpectedUpdater(unittest.TestC
         self.coalesce_results([new_manifest])
 
         self.assertFalse(new_manifest.is_empty)
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "osx"}), "FAIL")
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": True, "os": "osx"}), "TIMEOUT")
 
+    @pytest.mark.xfail
     def test_update_multiple_3(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected:
       if debug: FAIL
       if not debug and os == "osx": TIMEOUT""")]
@@ -275,16 +284,17 @@ class TestExpectedUpdater(unittest.TestC
         self.coalesce_results([new_manifest])
 
         self.assertFalse(new_manifest.is_empty)
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": False, "os": "osx"}), "FAIL")
         self.assertEquals(new_manifest.get_test(test_id).children[0].get(
             "expected", {"debug": True, "os": "osx"}), "TIMEOUT")
 
+    @pytest.mark.xfail
     def test_update_ignore_existing(self):
         test_id = "/path/to/test.htm"
         prev_data = [("path/to/test.htm.ini", [test_id], """[test.htm]
   type: testharness
   [test1]
     expected:
       if debug: TIMEOUT
       if not debug and os == "osx": NOTRUN""")]
--- a/testing/web-platform/harness/wptrunner/webdriver_server.py
+++ b/testing/web-platform/harness/wptrunner/webdriver_server.py
@@ -11,17 +11,18 @@ import threading
 import time
 import traceback
 import urlparse
 
 import mozprocess
 
 
 __all__ = ["SeleniumServer", "ChromeDriverServer",
-           "GeckoDriverServer", "WebDriverServer"]
+           "GeckoDriverServer", "ServoDriverServer",
+           "WebDriverServer"]
 
 
 class WebDriverServer(object):
     __metaclass__ = abc.ABCMeta
 
     default_base_path = "/"
     _used_ports = set()
 
@@ -39,17 +40,17 @@ class WebDriverServer(object):
         self._port = port
         self._cmd = None
         self._proc = None
 
     @abc.abstractmethod
     def make_command(self):
         """Returns the full command for starting the server process as a list."""
 
-    def start(self, block=True):
+    def start(self, block=False):
         try:
             self._run(block)
         except KeyboardInterrupt:
             self.stop()
 
     def _run(self, block):
         self._cmd = self.make_command()
         self._proc = mozprocess.ProcessHandler(
@@ -69,34 +70,29 @@ class WebDriverServer(object):
         self.logger.debug(
             "Waiting for server to become accessible: %s" % self.url)
         try:
             wait_for_service((self.host, self.port))
         except:
             self.logger.error(
                 "WebDriver HTTP server was not accessible "
                 "within the timeout:\n%s" % traceback.format_exc())
-            if self._proc.poll():
-                self.logger.error("Webdriver server process exited with code %i" %
-                                  self._proc.returncode)
             raise
 
         if block:
             self._proc.wait()
 
     def stop(self):
         if self.is_alive:
             return self._proc.kill()
         return not self.is_alive
 
     @property
     def is_alive(self):
-        return (self._proc is not None and
-                self._proc.proc is not None and
-                self._proc.poll() is None)
+        return hasattr(self._proc, "proc") and self._proc.poll() is None
 
     def on_output(self, line):
         self.logger.process_output(self.pid,
                                    line.decode("utf8", "replace"),
                                    command=" ".join(self._cmd))
 
     @property
     def pid(self):
@@ -136,32 +132,65 @@ class ChromeDriverServer(WebDriverServer
             self, logger, binary, port=port, base_path=base_path)
 
     def make_command(self):
         return [self.binary,
                 cmd_arg("port", str(self.port)),
                 cmd_arg("url-base", self.base_path) if self.base_path else ""]
 
 
+class EdgeDriverServer(WebDriverServer):
+    def __init__(self, logger, binary="MicrosoftWebDriver.exe", port=None,
+                 base_path="", host="localhost"):
+        WebDriverServer.__init__(
+            self, logger, binary, host=host, port=port)
+
+    def make_command(self):
+        return [self.binary,
+                "--port=%s" % str(self.port)]
+
+
 class GeckoDriverServer(WebDriverServer):
     def __init__(self, logger, marionette_port=2828, binary="wires",
                  host="127.0.0.1", port=None):
         env = os.environ.copy()
         env["RUST_BACKTRACE"] = "1"
         WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env)
         self.marionette_port = marionette_port
 
     def make_command(self):
         return [self.binary,
                 "--connect-existing",
                 "--marionette-port", str(self.marionette_port),
                 "--host", self.host,
                 "--port", str(self.port)]
 
 
+class ServoDriverServer(WebDriverServer):
+    def __init__(self, logger, binary="servo", binary_args=None, host="127.0.0.1", port=None, render_backend=None):
+        env = os.environ.copy()
+        env["RUST_BACKTRACE"] = "1"
+        WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env)
+        self.binary_args = binary_args
+        self.render_backend = render_backend
+
+    def make_command(self):
+        command = [self.binary,
+                   "--webdriver", str(self.port),
+                   "--hard-fail",
+                   "--headless"]
+        if self.binary_args:
+            command += self.binary_args
+        if self.render_backend == "cpu":
+            command += ["--cpu"]
+        elif self.render_backend == "webrender":
+            command += ["--webrender"]
+        return command
+
+
 def cmd_arg(name, value=None):
     prefix = "-" if platform.system() == "Windows" else "--"
     rv = prefix + name
     if value is not None:
         rv += "=" + value
     return rv
 
 
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -97,16 +97,19 @@ scheme host and port.""")
                                  help="Run tests in a loop until one returns an unexpected result")
     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
                                  help="Halt the test runner after each test (this happens by default if only a single test is run)")
     debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
                                  help="Don't halt the test runner irrespective of the number of tests run")
 
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
                                  help="Halt the test runner when an unexpected result is encountered")
+    debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected",
+                                 default=True, action="store_false",
+                                 help="Don't restart on an unexpected result")
 
     debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
                                  help="Path or url to symbols file used to analyse crash minidumps.")
     debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
                                  help="Path to stackwalker program used to analyse minidumps.")
 
     debugging_group.add_argument("--pdb", action="store_true",
                                  help="Drop into pdb on python exception")
@@ -168,26 +171,22 @@ scheme host and port.""")
     gecko_group = parser.add_argument_group("Gecko-specific")
     gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
                              help="Path to the folder containing browser prefs")
     gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
                              help="Run tests without electrolysis preferences")
     gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
                              help="Path to directory containing assertion stack fixing scripts")
 
-    b2g_group = parser.add_argument_group("B2G-specific")
-    b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False,
-                           help="Don't backup device before testrun with --product=b2g")
-
     servo_group = parser.add_argument_group("Servo-specific")
     servo_group.add_argument("--user-stylesheet",
                              default=[], action="append", dest="user_stylesheets",
                              help="Inject a user CSS stylesheet into every test.")
     servo_group.add_argument("--servo-backend",
-                             default="cpu", choices=["cpu", "webrender"],
+                             default="webrender", choices=["cpu", "webrender"],
                              help="Rendering backend to use with Servo.")
 
 
     parser.add_argument("test_list", nargs="*",
                         help="List of URLs for tests to run, or paths including tests to run. "
                              "(equivalent to --include)")
 
     commandline.add_logging_group(parser)
--- a/testing/web-platform/harness/wptrunner/wptmanifest/parser.py
+++ b/testing/web-platform/harness/wptrunner/wptmanifest/parser.py
@@ -11,16 +11,18 @@
 #  [subtest 1]
 #    expected:
 #      os == win: FAIL #This is a comment
 #      PASS
 #
 
 # TODO: keep comments in the tree
 
+from __future__ import unicode_literals
+
 import types
 from cStringIO import StringIO
 
 from node import *
 
 
 class ParseError(Exception):
     def __init__(self, filename, line, detail):
@@ -43,18 +45,19 @@ unary_operators = ["not"]
 binary_operators = ["==", "!=", "and", "or"]
 
 operators = ["==", "!=", "not", "and", "or"]
 
 atoms = {"True": True,
          "False": False,
          "Reset": object()}
 
-def decode(byte_str):
-    return byte_str.decode("utf8")
+def decode(s):
+    assert isinstance(s, unicode)
+    return s
 
 
 def precedence(operator_node):
     return len(operators) - operators.index(operator_node.data)
 
 
 class TokenTypes(object):
     def __init__(self):
@@ -71,32 +74,35 @@ class Tokenizer(object):
     def reset(self):
         self.indent_levels = [0]
         self.state = self.line_start_state
         self.next_state = self.data_line_state
         self.line_number = 0
 
     def tokenize(self, stream):
         self.reset()
-        if type(stream) in types.StringTypes:
+        assert not isinstance(stream, unicode)
+        if isinstance(stream, str):
             stream = StringIO(stream)
         if not hasattr(stream, "name"):
             self.filename = ""
         else:
             self.filename = stream.name
 
         self.next_line_state = self.line_start_state
         for i, line in enumerate(stream):
+            assert isinstance(line, str)
             self.state = self.next_line_state
             assert self.state is not None
             states = []
             self.next_line_state = None
             self.line_number = i + 1
             self.index = 0
-            self.line = line.rstrip()
+            self.line = line.decode('utf-8').rstrip()
+            assert isinstance(self.line, unicode)
             while self.state != self.eol_state:
                 states.append(self.state)
                 tokens = self.state()
                 if tokens:
                     for token in tokens:
                         yield token
             self.state()
         while True:
@@ -469,17 +475,17 @@ class Tokenizer(object):
     def decode_escape(self, length):
         value = 0
         for i in xrange(length):
             c = self.char()
             value *= 16
             value += self.escape_value(c)
             self.consume()
 
-        return unichr(value).encode("utf8")
+        return unichr(value)
 
     def escape_value(self, c):
         if '0' <= c <= '9':
             return ord(c) - ord('0')
         elif 'a' <= c <= 'f':
             return ord(c) - ord('a') + 10
         elif 'A' <= c <= 'F':
             return ord(c) - ord('A') + 10
--- a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py
+++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py
@@ -6,19 +6,16 @@ import unittest
 
 from cStringIO import StringIO
 
 from ..backends import conditional
 from ..node import BinaryExpressionNode, BinaryOperatorNode, VariableNode, NumberNode
 
 
 class TestConditional(unittest.TestCase):
-    def parse(self, input_str):
-        return self.parser.parse(StringIO(input_str))
-
     def compile(self, input_text):
         return conditional.compile(input_text)
 
     def test_get_0(self):
         data = """
 key: value
 
 [Heading 1]
--- a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py
+++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py
@@ -1,16 +1,19 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import sys
 import unittest
 
 from cStringIO import StringIO
 
+import pytest
+
 from .. import parser, serializer
 
 
 class TokenizerTest(unittest.TestCase):
     def setUp(self):
         self.serializer = serializer.ManifestSerializer()
         self.parser = parser.Parser()
 
@@ -191,16 +194,17 @@ class TokenizerTest(unittest.TestCase):
                      r"""key: [",]#"]
 """)
 
     def test_escape_8(self):
         self.compare(r"""key: \#""",
                      r"""key: "#"
 """)
 
+    @pytest.mark.xfail(sys.maxunicode == 0xFFFF, reason="narrow unicode")
     def test_escape_9(self):
         self.compare(r"""key: \U10FFFFabc""",
                      u"""key: \U0010FFFFabc
 """)
 
     def test_escape_10(self):
         self.compare(r"""key: \u10FFab""",
                      u"""key: \u10FFab
--- a/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py
+++ b/testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py
@@ -8,19 +8,16 @@ from cStringIO import StringIO
 
 from ..backends import static
 
 # There aren't many tests here because it turns out to be way more convenient to
 # use test_serializer for the majority of cases
 
 
 class TestStatic(unittest.TestCase):
-    def parse(self, input_str):
-        return self.parser.parse(StringIO(input_str))
-
     def compile(self, input_text, input_data):
         return static.compile(input_text, input_data)
 
     def test_get_0(self):
         data = """
 key: value
 
 [Heading 1]
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -199,16 +199,17 @@ def run_tests(config, test_paths, produc
                                       test_source_cls,
                                       test_source_kwargs,
                                       browser_cls,
                                       browser_kwargs,
                                       executor_cls,
                                       executor_kwargs,
                                       kwargs["pause_after_test"],
                                       kwargs["pause_on_unexpected"],
+                                      kwargs["restart_on_unexpected"],
                                       kwargs["debug_info"]) as manager_group:
                         try:
                             manager_group.run(test_type, test_loader.tests)
                         except KeyboardInterrupt:
                             logger.critical("Main thread got signal")
                             manager_group.stop()
                             raise
                     unexpected_count += manager_group.unexpected_count()
--- a/testing/web-platform/harness/wptrunner/wpttest.py
+++ b/testing/web-platform/harness/wptrunner/wpttest.py
@@ -63,20 +63,17 @@ class WdspecResult(Result):
 
 
 class WdspecSubtestResult(SubtestResult):
     default_expected = "PASS"
     statuses = set(["PASS", "FAIL", "ERROR"])
 
 
 def get_run_info(metadata_root, product, **kwargs):
-    if product == "b2g":
-        return B2GRunInfo(metadata_root, product, **kwargs)
-    else:
-        return RunInfo(metadata_root, product, **kwargs)
+    return RunInfo(metadata_root, product, **kwargs)
 
 
 class RunInfo(dict):
     def __init__(self, metadata_root, product, debug, extras=None):
         self._update_mozinfo(metadata_root)
         self.update(mozinfo.info)
         self["product"] = product
         if debug is not None:
@@ -96,22 +93,16 @@ class RunInfo(dict):
             if path in dirs:
                 break
             dirs.add(str(path))
             path = os.path.split(path)[0]
 
         mozinfo.find_and_update_from_json(*dirs)
 
 
-class B2GRunInfo(RunInfo):
-    def __init__(self, *args, **kwargs):
-        RunInfo.__init__(self, *args, **kwargs)
-        self["os"] = "b2g"
-
-
 class Test(object):
     result_cls = None
     subtest_result_cls = None
     test_type = None
 
     def __init__(self, tests_root, url, inherit_metadata, test_metadata,
                  timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
         self.tests_root = tests_root
@@ -129,17 +120,17 @@ class Test(object):
     def from_manifest(cls, manifest_item, inherit_metadata, test_metadata):
         timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT
         protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http"
         return cls(manifest_item.source_file.tests_root,
                    manifest_item.url,
                    inherit_metadata,
                    test_metadata,
                    timeout=timeout,
-                   path=manifest_item.path,
+                   path=manifest_item.source_file.path,
                    protocol=protocol)
 
     @property
     def id(self):
         return self.url
 
     @property
     def keys(self):