Bug 1345490 - Update to latest wptrunner, a=testonly draft
authorJames Graham <james@hoppipolla.co.uk>
Wed, 08 Mar 2017 16:24:00 +0000
changeset 497534 181ce62d67db66f2f16b94833d05ec0746c8f578
parent 497533 423c03a41f487994a8c4df434c4c5e841e63d956
child 497535 e1906968685c55c8defaaf4cf5bbbb553fc80b9e
push id48933
push userbmo:james@hoppipolla.co.uk
push dateMon, 13 Mar 2017 13:53:04 +0000
reviewerstestonly
bugs1345490
milestone55.0a1
Bug 1345490 - Update to latest wptrunner, a=testonly MozReview-Commit-ID: HeIGX0kPRFx
testing/web-platform/harness/setup.py
testing/web-platform/harness/test/testdata/reftest/reftest_wait_0.html
testing/web-platform/harness/wptrunner/browsers/base.py
testing/web-platform/harness/wptrunner/browsers/chrome.py
testing/web-platform/harness/wptrunner/browsers/firefox.py
testing/web-platform/harness/wptrunner/browsers/servo.py
testing/web-platform/harness/wptrunner/browsers/servodriver.py
testing/web-platform/harness/wptrunner/environment.py
testing/web-platform/harness/wptrunner/executors/executormarionette.py
testing/web-platform/harness/wptrunner/executors/executorselenium.py
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/executors/executorservodriver.py
testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js
testing/web-platform/harness/wptrunner/tests/test_chunker.py
testing/web-platform/harness/wptrunner/tests/test_testloader.py
testing/web-platform/harness/wptrunner/update/update.py
testing/web-platform/harness/wptrunner/webdriver_server.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptrunner.py
--- a/testing/web-platform/harness/setup.py
+++ b/testing/web-platform/harness/setup.py
@@ -14,17 +14,17 @@ here = os.path.split(__file__)[0]
 PACKAGE_NAME = 'wptrunner'
 PACKAGE_VERSION = '1.14'
 
 # Dependencies
 with open(os.path.join(here, "requirements.txt")) as f:
     deps = f.read().splitlines()
 
 # Browser-specific requirements
-requirements_files = glob.glob(os.path.join(here, "requirements_*.txt"))
+requirements_files = glob.glob("requirements_*.txt")
 
 profile_dest = None
 dest_exists = False
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,
       description="Harness for running the W3C web-platform-tests against various products",
       author='Mozilla Automation and Testing Team',
--- a/testing/web-platform/harness/test/testdata/reftest/reftest_wait_0.html
+++ b/testing/web-platform/harness/test/testdata/reftest/reftest_wait_0.html
@@ -1,11 +1,13 @@
+<html class="reftest-wait">
 <title>rel=match that should fail</title>
 <link rel=match href=red.html>
 <style>
 :root {background-color:red}
 </style>
-<body class="reftest-wait">
 <script>
 setTimeout(function() {
   document.documentElement.style.backgroundColor = "green";
-  body.className = "";
-}, 2000);
\ No newline at end of file
+  document.documentElement.className = "";
+}, 2000);
+</script>
+</html>
--- a/testing/web-platform/harness/wptrunner/browsers/base.py
+++ b/testing/web-platform/harness/wptrunner/browsers/base.py
@@ -122,16 +122,19 @@ class Browser(object):
 
     def log_crash(self, process, test):
         """Return a list of dictionaries containing information about crashes that happend
         in the browser, or an empty list if no crashes occurred"""
         self.logger.crash(process, test)
 
 
 class NullBrowser(Browser):
+    def __init__(self, logger, **kwargs):
+        super(NullBrowser, self).__init__(logger)
+
     def start(self):
         """No-op browser to use in scenarios where the TestRunnerManager shouldn't
         actually own the browser process (e.g. Servo where we start one browser
         per test)"""
         pass
 
     def stop(self):
         pass
--- a/testing/web-platform/harness/wptrunner/browsers/chrome.py
+++ b/testing/web-platform/harness/wptrunner/browsers/chrome.py
@@ -30,20 +30,28 @@ def browser_kwargs(**kwargs):
 
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
                     **kwargs):
     from selenium.webdriver import DesiredCapabilities
 
     executor_kwargs = base_executor_kwargs(test_type, server_config,
                                            cache_manager, **kwargs)
     executor_kwargs["close_after_done"] = True
-    executor_kwargs["capabilities"] = dict(DesiredCapabilities.CHROME.items())
-    if kwargs["binary"] is not None:
-        executor_kwargs["capabilities"]["chromeOptions"] = {"binary": kwargs["binary"]}
-
+    capabilities = dict(DesiredCapabilities.CHROME.items())
+    capabilities.setdefault("chromeOptions", {})["prefs"] = {
+        "profile": {
+            "default_content_setting_values": {
+                "popups": 1
+            }
+        }
+    }
+    for (kwarg, capability) in [("binary", "binary"), ("binary_args", "args")]:
+        if kwargs[kwarg] is not None:
+            capabilities["chromeOptions"][capability] = kwargs[kwarg]
+    executor_kwargs["capabilities"] = capabilities
     return executor_kwargs
 
 
 def env_options():
     return {"host": "web-platform.test",
             "bind_hostname": "true"}
 
 
--- a/testing/web-platform/harness/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/harness/wptrunner/browsers/firefox.py
@@ -53,17 +53,18 @@ def browser_kwargs(**kwargs):
     return {"binary": kwargs["binary"],
             "prefs_root": kwargs["prefs_root"],
             "debug_info": kwargs["debug_info"],
             "symbols_path": kwargs["symbols_path"],
             "stackwalk_binary": kwargs["stackwalk_binary"],
             "certutil_binary": kwargs["certutil_binary"],
             "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
             "e10s": kwargs["gecko_e10s"],
-            "stackfix_dir": kwargs["stackfix_dir"]}
+            "stackfix_dir": kwargs["stackfix_dir"],
+            "binary_args": kwargs["binary_args"]}
 
 
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
                     **kwargs):
     executor_kwargs = base_executor_kwargs(test_type, server_config,
                                            cache_manager, **kwargs)
     executor_kwargs["close_after_done"] = test_type != "reftest"
     if kwargs["timeout_multiplier"] is None:
@@ -71,16 +72,23 @@ def executor_kwargs(test_type, server_co
             if run_info_data["debug"] or run_info_data.get("asan"):
                 executor_kwargs["timeout_multiplier"] = 4
             else:
                 executor_kwargs["timeout_multiplier"] = 2
         elif run_info_data["debug"] or run_info_data.get("asan"):
             executor_kwargs["timeout_multiplier"] = 3
     if test_type == "wdspec":
         executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
+        fxOptions = {}
+        if kwargs["binary"]:
+            fxOptions["binary"] = kwargs["binary"]
+        if kwargs["binary_args"]:
+            fxOptions["args"] = kwargs["binary_args"]
+        capabilities = {"moz:firefoxOptions": fxOptions}
+        executor_kwargs["capabilities"] = capabilities
     return executor_kwargs
 
 
 def env_options():
     return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
             "bind_hostname": "false",
             "certificate_domain": "web-platform.test",
@@ -96,29 +104,31 @@ def update_properties():
 
 
 class FirefoxBrowser(Browser):
     used_ports = set()
     init_timeout = 60
 
     def __init__(self, logger, binary, prefs_root, debug_info=None,
                  symbols_path=None, stackwalk_binary=None, certutil_binary=None,
-                 ca_certificate_path=None, e10s=False, stackfix_dir=None):
+                 ca_certificate_path=None, e10s=False, stackfix_dir=None,
+                 binary_args=None):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
         self.marionette_port = None
         self.runner = None
         self.debug_info = debug_info
         self.profile = None
         self.symbols_path = symbols_path
         self.stackwalk_binary = stackwalk_binary
         self.ca_certificate_path = ca_certificate_path
         self.certutil_binary = certutil_binary
         self.e10s = e10s
+        self.binary_args = binary_args
         if self.symbols_path and stackfix_dir:
             self.stack_fixer = get_stack_fixer_function(stackfix_dir,
                                                         self.symbols_path)
         else:
             self.stack_fixer = None
 
     def start(self):
         self.marionette_port = get_free_port(2828, exclude=self.used_ports)
@@ -145,17 +155,19 @@ class FirefoxBrowser(Browser):
         # Bug 1262954: winxp + e10s, disable hwaccel
         if (self.e10s and platform.system() in ("Windows", "Microsoft") and
             '5.1' in platform.version()):
             self.profile.set_preferences({"layers.acceleration.disabled": True})
 
         if self.ca_certificate_path is not None:
             self.setup_ssl()
 
-        debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
+        debug_args, cmd = browser_command(self.binary,
+                                          self.binary_args if self.binary_args else [] +
+                                          [cmd_arg("marionette"), "about:blank"],
                                           self.debug_info)
 
         self.runner = FirefoxRunner(profile=self.profile,
                                     binary=cmd[0],
                                     cmdargs=cmd[1:],
                                     env=env,
                                     process_class=ProcessHandler,
                                     process_args={"processOutputLine": [self.on_output]})
--- a/testing/web-platform/harness/wptrunner/browsers/servo.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servo.py
@@ -5,39 +5,43 @@
 import os
 
 from .base import NullBrowser, ExecutorBrowser, require_arg
 from ..executors import executor_kwargs as base_executor_kwargs
 from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor
 
 here = os.path.join(os.path.split(__file__)[0])
 
-__wptrunner__ = {"product": "servo",
-                 "check_args": "check_args",
-                 "browser": "ServoBrowser",
-                 "executor": {"testharness": "ServoTestharnessExecutor",
-                              "reftest": "ServoRefTestExecutor",
-                              "wdspec": "ServoWdspecExecutor"},
-                 "browser_kwargs": "browser_kwargs",
-                 "executor_kwargs": "executor_kwargs",
-                 "env_options": "env_options",
-                 "run_info_extras": "run_info_extras",
-                 "update_properties": "update_properties"}
+__wptrunner__ = {
+    "product": "servo",
+    "check_args": "check_args",
+    "browser": "ServoBrowser",
+    "executor": {
+        "testharness": "ServoTestharnessExecutor",
+        "reftest": "ServoRefTestExecutor",
+        "wdspec": "ServoWdspecExecutor",
+    },
+    "browser_kwargs": "browser_kwargs",
+    "executor_kwargs": "executor_kwargs",
+    "env_options": "env_options",
+    "update_properties": "update_properties",
+}
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
 
 
 def browser_kwargs(**kwargs):
-    return {"binary": kwargs["binary"],
-            "debug_info": kwargs["debug_info"],
-            "binary_args": kwargs["binary_args"],
-            "user_stylesheets": kwargs.get("user_stylesheets"),
-            "render_backend": kwargs.get("servo_backend")}
+    return {
+        "binary": kwargs["binary"],
+        "debug_info": kwargs["debug_info"],
+        "binary_args": kwargs["binary_args"],
+        "user_stylesheets": kwargs.get("user_stylesheets"),
+    }
 
 
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
                     **kwargs):
     rv = base_executor_kwargs(test_type, server_config,
                               cache_manager, **kwargs)
     rv["pause_after_test"] = kwargs["pause_after_test"]
     return rv
@@ -46,36 +50,28 @@ def executor_kwargs(test_type, server_co
 def env_options():
     return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
             "bind_hostname": "true",
             "testharnessreport": "testharnessreport-servo.js",
             "supports_debugger": True}
 
 
-def run_info_extras(**kwargs):
-    return {"backend": kwargs["servo_backend"]}
-
-
 def update_properties():
-    return ["debug", "os", "version", "processor", "bits", "backend"], None
-
-
-def render_arg(render_backend):
-    return {"cpu": "--cpu", "webrender": "-w"}[render_backend]
+    return ["debug", "os", "version", "processor", "bits"], None
 
 
 class ServoBrowser(NullBrowser):
     def __init__(self, logger, binary, debug_info=None, binary_args=None,
-                 user_stylesheets=None, render_backend="webrender"):
+                 user_stylesheets=None):
         NullBrowser.__init__(self, logger)
         self.binary = binary
         self.debug_info = debug_info
         self.binary_args = binary_args or []
         self.user_stylesheets = user_stylesheets or []
-        self.render_backend = render_backend
 
     def executor_browser(self):
-        return ExecutorBrowser, {"binary": self.binary,
-                                 "debug_info": self.debug_info,
-                                 "binary_args": self.binary_args,
-                                 "user_stylesheets": self.user_stylesheets,
-                                 "render_backend": self.render_backend}
+        return ExecutorBrowser, {
+            "binary": self.binary,
+            "debug_info": self.debug_info,
+            "binary_args": self.binary_args,
+            "user_stylesheets": self.user_stylesheets,
+        }
--- a/testing/web-platform/harness/wptrunner/browsers/servodriver.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servodriver.py
@@ -4,112 +4,114 @@
 
 import os
 import subprocess
 import tempfile
 
 from mozprocess import ProcessHandler
 
 from .base import Browser, require_arg, get_free_port, browser_command, ExecutorBrowser
-from .servo import render_arg
 from ..executors import executor_kwargs as base_executor_kwargs
 from ..executors.executorservodriver import (ServoWebDriverTestharnessExecutor,
                                              ServoWebDriverRefTestExecutor)
 
 here = os.path.join(os.path.split(__file__)[0])
 
-__wptrunner__ = {"product": "servodriver",
-                 "check_args": "check_args",
-                 "browser": "ServoWebDriverBrowser",
-                 "executor": {"testharness": "ServoWebDriverTestharnessExecutor",
-                              "reftest": "ServoWebDriverRefTestExecutor"},
-                 "browser_kwargs": "browser_kwargs",
-                 "executor_kwargs": "executor_kwargs",
-                 "env_options": "env_options",
-                 "run_info_extras": "run_info_extras",
-                 "update_properties": "update_properties"}
+__wptrunner__ = {
+    "product": "servodriver",
+    "check_args": "check_args",
+    "browser": "ServoWebDriverBrowser",
+    "executor": {
+        "testharness": "ServoWebDriverTestharnessExecutor",
+        "reftest": "ServoWebDriverRefTestExecutor",
+    },
+    "browser_kwargs": "browser_kwargs",
+    "executor_kwargs": "executor_kwargs",
+    "env_options": "env_options",
+    "update_properties": "update_properties",
+}
 
 hosts_text = """127.0.0.1 web-platform.test
 127.0.0.1 www.web-platform.test
 127.0.0.1 www1.web-platform.test
 127.0.0.1 www2.web-platform.test
 127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test
 127.0.0.1 xn--lve-6lad.web-platform.test
 """
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
 
 
 def browser_kwargs(**kwargs):
-    return {"binary": kwargs["binary"],
-            "debug_info": kwargs["debug_info"],
-            "user_stylesheets": kwargs.get("user_stylesheets"),
-            "render_backend": kwargs.get("servo_backend")}
+    return {
+        "binary": kwargs["binary"],
+        "debug_info": kwargs["debug_info"],
+        "user_stylesheets": kwargs.get("user_stylesheets"),
+    }
 
 
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data, **kwargs):
     rv = base_executor_kwargs(test_type, server_config,
                               cache_manager, **kwargs)
     return rv
 
 
 def env_options():
     return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
             "bind_hostname": "true",
             "testharnessreport": "testharnessreport-servodriver.js",
             "supports_debugger": True}
 
 
-def run_info_extras(**kwargs):
-    return {"backend": kwargs["servo_backend"]}
-
-
 def update_properties():
-    return ["debug", "os", "version", "processor", "bits", "backend"], None
+    return ["debug", "os", "version", "processor", "bits"], None
 
 
 def make_hosts_file():
     hosts_fd, hosts_path = tempfile.mkstemp()
     with os.fdopen(hosts_fd, "w") as f:
         f.write(hosts_text)
     return hosts_path
 
 
 class ServoWebDriverBrowser(Browser):
     used_ports = set()
 
     def __init__(self, logger, binary, debug_info=None, webdriver_host="127.0.0.1",
-                 user_stylesheets=None, render_backend="webrender"):
+                 user_stylesheets=None):
         Browser.__init__(self, logger)
         self.binary = binary
         self.webdriver_host = webdriver_host
         self.webdriver_port = None
         self.proc = None
         self.debug_info = debug_info
         self.hosts_path = make_hosts_file()
         self.command = None
         self.user_stylesheets = user_stylesheets if user_stylesheets else []
-        self.render_backend = render_backend
 
     def start(self):
         self.webdriver_port = get_free_port(4444, exclude=self.used_ports)
         self.used_ports.add(self.webdriver_port)
 
         env = os.environ.copy()
         env["HOST_FILE"] = self.hosts_path
         env["RUST_BACKTRACE"] = "1"
 
-        debug_args, command = browser_command(self.binary,
-                                              [render_arg(self.render_backend), "--hard-fail",
-                                               "--webdriver", str(self.webdriver_port),
-                                               "about:blank"],
-                                              self.debug_info)
+        debug_args, command = browser_command(
+            self.binary,
+            [
+                "--hard-fail",
+                "--webdriver", str(self.webdriver_port),
+                "about:blank",
+            ],
+            self.debug_info
+        )
 
         for stylesheet in self.user_stylesheets:
             command += ["--user-stylesheet", stylesheet]
 
         self.command = command
 
         self.command = debug_args + self.command
 
--- a/testing/web-platform/harness/wptrunner/environment.py
+++ b/testing/web-platform/harness/wptrunner/environment.py
@@ -34,16 +34,18 @@ def do_delayed_imports(logger, test_path
     serve_root = serve_path(test_paths)
     sys.path.insert(0, serve_root)
 
     failed = []
 
     try:
         from tools.serve import serve
     except ImportError:
+        from wpt_tools.serve import serve
+    except ImportError:
         failed.append("serve")
 
     try:
         import sslutils
     except ImportError:
         failed.append("sslutils")
 
     if failed:
--- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -13,17 +13,16 @@ import urlparse
 import uuid
 from collections import defaultdict
 
 from ..wpttest import WdspecResult, WdspecSubtestResult
 
 errors = None
 marionette = None
 pytestrunner = None
-webdriver = None
 
 here = os.path.join(os.path.split(__file__)[0])
 
 from .base import (ExecutorException,
                    Protocol,
                    RefTestExecutor,
                    RefTestImplementation,
                    TestExecutor,
@@ -265,46 +264,39 @@ class MarionetteProtocol(Protocol):
         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
             self.marionette.execute_script(script)
 
 
 class RemoteMarionetteProtocol(Protocol):
     def __init__(self, executor, browser):
         do_delayed_imports()
         Protocol.__init__(self, executor, browser)
-        self.session = None
         self.webdriver_binary = executor.webdriver_binary
-        self.marionette_port = browser.marionette_port
+        self.capabilities = self.executor.capabilities
+        self.session_config = None
         self.server = None
 
     def setup(self, runner):
         """Connect to browser via the Marionette HTTP server."""
         try:
             self.server = GeckoDriverServer(
-                self.logger, self.marionette_port, binary=self.webdriver_binary)
+                self.logger, binary=self.webdriver_binary)
             self.server.start(block=False)
             self.logger.info(
                 "WebDriver HTTP server listening at %s" % self.server.url)
-
-            self.logger.info(
-                "Establishing new WebDriver session with %s" % self.server.url)
-            self.session = webdriver.Session(
-                self.server.host, self.server.port, self.server.base_path)
+            self.session_config = {"host": self.server.host,
+                                   "port": self.server.port,
+                                   "capabilities": self.capabilities}
         except Exception:
             self.logger.error(traceback.format_exc())
             self.executor.runner.send_message("init_failed")
         else:
             self.executor.runner.send_message("init_succeeded")
 
     def teardown(self):
-        try:
-            if self.session.session_id is not None:
-                self.session.end()
-        except Exception:
-            pass
         if self.server is not None and self.server.is_alive:
             self.server.stop()
 
     @property
     def is_alive(self):
         """Test that the Marionette connection is still alive.
 
         Because the remote communication happens over HTTP we need to
@@ -555,45 +547,48 @@ class WdspecRun(object):
             message += traceback.format_exc(e)
             self.result = False, ("ERROR", message)
         finally:
             self.result_flag.set()
 
 
 class MarionetteWdspecExecutor(WdspecExecutor):
     def __init__(self, browser, server_config, webdriver_binary,
-                 timeout_multiplier=1, close_after_done=True, debug_info=None):
+                 timeout_multiplier=1, close_after_done=True, debug_info=None,
+                 capabilities=None):
         self.do_delayed_imports()
         WdspecExecutor.__init__(self, browser, server_config,
                                 timeout_multiplier=timeout_multiplier,
                                 debug_info=debug_info)
         self.webdriver_binary = webdriver_binary
+        self.capabilities = capabilities
         self.protocol = RemoteMarionetteProtocol(self, browser)
 
     def is_alive(self):
         return self.protocol.is_alive
 
     def on_environment_change(self, new_environment):
         pass
 
     def do_test(self, test):
         timeout = test.timeout * self.timeout_multiplier + extra_timeout
 
         success, data = WdspecRun(self.do_wdspec,
-                                  self.protocol.session,
+                                  self.protocol.session_config,
                                   test.abs_path,
                                   timeout).run()
 
         if success:
             return self.convert_result(test, data)
 
         return (test.result_cls(*data), [])
 
-    def do_wdspec(self, session, path, timeout):
+    def do_wdspec(self, session_config, path, timeout):
         harness_result = ("OK", None)
-        subtest_results = pytestrunner.run(
-            path, session, self.server_url, timeout=timeout)
+        subtest_results = pytestrunner.run(path,
+                                           self.server_config,
+                                           session_config,
+                                           timeout=timeout)
         return (harness_result, subtest_results)
 
     def do_delayed_imports(self):
-        global pytestrunner, webdriver
+        global pytestrunner
         from . import pytestrunner
-        import webdriver
--- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
@@ -176,18 +176,19 @@ class SeleniumTestharnessExecutor(Testha
         with open(os.path.join(here, "testharness_webdriver.js")) as f:
             self.script = f.read()
         self.close_after_done = close_after_done
         self.window_id = str(uuid.uuid4())
 
     def is_alive(self):
         return self.protocol.is_alive()
 
-    def on_protocol_change(self, new_protocol):
-        self.protocol.load_runner(new_protocol)
+    def on_environment_change(self, new_environment):
+        if new_environment["protocol"] != self.last_environment["protocol"]:
+            self.protocol.load_runner(new_environment["protocol"])
 
     def do_test(self, test):
         url = self.test_url(test)
 
         success, data = SeleniumRun(self.do_testharness,
                                     self.protocol.webdriver,
                                     url,
                                     test.timeout * self.timeout_multiplier).run()
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -25,25 +25,20 @@ from .base import (ExecutorException,
                    WdspecExecutor)
 from .process import ProcessTestExecutor
 from ..browsers.base import browser_command
 from ..wpttest import WdspecResult, WdspecSubtestResult
 from ..webdriver_server import ServoDriverServer
 from .executormarionette import WdspecRun
 
 pytestrunner = None
-render_arg = None
 webdriver = None
 
 extra_timeout = 5 # seconds
 
-def do_delayed_imports():
-    global render_arg
-    from ..browsers.servo import render_arg
-
 hosts_text = """127.0.0.1 web-platform.test
 127.0.0.1 www.web-platform.test
 127.0.0.1 www1.web-platform.test
 127.0.0.1 www2.web-platform.test
 127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test
 127.0.0.1 xn--lve-6lad.web-platform.test
 """
 
@@ -75,18 +70,20 @@ class ServoTestharnessExecutor(ProcessTe
         except OSError:
             pass
         ProcessTestExecutor.teardown(self)
 
     def do_test(self, test):
         self.result_data = None
         self.result_flag = threading.Event()
 
-        args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner",
-                "-Z", "replace-surrogates", "-z", self.test_url(test)]
+        args = [
+            "--hard-fail", "-u", "Servo/wptrunner",
+            "-Z", "replace-surrogates", "-z", self.test_url(test),
+        ]
         for stylesheet in self.browser.user_stylesheets:
             args += ["--user-stylesheet", stylesheet]
         for pref, value in test.environment.get('prefs', {}).iteritems():
             args += ["--pref", "%s=%s" % (pref, value)]
         args += self.browser.binary_args
         debug_args, command = browser_command(self.binary, args, self.debug_info)
 
         self.command = command
@@ -208,19 +205,22 @@ class ServoRefTestExecutor(ProcessTestEx
         ProcessTestExecutor.teardown(self)
 
     def screenshot(self, test, viewport_size, dpi):
         full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
             debug_args, command = browser_command(
                 self.binary,
-                [render_arg(self.browser.render_backend), "--hard-fail", "--exit",
-                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
-                 "--output=%s" % output_path, full_url] + self.browser.binary_args,
+                [
+                    "--hard-fail", "--exit",
+                    "-u", "Servo/wptrunner",
+                    "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
+                    "--output=%s" % output_path, full_url
+                ] + self.browser.binary_args,
                 self.debug_info)
 
             for stylesheet in self.browser.user_stylesheets:
                 command += ["--user-stylesheet", stylesheet]
 
             for pref, value in test.environment.get('prefs', {}).iteritems():
                 command += ["--pref", "%s=%s" % (pref, value)]
 
@@ -290,17 +290,17 @@ class ServoWdspecProtocol(Protocol):
     def __init__(self, executor, browser):
         self.do_delayed_imports()
         Protocol.__init__(self, executor, browser)
         self.session = None
         self.server = None
 
     def setup(self, runner):
         try:
-            self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend)
+            self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args)
             self.server.start(block=False)
             self.logger.info(
                 "WebDriver HTTP server listening at %s" % self.server.url)
 
             self.logger.info(
                 "Establishing new WebDriver session with %s" % self.server.url)
             self.session = webdriver.Session(
                 self.server.host, self.server.port, self.server.base_path)
--- a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
@@ -20,17 +20,17 @@ webdriver = None
 
 here = os.path.join(os.path.split(__file__)[0])
 
 extra_timeout = 5
 
 
 def do_delayed_imports():
     global webdriver
-    from tools import webdriver
+    import webdriver
 
 
 class ServoWebDriverProtocol(Protocol):
     def __init__(self, executor, browser, capabilities, **kwargs):
         do_delayed_imports()
         Protocol.__init__(self, executor, browser)
         self.capabilities = capabilities
         self.host = browser.webdriver_host
--- a/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
+++ b/testing/web-platform/harness/wptrunner/executors/pytestrunner/runner.py
@@ -8,53 +8,57 @@ Usage::
 
     session = webdriver.client.Session("127.0.0.1", "4444", "/")
     harness_result = ("OK", None)
     subtest_results = pytestrunner.run("/path/to/test", session.url)
     return (harness_result, subtest_results)
 """
 
 import errno
+import json
+import os
 import shutil
 import tempfile
 
 from . import fixtures
 
 
 pytest = None
 
 
 def do_delayed_imports():
     global pytest
     import pytest
 
 
-def run(path, session, url_getter, timeout=0):
+def run(path, server_config, session_config, timeout=0):
     """Run Python test at ``path`` in pytest.  The provided ``session``
     is exposed as a fixture available in the scope of the test functions.
 
     :param path: Path to the test file.
-    :param session: WebDriver session to expose.
-    :param url_getter: Function to get server url from test environment, given
-        a protocol.
+    :param session_config: dictionary of host, port,capabilities parameters
+    to pass through to the webdriver session
     :param timeout: Duration before interrupting potentially hanging
         tests.  If 0, there is no timeout.
 
     :returns: List of subtest results, which are tuples of (test id,
         status, message, stacktrace).
     """
 
     if pytest is None:
         do_delayed_imports()
 
     recorder = SubtestResultRecorder()
-    plugins = [recorder,
-               fixtures,
-               fixtures.Session(session),
-               fixtures.Server(url_getter)]
+
+    os.environ["WD_HOST"] = session_config["host"]
+    os.environ["WD_PORT"] = str(session_config["port"])
+    os.environ["WD_CAPABILITIES"] = json.dumps(session_config["capabilities"])
+    os.environ["WD_SERVER_CONFIG"] = json.dumps(server_config)
+
+    plugins = [recorder]
 
     # TODO(ato): Deal with timeouts
 
     with TemporaryDirectory() as cache:
         pytest.main(["--strict",  # turn warnings into errors
                      "--verbose",  # show each individual subtest
                      "--capture", "no",  # enable stdout/stderr from tests
                      "--basetemp", cache,  # temporary directory
@@ -87,17 +91,19 @@ class SubtestResultRecorder(object):
 
     def record_error(self, report):
         # error in setup/teardown
         if report.when != "call":
             message = "%s error" % report.when
         self.record(report.nodeid, "ERROR", message, report.longrepr)
 
     def record_skip(self, report):
-        self.record(report.nodeid, "PASS")
+        self.record(report.nodeid, "ERROR",
+                    "In-test skip decorators are disallowed, "
+                    "please use WPT metadata to ignore tests.")
 
     def record(self, test, status, message=None, stack=None):
         if stack is not None:
             stack = str(stack)
         new_result = (test, status, message, stack)
         self.results.append(new_result)
 
 
--- a/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js
+++ b/testing/web-platform/harness/wptrunner/executors/reftest-wait_servodriver.js
@@ -1,17 +1,17 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
 callback = arguments[arguments.length - 1];
 
 function check_done() {
-    if (!document.body.classList.contains('reftest-wait')) {
+    if (!document.documentElement.classList.contains('reftest-wait')) {
         callback();
     } else {
         setTimeout(check_done, 50);
     }
 }
 
 if (document.readyState === 'complete') {
     check_done();
--- a/testing/web-platform/harness/wptrunner/tests/test_chunker.py
+++ b/testing/web-platform/harness/wptrunner/tests/test_chunker.py
@@ -19,68 +19,75 @@ class MockTest(object):
     def __init__(self, id, timeout=10):
         self.id = id
         self.item_type = "testharness"
         self.timeout = timeout
 
 
 def make_mock_manifest(*items):
     rv = []
-    for dir_path, num_tests in items:
+    for test_type, dir_path, num_tests in items:
         for i in range(num_tests):
-            rv.append((dir_path + "/%i.test" % i, set([MockTest(i)])))
+            rv.append((test_type,
+                       dir_path + "/%i.test" % i,
+                       set([MockTest(i)])))
     return rv
 
 
 class TestEqualTimeChunker(unittest.TestCase):
 
     def test_include_all(self):
-        tests = make_mock_manifest(("a", 10), ("a/b", 10), ("c", 10))
+        tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
+                                   ("test", "c", 10))
 
         chunk_1 = list(EqualTimeChunker(3, 1)(tests))
         chunk_2 = list(EqualTimeChunker(3, 2)(tests))
         chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:10], chunk_1)
         self.assertEquals(tests[10:20], chunk_2)
         self.assertEquals(tests[20:], chunk_3)
 
     def test_include_all_1(self):
-        tests = make_mock_manifest(("a", 5), ("a/b", 5), ("c", 10), ("d", 10))
+        tests = make_mock_manifest(("test", "a", 5), ("test", "a/b", 5),
+                                   ("test", "c", 10), ("test", "d", 10))
 
         chunk_1 = list(EqualTimeChunker(3, 1)(tests))
         chunk_2 = list(EqualTimeChunker(3, 2)(tests))
         chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:10], chunk_1)
         self.assertEquals(tests[10:20], chunk_2)
         self.assertEquals(tests[20:], chunk_3)
 
     def test_long(self):
-        tests = make_mock_manifest(("a", 100), ("a/b", 1), ("c", 1))
+        tests = make_mock_manifest(("test", "a", 100), ("test", "a/b", 1),
+                                   ("test", "c", 1))
 
         chunk_1 = list(EqualTimeChunker(3, 1)(tests))
         chunk_2 = list(EqualTimeChunker(3, 2)(tests))
         chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:100], chunk_1)
         self.assertEquals(tests[100:101], chunk_2)
         self.assertEquals(tests[101:102], chunk_3)
 
     def test_long_1(self):
-        tests = make_mock_manifest(("a", 1), ("a/b", 100), ("c", 1))
+        tests = make_mock_manifest(("test", "a", 1), ("test", "a/b", 100),
+                                   ("test", "c", 1))
 
         chunk_1 = list(EqualTimeChunker(3, 1)(tests))
         chunk_2 = list(EqualTimeChunker(3, 2)(tests))
         chunk_3 = list(EqualTimeChunker(3, 3)(tests))
 
         self.assertEquals(tests[:1], chunk_1)
         self.assertEquals(tests[1:101], chunk_2)
         self.assertEquals(tests[101:102], chunk_3)
 
     def test_too_few_dirs(self):
         with self.assertRaises(ValueError):
-            tests = make_mock_manifest(("a", 1), ("a/b", 100), ("c", 1))
+            tests = make_mock_manifest(("test", "a", 1), ("test", "a/b", 100),
+                                       ("test", "c", 1))
             list(EqualTimeChunker(4, 1)(tests))
 
 
 if __name__ == "__main__":
     unittest.main()
--- a/testing/web-platform/harness/wptrunner/tests/test_testloader.py
+++ b/testing/web-platform/harness/wptrunner/tests/test_testloader.py
@@ -18,15 +18,16 @@ structured.set_default_logger(structured
 
 include_ini = """\
 skip: true
 [test_\u53F0]
   skip: false
 """
 
 def test_filter_unicode():
-    tests = make_mock_manifest(("a", 10), ("a/b", 10), ("c", 10))
+    tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
+                               ("test", "c", 10))
 
     with tempfile.NamedTemporaryFile("wb", suffix=".ini") as f:
         f.write(include_ini.encode('utf-8'))
         f.flush()
 
         Filter(manifest_path=f.name, test_manifests=tests)
--- a/testing/web-platform/harness/wptrunner/update/update.py
+++ b/testing/web-platform/harness/wptrunner/update/update.py
@@ -10,17 +10,20 @@ from sync import SyncFromUpstreamRunner
 from tree import GitTree, HgTree, NoVCSTree
 
 from .. import environment as env
 from base import Step, StepRunner, exit_clean, exit_unclean
 from state import State
 
 def setup_paths(sync_path):
     sys.path.insert(0, os.path.abspath(sync_path))
-    from tools import localpaths
+    try:
+        from tools import localpaths
+    except ImportError:
+        from wpt_tools import localpaths
 
 class LoadConfig(Step):
     """Step for loading configuration from the ini file and kwargs."""
 
     provides = ["sync", "paths", "metadata_path", "tests_path"]
 
     def create(self, state):
         state.sync = {"remote_url": state.kwargs["remote_url"],
--- a/testing/web-platform/harness/wptrunner/webdriver_server.py
+++ b/testing/web-platform/harness/wptrunner/webdriver_server.py
@@ -23,16 +23,20 @@ import mozprocess
 class WebDriverServer(object):
     __metaclass__ = abc.ABCMeta
 
     default_base_path = "/"
     _used_ports = set()
 
     def __init__(self, logger, binary, host="127.0.0.1", port=None,
                  base_path="", env=None):
+        if binary is None:
+            raise ValueError("WebDriver server binary must be given "
+                             "to --webdriver-binary argument")
+
         self.logger = logger
         self.binary = binary
         self.host = host
         if base_path == "":
             self.base_path = self.default_base_path
         else:
             self.base_path = base_path
         self.env = os.environ.copy() if env is None else env
@@ -144,50 +148,44 @@ class EdgeDriverServer(WebDriverServer):
             self, logger, binary, host=host, port=port)
 
     def make_command(self):
         return [self.binary,
                 "--port=%s" % str(self.port)]
 
 
 class GeckoDriverServer(WebDriverServer):
-    def __init__(self, logger, marionette_port=2828, binary="wires",
+    def __init__(self, logger, marionette_port=2828, binary="geckodriver",
                  host="127.0.0.1", port=None):
         env = os.environ.copy()
         env["RUST_BACKTRACE"] = "1"
         WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env)
         self.marionette_port = marionette_port
 
     def make_command(self):
         return [self.binary,
-                "--connect-existing",
                 "--marionette-port", str(self.marionette_port),
                 "--host", self.host,
                 "--port", str(self.port)]
 
 
 class ServoDriverServer(WebDriverServer):
-    def __init__(self, logger, binary="servo", binary_args=None, host="127.0.0.1", port=None, render_backend=None):
+    def __init__(self, logger, binary="servo", binary_args=None, host="127.0.0.1", port=None):
         env = os.environ.copy()
         env["RUST_BACKTRACE"] = "1"
         WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env)
         self.binary_args = binary_args
-        self.render_backend = render_backend
 
     def make_command(self):
         command = [self.binary,
                    "--webdriver", str(self.port),
                    "--hard-fail",
                    "--headless"]
         if self.binary_args:
             command += self.binary_args
-        if self.render_backend == "cpu":
-            command += ["--cpu"]
-        elif self.render_backend == "webrender":
-            command += ["--webrender"]
         return command
 
 
 def cmd_arg(name, value=None):
     prefix = "-" if platform.system() == "Windows" else "--"
     rv = prefix + name
     if value is not None:
         rv += "=" + value
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -114,17 +114,17 @@ scheme host and port.""")
     debugging_group.add_argument("--pdb", action="store_true",
                                  help="Drop into pdb on python exception")
 
     config_group = parser.add_argument_group("Configuration")
     config_group.add_argument("--binary", action="store",
                               type=abs_path, help="Binary to run tests against")
     config_group.add_argument('--binary-arg',
                               default=[], action="append", dest="binary_args",
-                              help="Extra argument for the binary (servo)")
+                              help="Extra argument for the binary")
     config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY",
                               type=abs_path, help="WebDriver server binary to use")
 
     config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
                               help="Path to root directory containing test metadata"),
     config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
                               help="Path to root directory containing test files"),
     config_group.add_argument("--run-info", action="store", type=abs_path,
@@ -175,19 +175,16 @@ scheme host and port.""")
                              help="Run tests without electrolysis preferences")
     gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
                              help="Path to directory containing assertion stack fixing scripts")
 
     servo_group = parser.add_argument_group("Servo-specific")
     servo_group.add_argument("--user-stylesheet",
                              default=[], action="append", dest="user_stylesheets",
                              help="Inject a user CSS stylesheet into every test.")
-    servo_group.add_argument("--servo-backend",
-                             default="webrender", choices=["cpu", "webrender"],
-                             help="Rendering backend to use with Servo.")
 
 
     parser.add_argument("test_list", nargs="*",
                         help="List of URLs for tests to run, or paths including tests to run. "
                              "(equivalent to --include)")
 
     commandline.add_logging_group(parser)
     return parser
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -10,16 +10,17 @@ import sys
 
 import environment as env
 import products
 import testloader
 import wptcommandline
 import wptlogging
 import wpttest
 from testrunner import ManagerGroup
+from browsers.base import NullBrowser
 
 here = os.path.split(__file__)[0]
 
 logger = None
 
 """Runner for web-platform-tests
 
 The runner has several design goals:
@@ -110,17 +111,17 @@ def get_pause_after_test(test_loader, **
     return kwargs["pause_after_test"]
 
 
 def run_tests(config, test_paths, product, **kwargs):
     with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
         env.do_delayed_imports(logger, test_paths)
 
         (check_args,
-         browser_cls, get_browser_kwargs,
+         target_browser_cls, get_browser_kwargs,
          executor_classes, get_executor_kwargs,
          env_options, run_info_extras) = products.load_product(config, product)
 
         ssl_env = env.ssl_env(logger, **kwargs)
 
         check_args(**kwargs)
 
         if "test_loader" in kwargs:
@@ -172,16 +173,26 @@ def run_tests(config, test_paths, produc
                 elif repeat > 1:
                     logger.info("Repetition %i / %i" % (repeat_count, repeat))
 
                 unexpected_count = 0
                 logger.suite_start(test_loader.test_ids, run_info)
                 for test_type in kwargs["test_types"]:
                     logger.info("Running %s tests" % test_type)
 
+                    # WebDriver tests may create and destroy multiple browser
+                    # processes as part of their expected behavior. These
+                    # processes are managed by a WebDriver server binary. This
+                    # obviates the need for wptrunner to provide a browser, so
+                    # the NullBrowser is used in place of the "target" browser
+                    if test_type == "wdspec":
+                        browser_cls = NullBrowser
+                    else:
+                        browser_cls = target_browser_cls
+
                     for test in test_loader.disabled_tests[test_type]:
                         logger.test_start(test.id)
                         logger.test_end(test.id, status="SKIP")
 
                     executor_cls = executor_classes.get(test_type)
                     executor_kwargs = get_executor_kwargs(test_type,
                                                           test_environment.external_config,
                                                           test_environment.cache_manager,