Bug 1146321 - Update to latest wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Mon, 23 Mar 2015 08:49:38 +0000
changeset 263905 948262aa93347e3aa3db143e26d435257c776d00
parent 263904 027c4d441a02eb4101f483ab5a9941d64e0ca917
child 263906 f8de9b743d15a6a50f54230fe3b2e0a475abcadb
push id4718
push userraliiev@mozilla.com
push dateMon, 11 May 2015 18:39:53 +0000
treeherdermozilla-beta@c20c4ef55f08 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1146321
milestone39.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1146321 - Update to latest wptrunner, a=testonly
testing/web-platform/harness/setup.py
testing/web-platform/harness/wptrunner/browsers/b2g.py
testing/web-platform/harness/wptrunner/browsers/chrome.py
testing/web-platform/harness/wptrunner/browsers/firefox.py
testing/web-platform/harness/wptrunner/browsers/servo.py
testing/web-platform/harness/wptrunner/environment.py
testing/web-platform/harness/wptrunner/executors/base.py
testing/web-platform/harness/wptrunner/executors/executormarionette.py
testing/web-platform/harness/wptrunner/executors/executorselenium.py
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/executors/testharness_marionette.js
testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
testing/web-platform/harness/wptrunner/manifestexpected.py
testing/web-platform/harness/wptrunner/testharnessreport-servo.js
testing/web-platform/harness/wptrunner/testloader.py
testing/web-platform/harness/wptrunner/testrunner.py
testing/web-platform/harness/wptrunner/update/update.py
testing/web-platform/harness/wptrunner/wptrunner.py
testing/web-platform/harness/wptrunner/wpttest.py
--- a/testing/web-platform/harness/setup.py
+++ b/testing/web-platform/harness/setup.py
@@ -7,17 +7,17 @@ import os
 import sys
 import textwrap
 
 from setuptools import setup, find_packages
 
 here = os.path.split(__file__)[0]
 
 PACKAGE_NAME = 'wptrunner'
-PACKAGE_VERSION = '1.13'
+PACKAGE_VERSION = '1.14'
 
 # Dependencies
 with open(os.path.join(here, "requirements.txt")) as f:
     deps = f.read().splitlines()
 
 # Browser-specific requirements
 requirements_files = glob.glob(os.path.join(here, "requirements_*.txt"))
 
--- a/testing/web-platform/harness/wptrunner/browsers/b2g.py
+++ b/testing/web-platform/harness/wptrunner/browsers/b2g.py
@@ -36,27 +36,28 @@ def check_args(**kwargs):
     pass
 
 
 def browser_kwargs(test_environment, **kwargs):
     return {"prefs_root": kwargs["prefs_root"],
             "no_backup": kwargs.get("b2g_no_backup", False)}
 
 
-def executor_kwargs(test_type, http_server_url, cache_manager, **kwargs):
+def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
     timeout_multiplier = kwargs["timeout_multiplier"]
     if timeout_multiplier is None:
         timeout_multiplier = 2
 
+    executor_kwargs = {"server_config": server_config,
+                       "timeout_multiplier": timeout_multiplier,
+                       "close_after_done": False}
+
     if test_type == "reftest":
         executor_kwargs["cache_manager"] = cache_manager
 
-    executor_kwargs = {"http_server_url": http_server_url,
-                       "timeout_multiplier": timeout_multiplier,
-                       "close_after_done": False}
     return executor_kwargs
 
 
 def env_options():
     return {"host": "web-platform.test",
             "bind_hostname": "false",
             "test_server_port": False}
 
--- a/testing/web-platform/harness/wptrunner/browsers/chrome.py
+++ b/testing/web-platform/harness/wptrunner/browsers/chrome.py
@@ -23,20 +23,20 @@ def check_args(**kwargs):
     require_arg(kwargs, "binary")
 
 
 def browser_kwargs(**kwargs):
     return {"binary": kwargs["binary"],
             "webdriver_binary": kwargs["webdriver_binary"]}
 
 
-def executor_kwargs(test_type, http_server_url, cache_manager, **kwargs):
+def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
     from selenium.webdriver import DesiredCapabilities
 
-    executor_kwargs = base_executor_kwargs(test_type, http_server_url,
+    executor_kwargs = base_executor_kwargs(test_type, server_config,
                                            cache_manager, **kwargs)
     executor_kwargs["close_after_done"] = True
     executor_kwargs["capabilities"] = dict(DesiredCapabilities.CHROME.items() +
                                            {"chromeOptions":
                                             {"binary": kwargs["binary"]}}.items())
 
     return executor_kwargs
 
--- a/testing/web-platform/harness/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/harness/wptrunner/browsers/firefox.py
@@ -40,18 +40,18 @@ def browser_kwargs(**kwargs):
             "debug_args": kwargs["debug_args"],
             "interactive": kwargs["interactive"],
             "symbols_path": kwargs["symbols_path"],
             "stackwalk_binary": kwargs["stackwalk_binary"],
             "certutil_binary": kwargs["certutil_binary"],
             "ca_certificate_path": kwargs["ssl_env"].ca_cert_path()}
 
 
-def executor_kwargs(test_type, http_server_url, cache_manager, **kwargs):
-    executor_kwargs = base_executor_kwargs(test_type, http_server_url,
+def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
+    executor_kwargs = base_executor_kwargs(test_type, server_config,
                                            cache_manager, **kwargs)
     executor_kwargs["close_after_done"] = True
     return executor_kwargs
 
 
 def env_options():
     return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
--- a/testing/web-platform/harness/wptrunner/browsers/servo.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servo.py
@@ -25,18 +25,18 @@ def check_args(**kwargs):
 
 
 def browser_kwargs(**kwargs):
     return {"binary": kwargs["binary"],
             "debug_args": kwargs["debug_args"],
             "interactive": kwargs["interactive"]}
 
 
-def executor_kwargs(test_type, http_server_url, cache_manager, **kwargs):
-    rv = base_executor_kwargs(test_type, http_server_url,
+def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
+    rv = base_executor_kwargs(test_type, server_config,
                               cache_manager, **kwargs)
     rv["pause_after_test"] = kwargs["pause_after_test"]
     return rv
 
 def env_options():
     return {"host": "localhost",
             "bind_hostname": "true",
             "testharnessreport": "testharnessreport-servo.js"}
--- a/testing/web-platform/harness/wptrunner/environment.py
+++ b/testing/web-platform/harness/wptrunner/environment.py
@@ -65,54 +65,59 @@ def ssl_env(logger, **kwargs):
     ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
     return ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
 
 
 class TestEnvironmentError(Exception):
     pass
 
 
-def static_handler(path, format_args, content_type, **headers):
-    with open(path) as f:
-        data = f.read() % format_args
+class StaticHandler(object):
+    def __init__(self, path, format_args, content_type, **headers):
+        with open(path) as f:
+            self.data = f.read() % format_args
+
+        self.resp_headers = [("Content-Type", content_type)]
+        for k, v in headers.iteritems():
+            resp_headers.append((k.replace("_", "-"), v))
 
-    resp_headers = [("Content-Type", content_type)]
-    for k, v in headers.iteritems():
-        resp_headers.append((k.replace("_", "-"), v))
+        self.handler = serve.handlers.handler(self.handle_request)
 
-    @serve.handlers.handler
-    def func(request, response):
-        return resp_headers, data
+    def handle_request(self, request, response):
+        return self.resp_headers, self.data
 
-    return func
+    def __call__(self, request, response):
+        rv = self.handler(request, response)
+        return rv
 
 
 class TestEnvironment(object):
     def __init__(self, test_paths, ssl_env, pause_after_test, options):
         """Context manager that owns the test environment i.e. the http and
         websockets servers"""
         self.test_paths = test_paths
         self.ssl_env = ssl_env
         self.server = None
         self.config = None
         self.external_config = None
         self.pause_after_test = pause_after_test
         self.test_server_port = options.pop("test_server_port", True)
         self.options = options if options is not None else {}
 
         self.cache_manager = multiprocessing.Manager()
+        self.routes = self.get_routes()
 
     def __enter__(self):
         self.ssl_env.__enter__()
         self.cache_manager.__enter__()
         self.setup_server_logging()
-        self.setup_routes()
         self.config = self.load_config()
         serve.set_computed_defaults(self.config)
-        self.external_config, self.servers = serve.start(self.config, self.ssl_env)
+        self.external_config, self.servers = serve.start(self.config, self.ssl_env,
+                                                         self.routes)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.cache_manager.__exit__(exc_type, exc_val, exc_tb)
         self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
 
         for scheme, servers in self.servers.iteritems():
             for port, server in servers:
@@ -161,48 +166,51 @@ class TestEnvironment(object):
         try:
             #Set as the default logger for wptserve
             serve.set_logger(server_logger)
             serve.logger = server_logger
         except Exception:
             # This happens if logging has already been set up for wptserve
             pass
 
-    def setup_routes(self):
+    def get_routes(self):
+        routes = serve.default_routes()
         for path, format_args, content_type, route in [
-                ("testharness_runner.html", {}, "text/html", b"/testharness_runner.html"),
+                ("testharness_runner.html", {}, "text/html", "/testharness_runner.html"),
                 (self.options.get("testharnessreport", "testharnessreport.js"),
                  {"output": self.pause_after_test}, "text/javascript",
-                 b"/resources/testharnessreport.js")]:
-            handler = static_handler(os.path.join(here, path), format_args, content_type)
-            serve.routes.insert(0, (b"GET", route, handler))
+                 "/resources/testharnessreport.js")]:
+            handler = StaticHandler(os.path.join(here, path), format_args, content_type)
+            routes.insert(0, (b"GET", str(route), handler))
 
         for url, paths in self.test_paths.iteritems():
             if url == "/":
                 continue
 
             path = paths["tests_path"]
             url = "/%s/" % url.strip("/")
 
             for (method,
                  suffix,
-                 handler_cls) in [(serve.any_method,
+                 handler_cls) in [(b"*",
                                    b"*.py",
                                    serve.handlers.PythonScriptHandler),
                                   (b"GET",
                                    "*.asis",
                                    serve.handlers.AsIsHandler),
                                   (b"GET",
                                    "*",
                                    serve.handlers.FileHandler)]:
                 route = (method, b"%s%s" % (str(url), str(suffix)), handler_cls(path, url_base=url))
-                serve.routes.insert(-3, route)
+                routes.insert(-3, route)
 
         if "/" not in self.test_paths:
-            serve.routes = serve.routes[:-3]
+            routes = routes[:-3]
+
+        return routes
 
     def ensure_started(self):
         # Pause for a while to ensure that the server has a chance to start
         time.sleep(2)
         for scheme, servers in self.servers.iteritems():
             for port, server in servers:
                 if self.test_server_port:
                     s = socket.socket()
--- a/testing/web-platform/harness/wptrunner/executors/base.py
+++ b/testing/web-platform/harness/wptrunner/executors/base.py
@@ -1,39 +1,53 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import hashlib
 import json
 import os
 import traceback
+import urlparse
 from abc import ABCMeta, abstractmethod
-from multiprocessing import Manager
 
 from ..testrunner import Stop
 
 here = os.path.split(__file__)[0]
 
 
-def executor_kwargs(test_type, http_server_url, cache_manager, **kwargs):
+def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
     timeout_multiplier = kwargs["timeout_multiplier"]
     if timeout_multiplier is None:
         timeout_multiplier = 1
 
-    executor_kwargs = {"http_server_url": http_server_url,
+    executor_kwargs = {"server_config": server_config,
                        "timeout_multiplier": timeout_multiplier,
                        "debug_args": kwargs["debug_args"]}
 
     if test_type == "reftest":
         executor_kwargs["screenshot_cache"] = cache_manager.dict()
 
     return executor_kwargs
 
 
+def strip_server(url):
+    """Remove the scheme and netloc from a url, leaving only the path and any query
+    or fragment.
+
+    url - the url to strip
+
+    e.g. http://example.org:8000/tests?id=1#2 becomes /tests?id=1#2"""
+
+    url_parts = list(urlparse.urlsplit(url))
+    url_parts[0] = ""
+    url_parts[1] = ""
+    return urlparse.urlunsplit(url_parts)
+
+
 class TestharnessResultConverter(object):
     harness_codes = {0: "OK",
                      1: "ERROR",
                      2: "TIMEOUT"}
 
     test_codes = {0: "PASS",
                   1: "FAIL",
                   2: "TIMEOUT",
@@ -41,17 +55,17 @@ class TestharnessResultConverter(object)
 
     def __call__(self, test, result):
         """Convert a JSON result into a (TestResult, [SubtestResult]) tuple"""
         assert result["test"] == test.url, ("Got results from %s, expected %s" %
                                             (result["test"], test.url))
         harness_result = test.result_cls(self.harness_codes[result["status"]], result["message"])
         return (harness_result,
                 [test.subtest_result_cls(subtest["name"], self.test_codes[subtest["status"]],
-                                         subtest["message"]) for subtest in result["tests"]])
+                                         subtest["message"], subtest.get("stack", None)) for subtest in result["tests"]])
 testharness_result_converter = TestharnessResultConverter()
 
 
 def reftest_result_converter(self, test, result):
     return (test.result_cls(result["status"], result["message"],
                             extra=result.get("extra")), [])
 
 
@@ -61,34 +75,35 @@ class ExecutorException(Exception):
         self.message = message
 
 class TestExecutor(object):
     __metaclass__ = ABCMeta
 
     test_type = None
     convert_result = None
 
-    def __init__(self, browser, http_server_url, timeout_multiplier=1,
+    def __init__(self, browser, server_config, timeout_multiplier=1,
                  debug_args=None):
         """Abstract Base class for object that actually executes the tests in a
         specific browser. Typically there will be a different TestExecutor
         subclass for each test type and method of executing tests.
 
         :param browser: ExecutorBrowser instance providing properties of the
                         browser that will be tested.
-        :param http_server_url: Base url of the http server on which the tests
-                                are running.
+        :param server_config: Dictionary of wptserve server configuration of the
+                              form stored in TestEnvironment.external_config
         :param timeout_multiplier: Multiplier relative to base timeout to use
                                    when setting test timeout.
         """
         self.runner = None
         self.browser = browser
-        self.http_server_url = http_server_url
+        self.server_config = server_config
         self.timeout_multiplier = timeout_multiplier
         self.debug_args = debug_args
+        self.last_protocol = "http"
         self.protocol = None # This must be set in subclasses
 
     @property
     def logger(self):
         """StructuredLogger for this executor"""
         if self.runner is not None:
             return self.runner.logger
 
@@ -103,36 +118,55 @@ class TestExecutor(object):
     def teardown(self):
         """Run cleanup steps after tests have finished"""
         self.protocol.teardown()
 
     def run_test(self, test):
         """Run a particular test.
 
         :param test: The test to run"""
+
+        if test.protocol != self.last_protocol:
+            self.on_protocol_change(test.protocol)
+
         try:
             result = self.do_test(test)
         except Exception as e:
             result = self.result_from_exception(test, e)
 
         if result is Stop:
             return result
 
         if result[0].status == "ERROR":
             self.logger.debug(result[0].message)
+
+        self.last_protocol = test.protocol
+
         self.runner.send_message("test_ended", test, result)
 
+
+    def server_url(self, protocol):
+        return "%s://%s:%s" % (protocol,
+                               self.server_config["host"],
+                               self.server_config["ports"][protocol][0])
+
+    def test_url(self, test):
+        return urlparse.urljoin(self.server_url(test.protocol), test.url)
+
     @abstractmethod
     def do_test(self, test):
         """Test-type and protocol specific implmentation of running a
         specific test.
 
         :param test: The test to run."""
         pass
 
+    def on_protocol_change(self, new_protocol):
+        pass
+
     def result_from_exception(self, test, e):
         if hasattr(e, "status") and e.status in test.result_cls.statuses:
             status = e.status
         else:
             status = "ERROR"
         message = unicode(getattr(e, "message", ""))
         if message:
             message += "\n"
@@ -142,19 +176,19 @@ class TestExecutor(object):
 
 class TestharnessExecutor(TestExecutor):
     convert_result = testharness_result_converter
 
 
 class RefTestExecutor(TestExecutor):
     convert_result = reftest_result_converter
 
-    def __init__(self, browser, http_server_url, timeout_multiplier=1, screenshot_cache=None,
+    def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None,
                  debug_args=None):
-        TestExecutor.__init__(self, browser, http_server_url,
+        TestExecutor.__init__(self, browser, server_config,
                               timeout_multiplier=timeout_multiplier,
                               debug_args=debug_args)
 
         self.screenshot_cache = screenshot_cache
 
 class RefTestImplementation(object):
     def __init__(self, executor):
         self.timeout_multiplier = executor.timeout_multiplier
@@ -165,35 +199,35 @@ class RefTestImplementation(object):
         # retrieve the screenshot from the cache directly in the future
         self.screenshot_cache = self.executor.screenshot_cache
         self.message = None
 
     @property
     def logger(self):
         return self.executor.logger
 
-    def get_hash(self, url, timeout):
-        timeout = timeout * self.timeout_multiplier
+    def get_hash(self, test):
+        timeout = test.timeout * self.timeout_multiplier
 
-        if url not in self.screenshot_cache:
-            success, data = self.executor.screenshot(url, timeout)
+        if test.url not in self.screenshot_cache:
+            success, data = self.executor.screenshot(test)
 
             if not success:
                 return False, data
 
             screenshot = data
             hash_value = hashlib.sha1(screenshot).hexdigest()
 
-            self.screenshot_cache[url] = (hash_value, None)
+            self.screenshot_cache[test.url] = (hash_value, None)
 
             rv = True, (hash_value, screenshot)
         else:
-            rv = True, self.screenshot_cache[url]
+            rv = True, self.screenshot_cache[test.url]
 
-        self.message.append("%s %s" % (url, rv[1][0]))
+        self.message.append("%s %s" % (test.url, rv[1][0]))
         return rv
 
     def is_pass(self, lhs_hash, rhs_hash, relation):
         assert relation in ("==", "!=")
         self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash))
         return ((relation == "==" and lhs_hash == rhs_hash) or
                 (relation == "!=" and lhs_hash != rhs_hash))
 
@@ -206,58 +240,57 @@ class RefTestImplementation(object):
         stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
         while stack:
             hashes = [None, None]
             screenshots = [None, None]
 
             nodes, relation = stack.pop()
 
             for i, node in enumerate(nodes):
-                success, data = self.get_hash(node.url, node.timeout)
+                success, data = self.get_hash(node)
                 if success is False:
                     return {"status": data[0], "message": data[1]}
 
                 hashes[i], screenshots[i] = data
 
             if self.is_pass(hashes[0], hashes[1], relation):
                 if nodes[1].references:
                     stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
                 else:
                     # We passed
                     return {"status":"PASS", "message": None}
 
+        # We failed, so construct a failure message
+
         for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
             if screenshot is None:
                 success, screenshot = self.retake_screenshot(node)
                 if success:
                     screenshots[i] = screenshot
 
         log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation,
                     {"url": nodes[1].url, "screenshot": screenshots[1]}]
 
         return {"status": "FAIL",
                 "message": "\n".join(self.message),
                 "extra": {"reftest_screenshots": log_data}}
 
     def retake_screenshot(self, node):
-        success, data = self.executor.screenshot(node.url,
-                                                 node.timeout *
-                                                 self.timeout_multiplier)
+        success, data = self.executor.screenshot(node)
         if not success:
             return False, data
 
         hash_val, _ = self.screenshot_cache[node.url]
         self.screenshot_cache[node.url] = hash_val, data
         return True, data
 
 class Protocol(object):
-    def __init__(self, executor, browser, http_server_url):
+    def __init__(self, executor, browser):
         self.executor = executor
         self.browser = browser
-        self.http_server_url = http_server_url
 
     @property
     def logger(self):
         return self.executor.logger
 
     def setup(self, runner):
         pass
 
--- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -19,17 +19,18 @@ here = os.path.join(os.path.split(__file
 
 from .base import (ExecutorException,
                    Protocol,
                    RefTestExecutor,
                    RefTestImplementation,
                    TestExecutor,
                    TestharnessExecutor,
                    testharness_result_converter,
-                   reftest_result_converter)
+                   reftest_result_converter,
+                   strip_server)
 from ..testrunner import Stop
 
 # Extra timeout to use after internal test timeout at which the harness
 # should force a timeout
 extra_timeout = 5 # seconds
 
 def do_delayed_imports():
     global marionette
@@ -37,20 +38,20 @@ def do_delayed_imports():
     try:
         import marionette
         from marionette import errors
     except ImportError:
         from marionette_driver import marionette, errors
 
 
 class MarionetteProtocol(Protocol):
-    def __init__(self, executor, browser, http_server_url):
+    def __init__(self, executor, browser):
         do_delayed_imports()
 
-        Protocol.__init__(self, executor, browser, http_server_url)
+        Protocol.__init__(self, executor, browser)
         self.marionette = None
         self.marionette_port = browser.marionette_port
 
     def setup(self, runner):
         """Connect to browser via Marionette."""
         Protocol.setup(self, runner)
 
         self.logger.debug("Connecting to marionette on port %i" % self.marionette_port)
@@ -101,18 +102,22 @@ class MarionetteProtocol(Protocol):
         try:
             # Get a simple property over the connection
             self.marionette.current_window_handle
         except Exception:
             return False
         return True
 
     def after_connect(self):
-        url = urlparse.urljoin(
-            self.http_server_url, "/testharness_runner.html")
+        self.load_runner("http")
+
+    def load_runner(self, protocol):
+        # Check if we previously had a test window open, and if we did make sure it's closed
+        self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
+        url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
         self.logger.debug("Loading %s" % url)
         try:
             self.marionette.navigate(url)
         except Exception as e:
             self.logger.critical(
                 "Loading initial page %s failed. Ensure that the "
                 "there are no other programs bound to this port and "
                 "that your firewall rules or network setup does not "
@@ -192,77 +197,81 @@ class MarionetteRun(object):
             message += traceback.format_exc(e)
             self.result = False, ("ERROR", e)
 
         finally:
             self.result_flag.set()
 
 
 class MarionetteTestharnessExecutor(TestharnessExecutor):
-    def __init__(self, browser, http_server_url, timeout_multiplier=1, close_after_done=True,
+    def __init__(self, browser, server_config, timeout_multiplier=1, close_after_done=True,
                  debug_args=None):
         """Marionette-based executor for testharness.js tests"""
-        TestharnessExecutor.__init__(self, browser, http_server_url,
+        TestharnessExecutor.__init__(self, browser, server_config,
                                      timeout_multiplier=timeout_multiplier,
                                      debug_args=debug_args)
 
-        self.protocol = MarionetteProtocol(self, browser, http_server_url)
+        self.protocol = MarionetteProtocol(self, browser)
         self.script = open(os.path.join(here, "testharness_marionette.js")).read()
         self.close_after_done = close_after_done
         self.window_id = str(uuid.uuid4())
 
         if marionette is None:
             do_delayed_imports()
 
     def is_alive(self):
         return self.protocol.is_alive()
 
+    def on_protocol_change(self, new_protocol):
+        self.protocol.load_runner(new_protocol)
+
     def do_test(self, test):
         timeout = (test.timeout * self.timeout_multiplier if self.debug_args is None
                    else None)
+
         success, data = MarionetteRun(self.logger,
                                       self.do_testharness,
                                       self.protocol.marionette,
-                                      test.url,
+                                      self.test_url(test),
                                       timeout).run()
         if success:
             return self.convert_result(test, data)
 
         return (test.result_cls(*data), [])
 
     def do_testharness(self, marionette, url, timeout):
         if self.close_after_done:
             marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
 
         if timeout is not None:
             timeout_ms = str(timeout * 1000)
         else:
             timeout_ms = "null"
 
-        script = self.script % {"abs_url": urlparse.urljoin(self.http_server_url, url),
-                                "url": url,
+        script = self.script % {"abs_url": url,
+                                "url": strip_server(url),
                                 "window_id": self.window_id,
                                 "timeout_multiplier": self.timeout_multiplier,
                                 "timeout": timeout_ms,
                                 "explicit_timeout": timeout is None}
 
         return marionette.execute_async_script(script, new_sandbox=False)
 
 
 class MarionetteRefTestExecutor(RefTestExecutor):
-    def __init__(self, browser, http_server_url, timeout_multiplier=1,
+    def __init__(self, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True, debug_args=None):
         """Marionette-based executor for reftests"""
         RefTestExecutor.__init__(self,
                                  browser,
-                                 http_server_url,
+                                 server_config,
                                  screenshot_cache=screenshot_cache,
                                  timeout_multiplier=timeout_multiplier,
                                  debug_args=debug_args)
-        self.protocol = MarionetteProtocol(self, browser, http_server_url)
+        self.protocol = MarionetteProtocol(self, browser)
         self.implementation = RefTestImplementation(self)
         self.close_after_done = close_after_done
         self.has_window = False
 
         with open(os.path.join(here, "reftest.js")) as f:
             self.script = f.read()
         with open(os.path.join(here, "reftest-wait.js")) as f:
             self.wait_script = f.read()
@@ -281,31 +290,32 @@ class MarionetteRefTestExecutor(RefTestE
             self.protocol.marionette.execute_script(self.script)
             self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
             self.has_window = True
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
-    def screenshot(self, url, timeout):
-        timeout = timeout if self.debug_args is None else None
+    def screenshot(self, test):
+        timeout = test.timeout if self.debug_args is None else None
+
+        test_url = self.test_url(test)
 
         return MarionetteRun(self.logger,
                              self._screenshot,
                              self.protocol.marionette,
-                             url,
+                             test_url,
                              timeout).run()
 
     def _screenshot(self, marionette, url, timeout):
-        full_url = urlparse.urljoin(self.http_server_url, url)
         try:
-            marionette.navigate(full_url)
+            marionette.navigate(url)
         except errors.MarionetteException:
-            raise ExecutorException("ERROR", "Failed to load url %s" % (full_url,))
+            raise ExecutorException("ERROR", "Failed to load url %s" % (url,))
 
         marionette.execute_async_script(self.wait_script)
 
         screenshot = marionette.screenshot()
         # strip off the data:img/png, part of the url
         if screenshot.startswith("data:image/png;base64,"):
             screenshot = screenshot.split(",", 1)[1]
 
--- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
@@ -13,17 +13,18 @@ import uuid
 
 from .base import (ExecutorException,
                    Protocol,
                    RefTestExecutor,
                    RefTestImplementation,
                    TestExecutor,
                    TestharnessExecutor,
                    testharness_result_converter,
-                   reftest_result_converter)
+                   reftest_result_converter,
+                   strip_server)
 from ..testrunner import Stop
 
 
 here = os.path.join(os.path.split(__file__)[0])
 
 webdriver = None
 exceptions = None
 
@@ -32,20 +33,20 @@ extra_timeout = 5
 def do_delayed_imports():
     global webdriver
     global exceptions
     from selenium import webdriver
     from selenium.common import exceptions
 
 
 class SeleniumProtocol(Protocol):
-    def __init__(self, executor, browser, http_server_url, capabilities, **kwargs):
+    def __init__(self, executor, browser, capabilities, **kwargs):
         do_delayed_imports()
 
-        Protocol.__init__(self, executor, browser, http_server_url)
+        Protocol.__init__(self, executor, browser)
         self.capabilities = capabilities
         self.url = browser.webdriver_url
         self.webdriver = None
 
     def setup(self, runner):
         """Connect to browser via Selenium's WebDriver implementation."""
         self.runner = runner
         self.logger.debug("Connecting to Selenium on URL: %s" % self.url)
@@ -88,17 +89,21 @@ class SeleniumProtocol(Protocol):
             # Get a simple property over the connection
             self.webdriver.current_window_handle
         # TODO what exception?
         except (socket.timeout, exceptions.ErrorInResponseException):
             return False
         return True
 
     def after_connect(self):
-        url = urlparse.urljoin(self.http_server_url, "/testharness_runner.html")
+        self.load_runner("http")
+
+    def load_runner(self, protocol):
+        url = urlparse.urljoin(self.executor.server_url(protocol),
+                               "/testharness_runner.html")
         self.logger.debug("Loading %s" % url)
         self.webdriver.get(url)
         self.webdriver.execute_script("document.title = '%s'" %
                                       threading.current_thread().name.replace("'", '"'))
 
     def wait(self):
         while True:
             try:
@@ -154,59 +159,67 @@ class SeleniumRun(object):
                 message += "\n"
             message += traceback.format_exc(e)
             self.result = False, ("ERROR", e)
         finally:
             self.result_flag.set()
 
 
 class SeleniumTestharnessExecutor(TestharnessExecutor):
-    def __init__(self, browser, http_server_url, timeout_multiplier=1,
+    def __init__(self, browser, server_config, timeout_multiplier=1,
                  close_after_done=True, capabilities=None, debug_args=None):
         """Selenium-based executor for testharness.js tests"""
-        TestharnessExecutor.__init__(self, browser, http_server_url,
+        TestharnessExecutor.__init__(self, browser, server_config,
                                      timeout_multiplier=timeout_multiplier,
                                      debug_args=debug_args)
-        self.protocol = SeleniumProtocol(self, browser, http_server_url, capabilities)
+        self.protocol = SeleniumProtocol(self, browser, capabilities)
         with open(os.path.join(here, "testharness_webdriver.js")) as f:
             self.script = f.read()
         self.close_after_done = close_after_done
         self.window_id = str(uuid.uuid4())
 
     def is_alive(self):
         return self.protocol.is_alive()
 
+    def on_protocol_change(self, new_protocol):
+        self.protocol.load_runner(new_protocol)
+
     def do_test(self, test):
-        success, data = SeleniumRun(self.do_testharness, self.protocol.webdriver,
-                                    test.url, test.timeout * self.timeout_multiplier).run()
+        url = self.test_url(test)
+
+        success, data = SeleniumRun(self.do_testharness,
+                                    self.protocol.webdriver,
+                                    url,
+                                    test.timeout * self.timeout_multiplier).run()
+
         if success:
             return self.convert_result(test, data)
 
         return (test.result_cls(*data), [])
 
     def do_testharness(self, webdriver, url, timeout):
         return webdriver.execute_async_script(
-            self.script % {"abs_url": urlparse.urljoin(self.http_server_url, url),
-                           "url": url,
+            self.script % {"abs_url": url,
+                           "url": strip_server(url),
                            "window_id": self.window_id,
                            "timeout_multiplier": self.timeout_multiplier,
                            "timeout": timeout * 1000})
 
 class SeleniumRefTestExecutor(RefTestExecutor):
-    def __init__(self, browser, http_server_url, timeout_multiplier=1,
+    def __init__(self, browser, server_config, timeout_multiplier=1,
                  screenshot_cache=None, close_after_done=True,
                  debug_args=None, capabilities=None):
         """Selenium WebDriver-based executor for reftests"""
         RefTestExecutor.__init__(self,
                                  browser,
-                                 http_server_url,
+                                 server_config,
                                  screenshot_cache=screenshot_cache,
                                  timeout_multiplier=timeout_multiplier,
                                  debug_args=debug_args)
-        self.protocol = SeleniumProtocol(self, browser, http_server_url,
+        self.protocol = SeleniumProtocol(self, browser,
                                          capabilities=capabilities)
         self.implementation = RefTestImplementation(self)
         self.close_after_done = close_after_done
         self.has_window = False
 
         with open(os.path.join(here, "reftest.js")) as f:
             self.script = f.read()
         with open(os.path.join(here, "reftest-wait_webdriver.js")) as f:
@@ -229,23 +242,24 @@ class SeleniumRefTestExecutor(RefTestExe
             self.protocol.webdriver.switch_to_window(
                 self.protocol.webdriver.window_handles[-1])
             self.has_window = True
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
-    def screenshot(self, url, timeout):
-        return SeleniumRun(self._screenshot, self.protocol.webdriver,
-                           url, timeout).run()
+    def screenshot(self, test):
+        return SeleniumRun(self._screenshot,
+                           self.protocol.webdriver,
+                           self.test_url(test),
+                           test.timeout).run()
 
     def _screenshot(self, webdriver, url, timeout):
-        full_url = urlparse.urljoin(self.http_server_url, url)
-        webdriver.get(full_url)
+        webdriver.get(url)
 
         webdriver.execute_async_script(self.wait_script)
 
         screenshot = webdriver.get_screenshot_as_base64()
 
         # strip off the data:img/png, part of the url
         if screenshot.startswith("data:image/png;base64,"):
             screenshot = screenshot.split(",", 1)[1]
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -20,32 +20,31 @@ from .base import (ExecutorException,
                    testharness_result_converter,
                    reftest_result_converter)
 from .process import ProcessTestExecutor
 
 
 class ServoTestharnessExecutor(ProcessTestExecutor):
     convert_result = testharness_result_converter
 
-    def __init__(self, browser, http_server_url, timeout_multiplier=1, debug_args=None,
+    def __init__(self, browser, server_config, timeout_multiplier=1, debug_args=None,
                  pause_after_test=False):
-        ProcessTestExecutor.__init__(self, browser, http_server_url,
+        ProcessTestExecutor.__init__(self, browser, server_config,
                                      timeout_multiplier=timeout_multiplier,
                                      debug_args=debug_args)
         self.pause_after_test = pause_after_test
         self.result_data = None
         self.result_flag = None
-        self.protocol = Protocol(self, browser, http_server_url)
+        self.protocol = Protocol(self, browser)
 
     def do_test(self, test):
         self.result_data = None
         self.result_flag = threading.Event()
 
-        self.command = [self.binary, "--cpu", "--hard-fail", "-z",
-                        urlparse.urljoin(self.http_server_url, test.url)]
+        self.command = [self.binary, "--cpu", "--hard-fail", "-z", self.test_url(test)]
 
         if self.pause_after_test:
             self.command.remove("-z")
 
         if self.debug_args:
             self.command = list(self.debug_args) + self.command
 
 
@@ -115,49 +114,49 @@ class TempFilename(object):
             os.unlink(self.path)
         except OSError:
             pass
 
 
 class ServoRefTestExecutor(ProcessTestExecutor):
     convert_result = reftest_result_converter
 
-    def __init__(self, browser, http_server_url, binary=None, timeout_multiplier=1,
+    def __init__(self, browser, server_config, binary=None, timeout_multiplier=1,
                  screenshot_cache=None, debug_args=None, pause_after_test=False):
         ProcessTestExecutor.__init__(self,
                                      browser,
-                                     http_server_url,
+                                     server_config,
                                      timeout_multiplier=timeout_multiplier,
                                      debug_args=debug_args)
 
-        self.protocol = Protocol(self, browser, http_server_url)
+        self.protocol = Protocol(self, browser)
         self.screenshot_cache = screenshot_cache
         self.implementation = RefTestImplementation(self)
         self.tempdir = tempfile.mkdtemp()
 
     def teardown(self):
         os.rmdir(self.tempdir)
         ProcessTestExecutor.teardown(self)
 
-    def screenshot(self, url, timeout):
-        full_url = urlparse.urljoin(self.http_server_url, url)
+    def screenshot(self, test):
+        full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
             self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
                             "--output=%s" % output_path, full_url]
 
             self.proc = ProcessHandler(self.command,
                                        processOutputLine=[self.on_output])
             self.proc.run()
-            rv = self.proc.wait(timeout=timeout)
+            rv = self.proc.wait(timeout=test.timeout)
             if rv is None:
                 self.proc.kill()
                 return False, ("EXTERNAL-TIMEOUT", None)
 
-            if rv < 0:
+            if rv != 0 or not os.path.exists(output_path):
                 return False, ("CRASH", None)
 
             with open(output_path) as f:
                 # Might need to strip variable headers or something here
                 data = f.read()
                 return True, data
 
     def do_test(self, test):
--- a/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js
+++ b/testing/web-platform/harness/wptrunner/executors/testharness_marionette.js
@@ -3,22 +3,23 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 window.wrappedJSObject.timeout_multiplier = %(timeout_multiplier)d;
 window.wrappedJSObject.explicit_timeout = %(explicit_timeout)d;
 
 window.wrappedJSObject.done = function(tests, status) {
   clearTimeout(timer);
   var test_results = tests.map(function(x) {
-    return {name:x.name, status:x.status, message:x.message}
+    return {name:x.name, status:x.status, message:x.message, stack:x.stack}
   });
   marionetteScriptFinished({test:"%(url)s",
                             tests:test_results,
                             status: status.status,
-                            message: status.message});
+                            message: status.message,
+                            stack: status.stack});
 }
 
 window.wrappedJSObject.win = window.open("%(abs_url)s", "%(window_id)s");
 
 var timer = null;
 if (%(timeout)s) {
   timer = setTimeout(function() {
       log("Timeout fired");
--- a/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
+++ b/testing/web-platform/harness/wptrunner/executors/testharness_webdriver.js
@@ -3,22 +3,23 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 var callback = arguments[arguments.length - 1];
 window.timeout_multiplier = %(timeout_multiplier)d;
 
 window.done = function(tests, status) {
   clearTimeout(timer);
   var test_results = tests.map(function(x) {
-    return {name:x.name, status:x.status, message:x.message}
+    return {name:x.name, status:x.status, message:x.message, stack:x.stack}
   });
   callback({test:"%(url)s",
             tests:test_results,
             status: status.status,
-            message: status.message});
+            message: status.message,
+            stack: status.stack});
 }
 
 window.win = window.open("%(abs_url)s", "%(window_id)s");
 
 var timer = setTimeout(function() {
   window.win.timeout();
   window.win.close();
 }, %(timeout)s);
--- a/testing/web-platform/harness/wptrunner/manifestexpected.py
+++ b/testing/web-platform/harness/wptrunner/manifestexpected.py
@@ -49,17 +49,16 @@ class ExpectedManifest(ManifestItem):
         self.child_map = {}
         self.test_path = test_path
         self.url_base = url_base
 
     def append(self, child):
         """Add a test to the manifest"""
         ManifestItem.append(self, child)
         self.child_map[child.id] = child
-        #assert len(self.child_map) == len(self.children), "%r %r" % (self.child_map, self.children)
 
     def _remove_child(self, child):
         del self.child_map[child.id]
         ManifestItem.remove_child(self, child)
         assert len(self.child_map) == len(self.children)
 
     def get_test(self, test_id):
         """Get a test from the manifest by ID
--- a/testing/web-platform/harness/wptrunner/testharnessreport-servo.js
+++ b/testing/web-platform/harness/wptrunner/testharnessreport-servo.js
@@ -4,14 +4,15 @@
 
 var props = {output:%(output)d};
 
 setup(props);
 
 add_completion_callback(function (tests, harness_status) {
     alert("RESULT: " + JSON.stringify({
         tests: tests.map(function(t) {
-            return { name: t.name, status: t.status, message: t.message }
+            return { name: t.name, status: t.status, message: t.message, stack: t.stack}
         }),
         status: harness_status.status,
         message: harness_status.message,
+        stack: harness_status.stack,
     }));
 });
--- a/testing/web-platform/harness/wptrunner/testloader.py
+++ b/testing/web-platform/harness/wptrunner/testloader.py
@@ -250,19 +250,23 @@ class ManifestLoader(object):
                     json_data = json.load(f)
             except IOError:
                 #If the existing file doesn't exist just create one from scratch
                 pass
 
         if not json_data:
             manifest_file = manifest.Manifest(None, url_base)
         else:
-            manifest_file = manifest.Manifest.from_json(tests_path, json_data)
+            try:
+                manifest_file = manifest.Manifest.from_json(tests_path, json_data)
+            except manifest.ManifestVersionMismatch:
+                manifest_file = manifest.Manifest(None, url_base)
 
-        manifest_update.update(tests_path, url_base, manifest_file)
+            manifest_update.update(tests_path, url_base, manifest_file)
+
         manifest.write(manifest_file, manifest_path)
 
     def load_manifest(self, tests_path, metadata_path, url_base="/"):
         manifest_path = os.path.join(metadata_path, "MANIFEST.json")
         if (not os.path.exists(manifest_path) or
             self.force_manifest_update):
             self.update_manifest(manifest_path, tests_path, url_base)
         manifest_file = manifest.load(tests_path, manifest_path)
@@ -278,24 +282,25 @@ class TestLoader(object):
     def __init__(self,
                  test_manifests,
                  test_types,
                  test_filter,
                  run_info,
                  chunk_type="none",
                  total_chunks=1,
                  chunk_number=1,
-                 force_manifest_update=False):
+                 include_https=True):
 
         self.test_types = test_types
         self.test_filter = test_filter
         self.run_info = run_info
         self.manifests = test_manifests
         self.tests = None
         self.disabled_tests = None
+        self.include_https = include_https
 
         self.chunk_type = chunk_type
         self.total_chunks = total_chunks
         self.chunk_number = chunk_number
 
         self.chunker = {"none": Unchunked,
                         "hash": HashChunker,
                         "equal_time": EqualTimeChunker}[chunk_type](total_chunks,
@@ -344,17 +349,20 @@ class TestLoader(object):
                 yield test_path, test_type, test
 
     def _load_tests(self):
         """Read in the tests from the manifest file and add them to a queue"""
         tests = {"enabled":defaultdict(list),
                  "disabled":defaultdict(list)}
 
         for test_path, test_type, test in self.iter_tests():
-            key = "enabled" if not test.disabled() else "disabled"
+            enabled = not test.disabled()
+            if not self.include_https and test.protocol == "https":
+                enabled = False
+            key = "enabled" if enabled else "disabled"
             tests[key][test_type].append(test)
 
         self.tests = tests["enabled"]
         self.disabled_tests = tests["disabled"]
 
     def groups(self, test_types, chunk_type="none", total_chunks=1, chunk_number=1):
         groups = set()
 
--- a/testing/web-platform/harness/wptrunner/testrunner.py
+++ b/testing/web-platform/harness/wptrunner/testrunner.py
@@ -493,17 +493,18 @@ class TestRunnerManager(threading.Thread
             if is_unexpected:
                 self.unexpected_count += 1
                 self.logger.debug("Unexpected count in this thread %i" % self.unexpected_count)
                 subtest_unexpected = True
             self.logger.test_status(test.id,
                                     result.name,
                                     result.status,
                                     message=result.message,
-                                    expected=expected)
+                                    expected=expected,
+                                    stack=result.stack)
 
         # TODO: consider changing result if there is a crash dump file
 
         # Write the result of the test harness
         expected = test.expected()
         status = file_result.status if file_result.status != "EXTERNAL-TIMEOUT" else "TIMEOUT"
         is_unexpected = expected != status
         if is_unexpected:
--- a/testing/web-platform/harness/wptrunner/update/update.py
+++ b/testing/web-platform/harness/wptrunner/update/update.py
@@ -4,22 +4,22 @@
 
 import os
 import sys
 
 from metadata import MetadataUpdateRunner
 from sync import SyncFromUpstreamRunner
 from tree import GitTree, HgTree, NoVCSTree
 
-from .. import wptrunner
+from .. import environment as env
 from base import Step, StepRunner, exit_clean, exit_unclean
 from state import State
 
-def setup_paths(serve_root):
-    wptrunner.do_delayed_imports(serve_root)
+def setup_paths(logger, test_paths):
+    env.do_delayed_imports(logger, test_paths)
 
 class LoadConfig(Step):
     """Step for loading configuration from the ini file and kwargs."""
 
     provides = ["sync", "paths", "metadata_path", "tests_path"]
 
     def create(self, state):
         state.sync = {"remote_url": state.kwargs["remote_url"],
@@ -105,17 +105,17 @@ class WPTUpdate(object):
 
         :param runner_cls: Runner subclass holding the overall list of
                            steps to run.
         :param kwargs: Command line arguments
         """
         self.runner_cls = runner_cls
         self.serve_root = kwargs["test_paths"]["/"]["tests_path"]
         #This must be before we try to reload state
-        setup_paths(self.serve_root)
+        setup_paths(logger, kwargs["test_paths"])
 
         self.state = State(logger)
         self.kwargs = kwargs
         self.logger = logger
 
     def run(self, **kwargs):
         if self.kwargs["abort"]:
             self.abort()
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -35,51 +35,56 @@ The upstream repository has the facility
 format. This manifest is used directly to determine which tests exist. Local
 metadata files are used to store the expected test results.
 """
 
 def setup_logging(*args, **kwargs):
     global logger
     logger = wptlogging.setup(*args, **kwargs)
 
-def get_loader(test_paths, product, debug=False, **kwargs):
+def get_loader(test_paths, product, ssl_env, debug=False, **kwargs):
     run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
 
     test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load()
 
     test_filter = testloader.TestFilter(include=kwargs["include"],
                                         exclude=kwargs["exclude"],
                                         manifest_path=kwargs["include_manifest"],
                                         test_manifests=test_manifests)
 
     test_loader = testloader.TestLoader(test_manifests,
                                         kwargs["test_types"],
                                         test_filter,
                                         run_info,
                                         chunk_type=kwargs["chunk_type"],
                                         total_chunks=kwargs["total_chunks"],
-                                        chunk_number=kwargs["this_chunk"])
+                                        chunk_number=kwargs["this_chunk"],
+                                        include_https=ssl_env.ssl_enabled)
     return run_info, test_loader
 
 def list_test_groups(test_paths, product, **kwargs):
     env.do_delayed_imports(logger, test_paths)
 
-    run_info, test_loader = get_loader(test_paths, product,
+    ssl_env = env.ssl_env(logger, **kwargs)
+
+    run_info, test_loader = get_loader(test_paths, product, ssl_env,
                                        **kwargs)
 
     for item in sorted(test_loader.groups(kwargs["test_types"])):
         print item
 
 
 def list_disabled(test_paths, product, **kwargs):
     env.do_delayed_imports(logger, test_paths)
 
     rv = []
 
-    run_info, test_loader = get_loader(test_paths, product,
+    ssl_env = env.ssl_env(logger, **kwargs)
+
+    run_info, test_loader = get_loader(test_paths, product, ssl_env,
                                        **kwargs)
 
     for test_type, tests in test_loader.disabled_tests.iteritems():
         for test in tests:
             rv.append({"test": test.id, "reason": test.disabled()})
     print json.dumps(rv, indent=2)
 
 
@@ -104,17 +109,17 @@ def run_tests(config, test_paths, produc
         ssl_env = env.ssl_env(logger, **kwargs)
 
         check_args(**kwargs)
 
         if "test_loader" in kwargs:
             run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
             test_loader = kwargs["test_loader"]
         else:
-            run_info, test_loader = get_loader(test_paths, product,
+            run_info, test_loader = get_loader(test_paths, product, ssl_env,
                                                **kwargs)
 
         if kwargs["run_by_dir"] is False:
             test_source_cls = testloader.SingleTestSource
             test_source_kwargs = {}
         else:
             # A value of None indicates infinite depth
             test_source_cls = testloader.PathGroupedSource
@@ -132,18 +137,16 @@ def run_tests(config, test_paths, produc
                                  env_options) as test_environment:
             try:
                 test_environment.ensure_started()
             except env.TestEnvironmentError as e:
                 logger.critical("Error starting test environment: %s" % e.message)
                 raise
 
             browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
-            base_server = "http://%s:%i" % (test_environment.external_config["host"],
-                                            test_environment.external_config["ports"]["http"][0])
 
             repeat = kwargs["repeat"]
             for repeat_count in xrange(repeat):
                 if repeat > 1:
                     logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
 
 
                 unexpected_count = 0
@@ -152,17 +155,17 @@ def run_tests(config, test_paths, produc
                     logger.info("Running %s tests" % test_type)
 
                     for test in test_loader.disabled_tests[test_type]:
                         logger.test_start(test.id)
                         logger.test_end(test.id, status="SKIP")
 
                     executor_cls = executor_classes.get(test_type)
                     executor_kwargs = get_executor_kwargs(test_type,
-                                                          base_server,
+                                                          test_environment.external_config,
                                                           test_environment.cache_manager,
                                                           **kwargs)
 
                     if executor_cls is None:
                         logger.error("Unsupported test type %s for product %s" %
                                      (test_type, product))
                         continue
 
--- a/testing/web-platform/harness/wptrunner/wpttest.py
+++ b/testing/web-platform/harness/wptrunner/wpttest.py
@@ -16,22 +16,23 @@ class Result(object):
             raise ValueError("Unrecognised status %s" % status)
         self.status = status
         self.message = message
         self.expected = expected
         self.extra = extra
 
 
 class SubtestResult(object):
-    def __init__(self, name, status, message, expected=None):
+    def __init__(self, name, status, message, stack=None, expected=None):
         self.name = name
         if status not in self.statuses:
             raise ValueError("Unrecognised status %s" % status)
         self.status = status
         self.message = message
+        self.stack = stack
         self.expected = expected
 
 
 class TestharnessResult(Result):
     default_expected = "OK"
     statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
 
 
@@ -78,32 +79,35 @@ class B2GRunInfo(RunInfo):
         RunInfo.__init__(self, *args, **kwargs)
         self["os"] = "b2g"
 
 
 class Test(object):
     result_cls = None
     subtest_result_cls = None
 
-    def __init__(self, url, expected_metadata, timeout=DEFAULT_TIMEOUT, path=None):
+    def __init__(self, url, expected_metadata, timeout=DEFAULT_TIMEOUT, path=None,
+                 protocol="http"):
         self.url = url
         self._expected_metadata = expected_metadata
         self.timeout = timeout
         self.path = path
+        self.protocol = protocol
 
     def __eq__(self, other):
         return self.id == other.id
 
     @classmethod
     def from_manifest(cls, manifest_item, expected_metadata):
         timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT
         return cls(manifest_item.url,
                    expected_metadata,
                    timeout=timeout,
-                   path=manifest_item.path)
+                   path=manifest_item.path,
+                   protocol="https" if manifest_item.https else "http")
 
 
     @property
     def id(self):
         return self.url
 
     @property
     def keys(self):
@@ -155,28 +159,30 @@ class ManualTest(Test):
     @property
     def id(self):
         return self.url
 
 
 class ReftestTest(Test):
     result_cls = ReftestResult
 
-    def __init__(self, url, expected, references, timeout=DEFAULT_TIMEOUT, path=None):
+    def __init__(self, url, expected, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
         self.url = url
         for _, ref_type in references:
             if ref_type not in ("==", "!="):
                 raise ValueError
         self._expected_metadata = expected
         self.timeout = timeout
         self.path = path
+        self.protocol = protocol
         self.references = references
 
     @classmethod
-    def from_manifest(cls, manifest_test,
+    def from_manifest(cls,
+                      manifest_test,
                       expected_metadata,
                       nodes=None,
                       references_seen=None):
 
         timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT
 
         if nodes is None:
             nodes = {}
@@ -184,17 +190,18 @@ class ReftestTest(Test):
             references_seen = set()
 
         url = manifest_test.url
 
         node = cls(manifest_test.url,
                    expected_metadata,
                    [],
                    timeout=timeout,
-                   path=manifest_test.path)
+                   path=manifest_test.path,
+                   protocol="https" if manifest_test.https else "http")
 
         nodes[url] = node
 
         for ref_url, ref_type in manifest_test.references:
             comparison_key = (ref_type,) + tuple(sorted([url, ref_url]))
             if ref_url in nodes:
                 manifest_node = ref_url
                 if comparison_key in references_seen: